From 6726d2933a25137f8dce639c53da8aa1752be47e Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Fri, 29 Apr 2022 11:46:59 +0200 Subject: Change half-life of trending status scores from 6 hours to 2 hours (#18182) --- app/models/trends/statuses.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/trends/statuses.rb b/app/models/trends/statuses.rb index d9b07446e..3013bc1d1 100644 --- a/app/models/trends/statuses.rb +++ b/app/models/trends/statuses.rb @@ -6,7 +6,7 @@ class Trends::Statuses < Trends::Base self.default_options = { threshold: 5, review_threshold: 3, - score_halflife: 6.hours.freeze, + score_halflife: 2.hours.freeze, } class Query < Trends::Query -- cgit From 6476f7e4da4da7c353d497aae5a86fc3909ce532 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Fri, 29 Apr 2022 22:42:42 +0200 Subject: Change trending statuses to only show one status from each account (#18181) Calculate trends in temporary sets to avoid having to manage items that go below the decay threshold while not having any moments where a half-processed set is accessible to end-users --- app/models/trends/base.rb | 37 +++++++++++++----------- app/models/trends/links.rb | 42 ++++++++++++--------------- app/models/trends/statuses.rb | 67 +++++++++++++++++++------------------------ app/models/trends/tags.rb | 17 +++++++---- 4 files changed, 81 insertions(+), 82 deletions(-) (limited to 'app/models') diff --git a/app/models/trends/base.rb b/app/models/trends/base.rb index 200f8d635..047111248 100644 --- a/app/models/trends/base.rb +++ b/app/models/trends/base.rb @@ -64,33 +64,38 @@ class Trends::Base redis.expire(used_key(at_time), 1.day.seconds) end - def trim_older_items - redis.zremrangebyscore("#{key_prefix}:all", '-inf', '(0.3') - redis.zremrangebyscore("#{key_prefix}:allowed", '-inf', '(0.3') - end - def score_at_rank(rank) redis.zrevrange("#{key_prefix}:allowed", 0, rank, with_scores: true).last&.last || 0 end - # @param [Integer] id - # @param [Float] score - # @param [Hash] subsets - def add_to_and_remove_from_subsets(id, score, subsets = {}) - subsets.each_key do |subset| - key = [key_prefix, subset].compact.join(':') + def replace_items(suffix, items) + tmp_prefix = "#{key_prefix}:tmp:#{SecureRandom.alphanumeric(6)}#{suffix}" + allowed_items = filter_for_allowed_items(items) + + redis.pipelined do |pipeline| + items.each { |item| pipeline.zadd("#{tmp_prefix}:all", item[:score], item[:item].id) } + allowed_items.each { |item| pipeline.zadd("#{tmp_prefix}:allowed", item[:score], item[:item].id) } - if score.positive? && subsets[subset] - redis.zadd(key, score, id) - else - redis.zrem(key, id) - end + rename_set(pipeline, "#{tmp_prefix}:all", "#{key_prefix}:all#{suffix}", items) + rename_set(pipeline, "#{tmp_prefix}:allowed", "#{key_prefix}:allowed#{suffix}", allowed_items) end end + def filter_for_allowed_items(items) + raise NotImplementedError + end + private def used_key(at_time) "#{key_prefix}:used:#{at_time.beginning_of_day.to_i}" end + + def rename_set(pipeline, from_key, to_key, set_items) + if set_items.empty? + pipeline.del(to_key) + else + pipeline.rename(from_key, to_key) + end + end end diff --git a/app/models/trends/links.rb b/app/models/trends/links.rb index 5f046643a..604894cd6 100644 --- a/app/models/trends/links.rb +++ b/app/models/trends/links.rb @@ -8,14 +8,15 @@ class Trends::Links < Trends::Base review_threshold: 3, max_score_cooldown: 2.days.freeze, max_score_halflife: 8.hours.freeze, + decay_threshold: 1, } def register(status, at_time = Time.now.utc) - original_status = status.reblog? ? status.reblog : status + original_status = status.proper - return unless original_status.public_visibility? && status.public_visibility? && - !original_status.account.silenced? && !status.account.silenced? && - !original_status.spoiler_text? + return unless (original_status.public_visibility? && status.public_visibility?) && + !(original_status.account.silenced? || status.account.silenced?) && + !(original_status.spoiler_text? || original_status.sensitive?) original_status.preview_cards.each do |preview_card| add(preview_card, status.account_id, at_time) if preview_card.appropriate_for_trends? @@ -61,6 +62,9 @@ class Trends::Links < Trends::Base private def calculate_scores(preview_cards, at_time) + global_items = [] + locale_items = Hash.new { |h, key| h[key] = [] } + preview_cards.each do |preview_card| expected = preview_card.history.get(at_time - 1.day).accounts.to_f expected = 1.0 if expected.zero? @@ -87,33 +91,23 @@ class Trends::Links < Trends::Base decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / options[:max_score_halflife].to_f)) - add_to_and_remove_from_subsets(preview_card.id, decaying_score, { - all: true, - allowed: preview_card.trendable?, - }) - - next unless valid_locale?(preview_card.language) + next unless decaying_score >= options[:decay_threshold] - add_to_and_remove_from_subsets(preview_card.id, decaying_score, { - "all:#{preview_card.language}" => true, - "allowed:#{preview_card.language}" => preview_card.trendable?, - }) + global_items << { score: decaying_score, item: preview_card } + locale_items[preview_card.language] << { score: decaying_score, item: preview_card } if valid_locale?(preview_card.language) end - trim_older_items - - # Clean up localized sets by calculating the intersection with the main - # set. We do this instead of just deleting the localized sets to avoid - # having moments where the API returns empty results + replace_items('', global_items) - redis.pipelined do - Trends.available_locales.each do |locale| - redis.zinterstore("#{key_prefix}:all:#{locale}", ["#{key_prefix}:all:#{locale}", "#{key_prefix}:all"], aggregate: 'max') - redis.zinterstore("#{key_prefix}:allowed:#{locale}", ["#{key_prefix}:allowed:#{locale}", "#{key_prefix}:allowed"], aggregate: 'max') - end + Trends.available_locales.each do |locale| + replace_items(":#{locale}", locale_items[locale]) end end + def filter_for_allowed_items(items) + items.select { |item| item[:item].trendable? } + end + def would_be_trending?(id) score(id) > score_at_rank(options[:review_threshold] - 1) end diff --git a/app/models/trends/statuses.rb b/app/models/trends/statuses.rb index 3013bc1d1..777065d3e 100644 --- a/app/models/trends/statuses.rb +++ b/app/models/trends/statuses.rb @@ -7,6 +7,7 @@ class Trends::Statuses < Trends::Base threshold: 5, review_threshold: 3, score_halflife: 2.hours.freeze, + decay_threshold: 0.3, } class Query < Trends::Query @@ -31,7 +32,7 @@ class Trends::Statuses < Trends::Base end def register(status, at_time = Time.now.utc) - add(status.proper, status.account_id, at_time) if eligible?(status) + add(status.proper, status.account_id, at_time) if eligible?(status.proper) end def add(status, _account_id, at_time = Time.now.utc) @@ -74,53 +75,45 @@ class Trends::Statuses < Trends::Base private def eligible?(status) - original_status = status.proper - - original_status.public_visibility? && - original_status.account.discoverable? && !original_status.account.silenced? && - original_status.spoiler_text.blank? && !original_status.sensitive? && !original_status.reply? + status.public_visibility? && status.account.discoverable? && !status.account.silenced? && status.spoiler_text.blank? && !status.sensitive? && !status.reply? end def calculate_scores(statuses, at_time) - redis.pipelined do - statuses.each do |status| - expected = 1.0 - observed = (status.reblogs_count + status.favourites_count).to_f - - score = begin - if expected > observed || observed < options[:threshold] - 0 - else - ((observed - expected)**2) / expected - end + global_items = [] + locale_items = Hash.new { |h, key| h[key] = [] } + + statuses.each do |status| + expected = 1.0 + observed = (status.reblogs_count + status.favourites_count).to_f + + score = begin + if expected > observed || observed < options[:threshold] + 0 + else + ((observed - expected)**2) / expected end + end - decaying_score = score * (0.5**((at_time.to_f - status.created_at.to_f) / options[:score_halflife].to_f)) + decaying_score = score * (0.5**((at_time.to_f - status.created_at.to_f) / options[:score_halflife].to_f)) - add_to_and_remove_from_subsets(status.id, decaying_score, { - all: true, - allowed: status.trendable? && status.account.discoverable?, - }) + next unless decaying_score >= options[:decay_threshold] - next unless valid_locale?(status.language) + global_items << { score: decaying_score, item: status } + locale_items[status.language] << { account_id: status.account_id, score: decaying_score, item: status } if valid_locale?(status.language) + end - add_to_and_remove_from_subsets(status.id, decaying_score, { - "all:#{status.language}" => true, - "allowed:#{status.language}" => status.trendable? && status.account.discoverable?, - }) - end + replace_items('', global_items) - trim_older_items + Trends.available_locales.each do |locale| + replace_items(":#{locale}", locale_items[locale]) + end + end - # Clean up localized sets by calculating the intersection with the main - # set. We do this instead of just deleting the localized sets to avoid - # having moments where the API returns empty results + def filter_for_allowed_items(items) + # Show only one status per account, pick the one with the highest score + # that's also eligible to trend - Trends.available_locales.each do |locale| - redis.zinterstore("#{key_prefix}:all:#{locale}", ["#{key_prefix}:all:#{locale}", "#{key_prefix}:all"], aggregate: 'max') - redis.zinterstore("#{key_prefix}:allowed:#{locale}", ["#{key_prefix}:allowed:#{locale}", "#{key_prefix}:allowed"], aggregate: 'max') - end - end + items.group_by { |item| item[:account_id] }.values.filter_map { |account_items| account_items.select { |item| item[:item].trendable? && item[:item].account.discoverable? }.max_by { |item| item[:score] } } end def would_be_trending?(id) diff --git a/app/models/trends/tags.rb b/app/models/trends/tags.rb index 3caa58815..19ade52ba 100644 --- a/app/models/trends/tags.rb +++ b/app/models/trends/tags.rb @@ -8,6 +8,7 @@ class Trends::Tags < Trends::Base review_threshold: 3, max_score_cooldown: 2.days.freeze, max_score_halflife: 4.hours.freeze, + decay_threshold: 1, } def register(status, at_time = Time.now.utc) @@ -26,7 +27,6 @@ class Trends::Tags < Trends::Base def refresh(at_time = Time.now.utc) tags = Tag.where(id: (recently_used_ids(at_time) + currently_trending_ids(false, -1)).uniq) calculate_scores(tags, at_time) - trim_older_items end def request_review @@ -53,6 +53,8 @@ class Trends::Tags < Trends::Base private def calculate_scores(tags, at_time) + items = [] + tags.each do |tag| expected = tag.history.get(at_time - 1.day).accounts.to_f expected = 1.0 if expected.zero? @@ -79,11 +81,16 @@ class Trends::Tags < Trends::Base decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / options[:max_score_halflife].to_f)) - add_to_and_remove_from_subsets(tag.id, decaying_score, { - all: true, - allowed: tag.trendable?, - }) + next unless decaying_score >= options[:decay_threshold] + + items << { score: decaying_score, item: tag } end + + replace_items('', items) + end + + def filter_for_allowed_items(items) + items.select { |item| item[:item].trendable? } end def would_be_trending?(id) -- cgit From 7b0fe4aef97c6a5f73a03146b669a415f396799c Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Fri, 29 Apr 2022 22:43:07 +0200 Subject: Fix opening and closing Redis connections instead of using a pool (#18171) * Fix opening and closing Redis connections instead of using a pool * Fix Redis connections not being returned to the pool in CLI commands --- app/lib/redis_configuration.rb | 7 ++++++- app/models/concerns/redisable.rb | 2 +- config/initializers/stoplight.rb | 6 ++++-- lib/mastodon/cli_helper.rb | 12 +++++++++--- lib/mastodon/rack_middleware.rb | 2 +- lib/mastodon/search_cli.rb | 7 ++++--- lib/mastodon/sidekiq_middleware.rb | 2 +- spec/services/resolve_account_service_spec.rb | 2 ++ 8 files changed, 28 insertions(+), 12 deletions(-) (limited to 'app/models') diff --git a/app/lib/redis_configuration.rb b/app/lib/redis_configuration.rb index fc8cf2f80..e14d6c8b6 100644 --- a/app/lib/redis_configuration.rb +++ b/app/lib/redis_configuration.rb @@ -2,12 +2,17 @@ class RedisConfiguration class << self + def establish_pool(new_pool_size) + @pool&.shutdown(&:close) + @pool = ConnectionPool.new(size: new_pool_size) { new.connection } + end + def with pool.with { |redis| yield redis } end def pool - @pool ||= ConnectionPool.new(size: pool_size) { new.connection } + @pool ||= establish_pool(pool_size) end def pool_size diff --git a/app/models/concerns/redisable.rb b/app/models/concerns/redisable.rb index cce8efb86..8d76b6b82 100644 --- a/app/models/concerns/redisable.rb +++ b/app/models/concerns/redisable.rb @@ -6,6 +6,6 @@ module Redisable private def redis - Thread.current[:redis] ||= RedisConfiguration.new.connection + Thread.current[:redis] ||= RedisConfiguration.pool.checkout end end diff --git a/config/initializers/stoplight.rb b/config/initializers/stoplight.rb index d9ebca76c..8c3c5755a 100644 --- a/config/initializers/stoplight.rb +++ b/config/initializers/stoplight.rb @@ -1,4 +1,6 @@ require 'stoplight' -Stoplight::Light.default_data_store = Stoplight::DataStore::Redis.new(RedisConfiguration.new.connection) -Stoplight::Light.default_notifiers = [Stoplight::Notifier::Logger.new(Rails.logger)] +Rails.application.reloader.to_prepare do + Stoplight::Light.default_data_store = Stoplight::DataStore::Redis.new(RedisConfiguration.new.connection) + Stoplight::Light.default_notifiers = [Stoplight::Notifier::Logger.new(Rails.logger)] +end diff --git a/lib/mastodon/cli_helper.rb b/lib/mastodon/cli_helper.rb index aaee1fa91..a78a28e27 100644 --- a/lib/mastodon/cli_helper.rb +++ b/lib/mastodon/cli_helper.rb @@ -19,15 +19,18 @@ module Mastodon ProgressBar.create(total: total, format: '%c/%u |%b%i| %e') end + def reset_connection_pools! + ActiveRecord::Base.establish_connection(ActiveRecord::Base.configurations[Rails.env].dup.tap { |config| config['pool'] = options[:concurrency] + 1 }) + RedisConfiguration.establish_pool(options[:concurrency]) + end + def parallelize_with_progress(scope) if options[:concurrency] < 1 say('Cannot run with this concurrency setting, must be at least 1', :red) exit(1) end - db_config = ActiveRecord::Base.configurations[Rails.env].dup - db_config['pool'] = options[:concurrency] + 1 - ActiveRecord::Base.establish_connection(db_config) + reset_connection_pools! progress = create_progress_bar(scope.count) pool = Concurrent::FixedThreadPool.new(options[:concurrency]) @@ -52,6 +55,9 @@ module Mastodon result = ActiveRecord::Base.connection_pool.with_connection do yield(item) + ensure + RedisConfiguration.pool.checkin if Thread.current[:redis] + Thread.current[:redis] = nil end aggregate.increment(result) if result.is_a?(Integer) diff --git a/lib/mastodon/rack_middleware.rb b/lib/mastodon/rack_middleware.rb index 619a2c36d..8aa7911fe 100644 --- a/lib/mastodon/rack_middleware.rb +++ b/lib/mastodon/rack_middleware.rb @@ -19,7 +19,7 @@ class Mastodon::RackMiddleware end def clean_up_redis_socket! - Thread.current[:redis]&.close + RedisConfiguration.pool.checkin if Thread.current[:redis] Thread.current[:redis] = nil end diff --git a/lib/mastodon/search_cli.rb b/lib/mastodon/search_cli.rb index 6ad9d7b6a..74f980ba1 100644 --- a/lib/mastodon/search_cli.rb +++ b/lib/mastodon/search_cli.rb @@ -59,9 +59,7 @@ module Mastodon index.specification.lock! end - db_config = ActiveRecord::Base.configurations[Rails.env].dup - db_config['pool'] = options[:concurrency] + 1 - ActiveRecord::Base.establish_connection(db_config) + reset_connection_pools! pool = Concurrent::FixedThreadPool.new(options[:concurrency]) added = Concurrent::AtomicFixnum.new(0) @@ -139,6 +137,9 @@ module Mastodon sleep 1 rescue => e progress.log pastel.red("Error importing #{index}: #{e}") + ensure + RedisConfiguration.pool.checkin if Thread.current[:redis] + Thread.current[:redis] = nil end end end diff --git a/lib/mastodon/sidekiq_middleware.rb b/lib/mastodon/sidekiq_middleware.rb index 7ec4097df..c75e8401f 100644 --- a/lib/mastodon/sidekiq_middleware.rb +++ b/lib/mastodon/sidekiq_middleware.rb @@ -26,7 +26,7 @@ class Mastodon::SidekiqMiddleware end def clean_up_redis_socket! - Thread.current[:redis]&.close + RedisConfiguration.pool.checkin if Thread.current[:redis] Thread.current[:redis] = nil end diff --git a/spec/services/resolve_account_service_spec.rb b/spec/services/resolve_account_service_spec.rb index 7b1e8885c..8c302e1d8 100644 --- a/spec/services/resolve_account_service_spec.rb +++ b/spec/services/resolve_account_service_spec.rb @@ -220,6 +220,8 @@ RSpec.describe ResolveAccountService, type: :service do return_values << described_class.new.call('foo@ap.example.com') rescue ActiveRecord::RecordNotUnique fail_occurred = true + ensure + RedisConfiguration.pool.checkin if Thread.current[:redis] end end end -- cgit From f6d35ed57d156f4225338a89372c8e83721e46c9 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Fri, 29 Apr 2022 23:27:03 +0200 Subject: Remove IP matching from e-mail domain blocks (#18190) Clear out e-mail domain blocks created from automatically resolved DNS records --- app/models/email_domain_block.rb | 24 ++++----- app/validators/email_mx_validator.rb | 6 +-- .../email_domain_block_refresh_scheduler.rb | 31 ----------- config/sidekiq.yml | 4 -- .../20180615122121_add_autofollow_to_invites.rb | 2 +- ...29101025_remove_ips_from_email_domain_blocks.rb | 12 +++++ .../20220429101850_clear_email_domain_blocks.rb | 14 +++++ db/schema.rb | 4 +- spec/validators/email_mx_validator_spec.rb | 60 ---------------------- 9 files changed, 43 insertions(+), 114 deletions(-) delete mode 100644 app/workers/scheduler/email_domain_block_refresh_scheduler.rb create mode 100644 db/post_migrate/20220429101025_remove_ips_from_email_domain_blocks.rb create mode 100644 db/post_migrate/20220429101850_clear_email_domain_blocks.rb (limited to 'app/models') diff --git a/app/models/email_domain_block.rb b/app/models/email_domain_block.rb index 36e7e62ab..0e1e663c1 100644 --- a/app/models/email_domain_block.rb +++ b/app/models/email_domain_block.rb @@ -3,16 +3,19 @@ # # Table name: email_domain_blocks # -# id :bigint(8) not null, primary key -# domain :string default(""), not null -# created_at :datetime not null -# updated_at :datetime not null -# parent_id :bigint(8) -# ips :inet is an Array -# last_refresh_at :datetime +# id :bigint(8) not null, primary key +# domain :string default(""), not null +# created_at :datetime not null +# updated_at :datetime not null +# parent_id :bigint(8) # class EmailDomainBlock < ApplicationRecord + self.ignored_columns = %w( + ips + last_refresh_at + ) + include DomainNormalizable belongs_to :parent, class_name: 'EmailDomainBlock', optional: true @@ -27,7 +30,7 @@ class EmailDomainBlock < ApplicationRecord @history ||= Trends::History.new('email_domain_blocks', id) end - def self.block?(domain_or_domains, ips: [], attempt_ip: nil) + def self.block?(domain_or_domains, attempt_ip: nil) domains = Array(domain_or_domains).map do |str| domain = begin if str.include?('@') @@ -48,10 +51,7 @@ class EmailDomainBlock < ApplicationRecord blocked = domains.any?(&:nil?) - scope = where(domain: domains) - scope = scope.or(where('ips && ARRAY[?]::inet[]', ips)) if ips.any? - - scope.find_each do |block| + where(domain: domains).find_each do |block| blocked = true block.history.add(attempt_ip) if attempt_ip.present? end diff --git a/app/validators/email_mx_validator.rb b/app/validators/email_mx_validator.rb index 237ca4c7b..20f2fd37c 100644 --- a/app/validators/email_mx_validator.rb +++ b/app/validators/email_mx_validator.rb @@ -15,7 +15,7 @@ class EmailMxValidator < ActiveModel::Validator if resolved_ips.empty? user.errors.add(:email, :unreachable) - elsif on_blacklist?(resolved_domains, resolved_ips, user.sign_up_ip) + elsif on_blacklist?(resolved_domains, user.sign_up_ip) user.errors.add(:email, :blocked) end end @@ -57,7 +57,7 @@ class EmailMxValidator < ActiveModel::Validator [ips, records] end - def on_blacklist?(domains, resolved_ips, attempt_ip) - EmailDomainBlock.block?(domains, ips: resolved_ips, attempt_ip: attempt_ip) + def on_blacklist?(domains, attempt_ip) + EmailDomainBlock.block?(domains, attempt_ip: attempt_ip) end end diff --git a/app/workers/scheduler/email_domain_block_refresh_scheduler.rb b/app/workers/scheduler/email_domain_block_refresh_scheduler.rb deleted file mode 100644 index e0ad89866..000000000 --- a/app/workers/scheduler/email_domain_block_refresh_scheduler.rb +++ /dev/null @@ -1,31 +0,0 @@ -# frozen_string_literal: true - -class Scheduler::EmailDomainBlockRefreshScheduler - include Sidekiq::Worker - include Redisable - - sidekiq_options retry: 0 - - def perform - Resolv::DNS.open do |dns| - dns.timeouts = 5 - - EmailDomainBlock.find_each do |email_domain_block| - ips = begin - if ip?(email_domain_block.domain) - [email_domain_block.domain] - else - resources = dns.getresources(email_domain_block.domain, Resolv::DNS::Resource::IN::A).to_a + dns.getresources(email_domain_block.domain, Resolv::DNS::Resource::IN::AAAA).to_a - resources.map { |resource| resource.address.to_s } - end - end - - email_domain_block.update(ips: ips, last_refresh_at: Time.now.utc) - end - end - end - - def ip?(str) - str =~ Regexp.union([Resolv::IPv4::Regex, Resolv::IPv6::Regex]) - end -end diff --git a/config/sidekiq.yml b/config/sidekiq.yml index f2ae9279b..26be26326 100644 --- a/config/sidekiq.yml +++ b/config/sidekiq.yml @@ -17,10 +17,6 @@ every: '5m' class: Scheduler::Trends::RefreshScheduler queue: scheduler - email_domain_block_refresh_scheduler: - every: '1h' - class: Scheduler::EmailDomainBlockRefreshScheduler - queue: scheduler trends_review_notifications_scheduler: every: '6h' class: Scheduler::Trends::ReviewNotificationsScheduler diff --git a/db/migrate/20180615122121_add_autofollow_to_invites.rb b/db/migrate/20180615122121_add_autofollow_to_invites.rb index 850b1d693..8c5fb7410 100644 --- a/db/migrate/20180615122121_add_autofollow_to_invites.rb +++ b/db/migrate/20180615122121_add_autofollow_to_invites.rb @@ -5,7 +5,7 @@ class AddAutofollowToInvites < ActiveRecord::Migration[5.2] disable_ddl_transaction! - def change + def up safety_assured do add_column_with_default :invites, :autofollow, :bool, default: false, allow_null: false end diff --git a/db/post_migrate/20220429101025_remove_ips_from_email_domain_blocks.rb b/db/post_migrate/20220429101025_remove_ips_from_email_domain_blocks.rb new file mode 100644 index 000000000..fbb74d99e --- /dev/null +++ b/db/post_migrate/20220429101025_remove_ips_from_email_domain_blocks.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +class RemoveIpsFromEmailDomainBlocks < ActiveRecord::Migration[5.2] + disable_ddl_transaction! + + def change + safety_assured do + remove_column :email_domain_blocks, :ips, :inet, array: true + remove_column :email_domain_blocks, :last_refresh_at, :datetime + end + end +end diff --git a/db/post_migrate/20220429101850_clear_email_domain_blocks.rb b/db/post_migrate/20220429101850_clear_email_domain_blocks.rb new file mode 100644 index 000000000..ff525b650 --- /dev/null +++ b/db/post_migrate/20220429101850_clear_email_domain_blocks.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +class ClearEmailDomainBlocks < ActiveRecord::Migration[5.2] + disable_ddl_transaction! + + class EmailDomainBlock < ApplicationRecord + end + + def up + EmailDomainBlock.where.not(parent_id: nil).in_batches.delete_all + end + + def down; end +end diff --git a/db/schema.rb b/db/schema.rb index a58f42400..726989bef 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2022_04_28_114902) do +ActiveRecord::Schema.define(version: 2022_04_29_101850) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -389,8 +389,6 @@ ActiveRecord::Schema.define(version: 2022_04_28_114902) do t.datetime "created_at", null: false t.datetime "updated_at", null: false t.bigint "parent_id" - t.inet "ips", array: true - t.datetime "last_refresh_at" t.index ["domain"], name: "index_email_domain_blocks_on_domain", unique: true end diff --git a/spec/validators/email_mx_validator_spec.rb b/spec/validators/email_mx_validator_spec.rb index af0eb98f5..4feedd0c7 100644 --- a/spec/validators/email_mx_validator_spec.rb +++ b/spec/validators/email_mx_validator_spec.rb @@ -56,66 +56,6 @@ describe EmailMxValidator do expect(user.errors).to have_received(:add) end - it 'adds an error if the A record is blacklisted' do - EmailDomainBlock.create!(domain: 'alternate-example.com', ips: ['1.2.3.4']) - resolver = double - - allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::MX).and_return([]) - allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::A).and_return([double(address: '1.2.3.4')]) - allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::AAAA).and_return([]) - allow(resolver).to receive(:timeouts=).and_return(nil) - allow(Resolv::DNS).to receive(:open).and_yield(resolver) - - subject.validate(user) - expect(user.errors).to have_received(:add) - end - - it 'adds an error if the AAAA record is blacklisted' do - EmailDomainBlock.create!(domain: 'alternate-example.com', ips: ['fd00::1']) - resolver = double - - allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::MX).and_return([]) - allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::A).and_return([]) - allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::AAAA).and_return([double(address: 'fd00::1')]) - allow(resolver).to receive(:timeouts=).and_return(nil) - allow(Resolv::DNS).to receive(:open).and_yield(resolver) - - subject.validate(user) - expect(user.errors).to have_received(:add) - end - - it 'adds an error if the A record of the MX record is blacklisted' do - EmailDomainBlock.create!(domain: 'mail.other-domain.com', ips: ['2.3.4.5']) - resolver = double - - allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::MX).and_return([double(exchange: 'mail.example.com')]) - allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::A).and_return([]) - allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::AAAA).and_return([]) - allow(resolver).to receive(:getresources).with('mail.example.com', Resolv::DNS::Resource::IN::A).and_return([double(address: '2.3.4.5')]) - allow(resolver).to receive(:getresources).with('mail.example.com', Resolv::DNS::Resource::IN::AAAA).and_return([]) - allow(resolver).to receive(:timeouts=).and_return(nil) - allow(Resolv::DNS).to receive(:open).and_yield(resolver) - - subject.validate(user) - expect(user.errors).to have_received(:add) - end - - it 'adds an error if the AAAA record of the MX record is blacklisted' do - EmailDomainBlock.create!(domain: 'mail.other-domain.com', ips: ['fd00::2']) - resolver = double - - allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::MX).and_return([double(exchange: 'mail.example.com')]) - allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::A).and_return([]) - allow(resolver).to receive(:getresources).with('example.com', Resolv::DNS::Resource::IN::AAAA).and_return([]) - allow(resolver).to receive(:getresources).with('mail.example.com', Resolv::DNS::Resource::IN::A).and_return([]) - allow(resolver).to receive(:getresources).with('mail.example.com', Resolv::DNS::Resource::IN::AAAA).and_return([double(address: 'fd00::2')]) - allow(resolver).to receive(:timeouts=).and_return(nil) - allow(Resolv::DNS).to receive(:open).and_yield(resolver) - - subject.validate(user) - expect(user.errors).to have_received(:add) - end - it 'adds an error if the MX record is blacklisted' do EmailDomainBlock.create!(domain: 'mail.example.com') resolver = double -- cgit From 6e4d932da53ba236699285ce2bdfcfa07b259fa2 Mon Sep 17 00:00:00 2001 From: Claire Date: Sun, 1 May 2022 00:55:26 +0200 Subject: Fix possible crash when a post references an invalid media attachment (#18211) --- app/models/status.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/models') diff --git a/app/models/status.rb b/app/models/status.rb index 288d374fd..a4e685ce3 100644 --- a/app/models/status.rb +++ b/app/models/status.rb @@ -254,7 +254,7 @@ class Status < ApplicationRecord media_attachments else map = media_attachments.index_by(&:id) - ordered_media_attachment_ids.map { |media_attachment_id| map[media_attachment_id] } + ordered_media_attachment_ids.filter_map { |media_attachment_id| map[media_attachment_id] } end end -- cgit