From ed7d459d7fedd1862897f99a75a0876b3a34a963 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Fri, 9 Apr 2021 02:31:11 +0200 Subject: Fix deprecation warning for Sidekiq web session secret (#16009) --- config/routes.rb | 2 -- 1 file changed, 2 deletions(-) (limited to 'config') diff --git a/config/routes.rb b/config/routes.rb index fa1138868..eedd0de69 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -3,8 +3,6 @@ require 'sidekiq_unique_jobs/web' require 'sidekiq-scheduler/web' -Sidekiq::Web.set :session_secret, Rails.application.secrets[:secret_key_base] - Rails.application.routes.draw do root 'home#index' -- cgit From 3f2533ca8e2465fbc742df02741e7dbe633d0da0 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Fri, 9 Apr 2021 02:31:20 +0200 Subject: Fix autoloading deprecation warnings from Rails 6 (#16010) --- config/initializers/content_security_policy.rb | 14 ++++++++------ config/initializers/paperclip.rb | 4 +++- config/initializers/suppress_csrf_warnings.rb | 4 +++- 3 files changed, 14 insertions(+), 8 deletions(-) (limited to 'config') diff --git a/config/initializers/content_security_policy.rb b/config/initializers/content_security_policy.rb index 92645ff28..b377b7b4d 100644 --- a/config/initializers/content_security_policy.rb +++ b/config/initializers/content_security_policy.rb @@ -51,11 +51,13 @@ Rails.application.config.content_security_policy_nonce_generator = -> request { Rails.application.config.content_security_policy_nonce_directives = %w(style-src) -PgHero::HomeController.content_security_policy do |p| - p.script_src :self, :unsafe_inline, assets_host - p.style_src :self, :unsafe_inline, assets_host -end +Rails.application.reloader.to_prepare do + PgHero::HomeController.content_security_policy do |p| + p.script_src :self, :unsafe_inline, assets_host + p.style_src :self, :unsafe_inline, assets_host + end -PgHero::HomeController.after_action do - request.content_security_policy_nonce_generator = nil + PgHero::HomeController.after_action do + request.content_security_policy_nonce_generator = nil + end end diff --git a/config/initializers/paperclip.rb b/config/initializers/paperclip.rb index 9ad7fd814..e2a045647 100644 --- a/config/initializers/paperclip.rb +++ b/config/initializers/paperclip.rb @@ -112,7 +112,9 @@ else ) end -Paperclip.options[:content_type_mappings] = { csv: Import::FILE_TYPES } +Rails.application.reloader.to_prepare do + Paperclip.options[:content_type_mappings] = { csv: Import::FILE_TYPES } +end # In some places in the code, we rescue this exception, but we don't always # load the S3 library, so it may be an undefined constant: diff --git a/config/initializers/suppress_csrf_warnings.rb b/config/initializers/suppress_csrf_warnings.rb index 410ab585b..b86adc6f1 100644 --- a/config/initializers/suppress_csrf_warnings.rb +++ b/config/initializers/suppress_csrf_warnings.rb @@ -1,3 +1,5 @@ # frozen_string_literal: true -ActionController::Base.log_warning_on_csrf_failure = false +Rails.application.reloader.to_prepare do + ActionController::Base.log_warning_on_csrf_failure = false +end -- cgit From 619fad6cf8078ea997554081febe850404bee73c Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sun, 11 Apr 2021 11:22:50 +0200 Subject: Remove spam check and dependency on nilsimsa gem (#16011) --- Gemfile | 1 - Gemfile.lock | 8 -- app/controllers/admin/dashboard_controller.rb | 1 - app/lib/activitypub/activity/create.rb | 5 - app/lib/spam_check.rb | 198 -------------------------- app/models/form/admin_settings.rb | 2 - app/services/process_mentions_service.rb | 5 - app/services/remove_status_service.rb | 5 - app/views/admin/dashboard/index.html.haml | 2 - app/views/admin/settings/edit.html.haml | 3 - config/locales/en.yml | 6 - config/settings.yml | 1 - spec/lib/spam_check_spec.rb | 192 ------------------------- 13 files changed, 429 deletions(-) delete mode 100644 app/lib/spam_check.rb delete mode 100644 spec/lib/spam_check_spec.rb (limited to 'config') diff --git a/Gemfile b/Gemfile index cb24207ca..1190f2558 100644 --- a/Gemfile +++ b/Gemfile @@ -62,7 +62,6 @@ gem 'idn-ruby', require: 'idn' gem 'kaminari', '~> 1.2' gem 'link_header', '~> 0.0' gem 'mime-types', '~> 3.3.1', require: 'mime/types/columnar' -gem 'nilsimsa', git: 'https://github.com/witgo/nilsimsa', ref: 'fd184883048b922b176939f851338d0a4971a532' gem 'nokogiri', '~> 1.11' gem 'nsa', git: 'https://github.com/Gargron/nsa', ref: 'd1079e0cdafdfed7f9f35478d13b9bdaa65965c0' gem 'oj', '~> 3.11' diff --git a/Gemfile.lock b/Gemfile.lock index 11765a967..8bea31332 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -18,13 +18,6 @@ GIT activerecord (>= 6.1.0) activesupport (>= 6.1.0) -GIT - remote: https://github.com/witgo/nilsimsa - revision: fd184883048b922b176939f851338d0a4971a532 - ref: fd184883048b922b176939f851338d0a4971a532 - specs: - nilsimsa (1.1.2) - GEM remote: https://rubygems.org/ specs: @@ -762,7 +755,6 @@ DEPENDENCIES microformats (~> 4.2) mime-types (~> 3.3.1) net-ldap (~> 0.17) - nilsimsa! nokogiri (~> 1.11) nsa! oj (~> 3.11) diff --git a/app/controllers/admin/dashboard_controller.rb b/app/controllers/admin/dashboard_controller.rb index 4422825ee..c829ed98f 100644 --- a/app/controllers/admin/dashboard_controller.rb +++ b/app/controllers/admin/dashboard_controller.rb @@ -35,7 +35,6 @@ module Admin @whitelist_enabled = whitelist_mode? @profile_directory = Setting.profile_directory @timeline_preview = Setting.timeline_preview - @spam_check_enabled = Setting.spam_check_enabled @trends_enabled = Setting.trends end diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index 0fa306cdd..9f6dd9ce0 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -88,7 +88,6 @@ class ActivityPub::Activity::Create < ActivityPub::Activity resolve_thread(@status) fetch_replies(@status) - check_for_spam distribute(@status) forward_for_reply end @@ -492,10 +491,6 @@ class ActivityPub::Activity::Create < ActivityPub::Activity Tombstone.exists?(uri: object_uri) end - def check_for_spam - SpamCheck.perform(@status) - end - def forward_for_reply return unless @status.distributable? && @json['signature'].present? && reply_to_local? diff --git a/app/lib/spam_check.rb b/app/lib/spam_check.rb deleted file mode 100644 index dcb2db9ca..000000000 --- a/app/lib/spam_check.rb +++ /dev/null @@ -1,198 +0,0 @@ -# frozen_string_literal: true - -class SpamCheck - include Redisable - include ActionView::Helpers::TextHelper - - # Threshold over which two Nilsimsa values are considered - # to refer to the same text - NILSIMSA_COMPARE_THRESHOLD = 95 - - # Nilsimsa doesn't work well on small inputs, so below - # this size, we check only for exact matches with MD5 - NILSIMSA_MIN_SIZE = 10 - - # How long to keep the trail of digests between updates, - # there is no reason to store it forever - EXPIRE_SET_AFTER = 1.week.seconds - - # How many digests to keep in an account's trail. If it's - # too small, spam could rotate around different message templates - MAX_TRAIL_SIZE = 10 - - # How many detected duplicates to allow through before - # considering the message as spam - THRESHOLD = 5 - - def initialize(status) - @account = status.account - @status = status - end - - def skip? - disabled? || already_flagged? || trusted? || no_unsolicited_mentions? || solicited_reply? - end - - def spam? - if insufficient_data? - false - elsif nilsimsa? - digests_over_threshold?('nilsimsa') { |_, other_digest| nilsimsa_compare_value(digest, other_digest) >= NILSIMSA_COMPARE_THRESHOLD } - else - digests_over_threshold?('md5') { |_, other_digest| other_digest == digest } - end - end - - def flag! - auto_report_status! - end - - def remember! - # The scores in sorted sets don't actually have enough bits to hold an exact - # value of our snowflake IDs, so we use it only for its ordering property. To - # get the correct status ID back, we have to save it in the string value - - redis.zadd(redis_key, @status.id, digest_with_algorithm) - redis.zremrangebyrank(redis_key, 0, -(MAX_TRAIL_SIZE + 1)) - redis.expire(redis_key, EXPIRE_SET_AFTER) - end - - def reset! - redis.del(redis_key) - end - - def hashable_text - return @hashable_text if defined?(@hashable_text) - - @hashable_text = @status.text - @hashable_text = remove_mentions(@hashable_text) - @hashable_text = strip_tags(@hashable_text) unless @status.local? - @hashable_text = normalize_unicode(@status.spoiler_text + ' ' + @hashable_text) - @hashable_text = remove_whitespace(@hashable_text) - end - - def insufficient_data? - hashable_text.blank? - end - - def digest - @digest ||= begin - if nilsimsa? - Nilsimsa.new(hashable_text).hexdigest - else - Digest::MD5.hexdigest(hashable_text) - end - end - end - - def digest_with_algorithm - if nilsimsa? - ['nilsimsa', digest, @status.id].join(':') - else - ['md5', digest, @status.id].join(':') - end - end - - class << self - def perform(status) - spam_check = new(status) - - return if spam_check.skip? - - if spam_check.spam? - spam_check.flag! - else - spam_check.remember! - end - end - end - - private - - def disabled? - !Setting.spam_check_enabled - end - - def remove_mentions(text) - return text.gsub(Account::MENTION_RE, '') if @status.local? - - Nokogiri::HTML.fragment(text).tap do |html| - mentions = @status.mentions.map { |mention| ActivityPub::TagManager.instance.url_for(mention.account) } - - html.traverse do |element| - element.unlink if element.name == 'a' && mentions.include?(element['href']) - end - end.to_s - end - - def normalize_unicode(text) - text.unicode_normalize(:nfkc).downcase - end - - def remove_whitespace(text) - text.gsub(/\s+/, ' ').strip - end - - def auto_report_status! - status_ids = Status.where(visibility: %i(public unlisted)).where(id: matching_status_ids).pluck(:id) + [@status.id] if @status.distributable? - ReportService.new.call(Account.representative, @account, status_ids: status_ids, comment: I18n.t('spam_check.spam_detected')) - end - - def already_flagged? - @account.silenced? || @account.targeted_reports.unresolved.where(account_id: -99).exists? - end - - def trusted? - @account.trust_level > Account::TRUST_LEVELS[:untrusted] || (@account.local? && @account.user_staff?) - end - - def no_unsolicited_mentions? - @status.mentions.all? { |mention| mention.silent? || (!@account.local? && !mention.account.local?) || mention.account.following?(@account) } - end - - def solicited_reply? - !@status.thread.nil? && @status.thread.mentions.where(account: @account).exists? - end - - def nilsimsa_compare_value(first, second) - first = [first].pack('H*') - second = [second].pack('H*') - bits = 0 - - 0.upto(31) do |i| - bits += Nilsimsa::POPC[255 & (first[i].ord ^ second[i].ord)].ord - end - - 128 - bits # -128 <= Nilsimsa Compare Value <= 128 - end - - def nilsimsa? - hashable_text.size > NILSIMSA_MIN_SIZE - end - - def other_digests - redis.zrange(redis_key, 0, -1) - end - - def digests_over_threshold?(filter_algorithm) - other_digests.select do |record| - algorithm, other_digest, status_id = record.split(':') - - next unless algorithm == filter_algorithm - - yield algorithm, other_digest, status_id - end.size >= THRESHOLD - end - - def matching_status_ids - if nilsimsa? - other_digests.filter_map { |record| record.split(':')[2] if record.start_with?('nilsimsa') && nilsimsa_compare_value(digest, record.split(':')[1]) >= NILSIMSA_COMPARE_THRESHOLD } - else - other_digests.filter_map { |record| record.split(':')[2] if record.start_with?('md5') && record.split(':')[1] == digest } - end - end - - def redis_key - @redis_key ||= "spam_check:#{@account.id}" - end -end diff --git a/app/models/form/admin_settings.rb b/app/models/form/admin_settings.rb index e9f78da21..b5c3dcdbe 100644 --- a/app/models/form/admin_settings.rb +++ b/app/models/form/admin_settings.rb @@ -29,7 +29,6 @@ class Form::AdminSettings thumbnail hero mascot - spam_check_enabled trends trendable_by_default show_domain_blocks @@ -48,7 +47,6 @@ class Form::AdminSettings show_known_fediverse_at_about_page preview_sensitive_media profile_directory - spam_check_enabled trends trendable_by_default noindex diff --git a/app/services/process_mentions_service.rb b/app/services/process_mentions_service.rb index 4c02c7865..73dbb1834 100644 --- a/app/services/process_mentions_service.rb +++ b/app/services/process_mentions_service.rb @@ -43,7 +43,6 @@ class ProcessMentionsService < BaseService end status.save! - check_for_spam(status) mentions.each { |mention| create_notification(mention) } end @@ -72,8 +71,4 @@ class ProcessMentionsService < BaseService def resolve_account_service ResolveAccountService.new end - - def check_for_spam(status) - SpamCheck.perform(status) - end end diff --git a/app/services/remove_status_service.rb b/app/services/remove_status_service.rb index d6043fb5d..d642beeaa 100644 --- a/app/services/remove_status_service.rb +++ b/app/services/remove_status_service.rb @@ -41,7 +41,6 @@ class RemoveStatusService < BaseService remove_from_hashtags remove_from_public remove_from_media if @status.media_attachments.any? - remove_from_spam_check remove_media end @@ -163,10 +162,6 @@ class RemoveStatusService < BaseService @status.media_attachments.destroy_all end - def remove_from_spam_check - redis.zremrangebyscore("spam_check:#{@status.account_id}", @status.id, @status.id) - end - def lock_options { redis: Redis.current, key: "distribute:#{@status.id}" } end diff --git a/app/views/admin/dashboard/index.html.haml b/app/views/admin/dashboard/index.html.haml index 205538402..57a753e6b 100644 --- a/app/views/admin/dashboard/index.html.haml +++ b/app/views/admin/dashboard/index.html.haml @@ -77,8 +77,6 @@ = feature_hint(link_to(t('admin.dashboard.trends'), edit_admin_settings_path), @trends_enabled) %li = feature_hint(link_to(t('admin.dashboard.feature_relay'), admin_relays_path), @relay_enabled) - %li - = feature_hint(link_to(t('admin.dashboard.feature_spam_check'), edit_admin_settings_path), @spam_check_enabled) .dashboard__widgets__versions %div diff --git a/app/views/admin/settings/edit.html.haml b/app/views/admin/settings/edit.html.haml index 159bd4b0a..0e705f205 100644 --- a/app/views/admin/settings/edit.html.haml +++ b/app/views/admin/settings/edit.html.haml @@ -92,9 +92,6 @@ .fields-group = f.input :noindex, as: :boolean, wrapper: :with_label, label: t('admin.settings.default_noindex.title'), hint: t('admin.settings.default_noindex.desc_html') - .fields-group - = f.input :spam_check_enabled, as: :boolean, wrapper: :with_label, label: t('admin.settings.spam_check_enabled.title'), hint: t('admin.settings.spam_check_enabled.desc_html') - %hr.spacer/ .fields-group diff --git a/config/locales/en.yml b/config/locales/en.yml index 182a8e985..3387b4df6 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -363,7 +363,6 @@ en: feature_profile_directory: Profile directory feature_registrations: Registrations feature_relay: Federation relay - feature_spam_check: Anti-spam feature_timeline_preview: Timeline preview features: Features hidden_service: Federation with hidden services @@ -627,9 +626,6 @@ en: desc_html: You can write your own privacy policy, terms of service or other legalese. You can use HTML tags title: Custom terms of service site_title: Server name - spam_check_enabled: - desc_html: Mastodon can auto-report accounts that send repeated unsolicited messages. There may be false positives. - title: Anti-spam automation thumbnail: desc_html: Used for previews via OpenGraph and API. 1200x630px recommended title: Server thumbnail @@ -1209,8 +1205,6 @@ en: relationships: Follows and followers two_factor_authentication: Two-factor Auth webauthn_authentication: Security keys - spam_check: - spam_detected: This is an automated report. Spam has been detected. statuses: attached: audio: diff --git a/config/settings.yml b/config/settings.yml index 9cf68a096..b79ea620c 100644 --- a/config/settings.yml +++ b/config/settings.yml @@ -67,7 +67,6 @@ defaults: &defaults activity_api_enabled: true peers_api_enabled: true show_known_fediverse_at_about_page: true - spam_check_enabled: true show_domain_blocks: 'disabled' show_domain_blocks_rationale: 'disabled' require_invite_text: false diff --git a/spec/lib/spam_check_spec.rb b/spec/lib/spam_check_spec.rb deleted file mode 100644 index 159d83257..000000000 --- a/spec/lib/spam_check_spec.rb +++ /dev/null @@ -1,192 +0,0 @@ -# frozen_string_literal: true - -require 'rails_helper' - -RSpec.describe SpamCheck do - let!(:sender) { Fabricate(:account) } - let!(:alice) { Fabricate(:account, username: 'alice') } - let!(:bob) { Fabricate(:account, username: 'bob') } - - def status_with_html(text, options = {}) - status = PostStatusService.new.call(sender, { text: text }.merge(options)) - status.update_columns(text: Formatter.instance.format(status), local: false) - status - end - - describe '#hashable_text' do - it 'removes mentions from HTML for remote statuses' do - status = status_with_html('@alice Hello') - expect(described_class.new(status).hashable_text).to eq 'hello' - end - - it 'removes mentions from text for local statuses' do - status = PostStatusService.new.call(alice, text: "Hey @#{sender.username}, how are you?") - expect(described_class.new(status).hashable_text).to eq 'hey , how are you?' - end - end - - describe '#insufficient_data?' do - it 'returns true when there is no text' do - status = status_with_html('@alice') - expect(described_class.new(status).insufficient_data?).to be true - end - - it 'returns false when there is text' do - status = status_with_html('@alice h') - expect(described_class.new(status).insufficient_data?).to be false - end - end - - describe '#digest' do - it 'returns a string' do - status = status_with_html('@alice Hello world') - expect(described_class.new(status).digest).to be_a String - end - end - - describe '#spam?' do - it 'returns false for a unique status' do - status = status_with_html('@alice Hello') - expect(described_class.new(status).spam?).to be false - end - - it 'returns false for different statuses to the same recipient' do - status1 = status_with_html('@alice Hello') - described_class.new(status1).remember! - status2 = status_with_html('@alice Are you available to talk?') - expect(described_class.new(status2).spam?).to be false - end - - it 'returns false for statuses with different content warnings' do - status1 = status_with_html('@alice Are you available to talk?') - described_class.new(status1).remember! - status2 = status_with_html('@alice Are you available to talk?', spoiler_text: 'This is a completely different matter than what I was talking about previously, I swear!') - expect(described_class.new(status2).spam?).to be false - end - - it 'returns false for different statuses to different recipients' do - status1 = status_with_html('@alice How is it going?') - described_class.new(status1).remember! - status2 = status_with_html('@bob Are you okay?') - expect(described_class.new(status2).spam?).to be false - end - - it 'returns false for very short different statuses to different recipients' do - status1 = status_with_html('@alice 🙄') - described_class.new(status1).remember! - status2 = status_with_html('@bob Huh?') - expect(described_class.new(status2).spam?).to be false - end - - it 'returns false for statuses with no text' do - status1 = status_with_html('@alice') - described_class.new(status1).remember! - status2 = status_with_html('@bob') - expect(described_class.new(status2).spam?).to be false - end - - it 'returns true for duplicate statuses to the same recipient' do - described_class::THRESHOLD.times do - status1 = status_with_html('@alice Hello') - described_class.new(status1).remember! - end - - status2 = status_with_html('@alice Hello') - expect(described_class.new(status2).spam?).to be true - end - - it 'returns true for duplicate statuses to different recipients' do - described_class::THRESHOLD.times do - status1 = status_with_html('@alice Hello') - described_class.new(status1).remember! - end - - status2 = status_with_html('@bob Hello') - expect(described_class.new(status2).spam?).to be true - end - - it 'returns true for nearly identical statuses with random numbers' do - source_text = 'Sodium, atomic number 11, was first isolated by Humphry Davy in 1807. A chemical component of salt, he named it Na in honor of the saltiest region on earth, North America.' - - described_class::THRESHOLD.times do - status1 = status_with_html('@alice ' + source_text + ' 1234') - described_class.new(status1).remember! - end - - status2 = status_with_html('@bob ' + source_text + ' 9568') - expect(described_class.new(status2).spam?).to be true - end - end - - describe '#skip?' do - it 'returns true when the sender is already silenced' do - status = status_with_html('@alice Hello') - sender.silence! - expect(described_class.new(status).skip?).to be true - end - - it 'returns true when the mentioned person follows the sender' do - status = status_with_html('@alice Hello') - alice.follow!(sender) - expect(described_class.new(status).skip?).to be true - end - - it 'returns false when even one mentioned person doesn\'t follow the sender' do - status = status_with_html('@alice @bob Hello') - alice.follow!(sender) - expect(described_class.new(status).skip?).to be false - end - - it 'returns true when the sender is replying to a status that mentions the sender' do - parent = PostStatusService.new.call(alice, text: "Hey @#{sender.username}, how are you?") - status = status_with_html('@alice @bob Hello', thread: parent) - expect(described_class.new(status).skip?).to be true - end - end - - describe '#remember!' do - let(:status) { status_with_html('@alice') } - let(:spam_check) { described_class.new(status) } - let(:redis_key) { spam_check.send(:redis_key) } - - it 'remembers' do - expect(Redis.current.exists?(redis_key)).to be true - spam_check.remember! - expect(Redis.current.exists?(redis_key)).to be true - end - end - - describe '#reset!' do - let(:status) { status_with_html('@alice') } - let(:spam_check) { described_class.new(status) } - let(:redis_key) { spam_check.send(:redis_key) } - - before do - spam_check.remember! - end - - it 'resets' do - expect(Redis.current.exists?(redis_key)).to be true - spam_check.reset! - expect(Redis.current.exists?(redis_key)).to be false - end - end - - describe '#flag!' do - let!(:status1) { status_with_html('@alice General Kenobi you are a bold one') } - let!(:status2) { status_with_html('@alice @bob General Kenobi, you are a bold one') } - - before do - described_class.new(status1).remember! - described_class.new(status2).flag! - end - - it 'creates a report about the account' do - expect(sender.targeted_reports.unresolved.count).to eq 1 - end - - it 'attaches both matching statuses to the report' do - expect(sender.targeted_reports.first.status_ids).to include(status1.id, status2.id) - end - end -end -- cgit From ad61265268f13d9b2a04e2e176724d8a7376f85a Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Mon, 12 Apr 2021 03:35:58 +0200 Subject: Remove dependency on pluck_each gem (#16012) --- Gemfile | 1 - Gemfile.lock | 10 ---------- config/application.rb | 1 + lib/active_record/batches.rb | 44 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 45 insertions(+), 11 deletions(-) create mode 100644 lib/active_record/batches.rb (limited to 'config') diff --git a/Gemfile b/Gemfile index 1190f2558..d4385f014 100644 --- a/Gemfile +++ b/Gemfile @@ -157,4 +157,3 @@ gem 'concurrent-ruby', require: false gem 'connection_pool', require: false gem 'xorcist', '~> 1.1' -gem 'pluck_each', git: 'https://github.com/nsommer/pluck_each', ref: '73be0947c52fc54bf6d7085378db008358aac5eb' diff --git a/Gemfile.lock b/Gemfile.lock index 8bea31332..a3b11ab6c 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -9,15 +9,6 @@ GIT sidekiq (>= 3.5) statsd-ruby (~> 1.4, >= 1.4.0) -GIT - remote: https://github.com/nsommer/pluck_each - revision: 73be0947c52fc54bf6d7085378db008358aac5eb - ref: 73be0947c52fc54bf6d7085378db008358aac5eb - specs: - pluck_each (0.1.3) - activerecord (>= 6.1.0) - activesupport (>= 6.1.0) - GEM remote: https://rubygems.org/ specs: @@ -771,7 +762,6 @@ DEPENDENCIES pg (~> 1.2) pghero (~> 2.8) pkg-config (~> 1.4) - pluck_each! posix-spawn premailer-rails private_address_check (~> 0.5) diff --git a/config/application.rb b/config/application.rb index c911e76dc..eb2c91677 100644 --- a/config/application.rb +++ b/config/application.rb @@ -29,6 +29,7 @@ require_relative '../lib/webpacker/helper_extensions' require_relative '../lib/action_dispatch/cookie_jar_extensions' require_relative '../lib/rails/engine_extensions' require_relative '../lib/active_record/database_tasks_extensions' +require_relative '../lib/active_record/batches' Dotenv::Railtie.load diff --git a/lib/active_record/batches.rb b/lib/active_record/batches.rb new file mode 100644 index 000000000..55d29e52e --- /dev/null +++ b/lib/active_record/batches.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +module ActiveRecord + module Batches + def pluck_each(*column_names) + relation = self + + options = column_names.extract_options! + + flatten = column_names.size == 1 + batch_limit = options[:batch_limit] || 1_000 + order = options[:order] || :asc + + column_names.unshift(primary_key) + + relation = relation.reorder(batch_order(order)).limit(batch_limit) + relation.skip_query_cache! + + batch_relation = relation + + loop do + batch = batch_relation.pluck(*column_names) + + break if batch.empty? + + primary_key_offset = batch.last[0] + + batch.each do |record| + if flatten + yield record[1] + else + yield record[1..-1] + end + end + + break if batch.size < batch_limit + + batch_relation = relation.where( + predicate_builder[primary_key, primary_key_offset, order == :desc ? :lt : :gt] + ) + end + end + end +end -- cgit From f7117646afddb2676e9275d8efe90c3a20c59021 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Mon, 12 Apr 2021 12:37:14 +0200 Subject: Add cold-start follow recommendations (#15945) --- .../admin/follow_recommendations_controller.rb | 53 ++++++++++++++++++ app/controllers/api/v1/suggestions_controller.rb | 2 +- app/controllers/api/v2/suggestions_controller.rb | 19 +++++++ app/javascript/mastodon/actions/suggestions.js | 8 +-- .../features/compose/components/search_results.js | 12 ++--- app/javascript/mastodon/reducers/suggestions.js | 8 +-- app/lib/potential_friendship_tracker.rb | 12 +++-- app/models/account.rb | 3 +- app/models/account_suggestions.rb | 17 ++++++ app/models/account_summary.rb | 25 +++++++++ app/models/concerns/account_associations.rb | 3 ++ app/models/follow_recommendation.rb | 39 ++++++++++++++ app/models/follow_recommendation_filter.rb | 26 +++++++++ app/models/follow_recommendation_suppression.rb | 28 ++++++++++ app/models/form/account_batch.rb | 18 +++++++ app/policies/follow_recommendation_policy.rb | 15 ++++++ app/serializers/rest/suggestion_serializer.rb | 7 +++ .../follow_recommendations/_account.html.haml | 20 +++++++ .../admin/follow_recommendations/show.html.haml | 42 +++++++++++++++ .../scheduler/follow_recommendations_scheduler.rb | 61 +++++++++++++++++++++ config/locales/en.yml | 8 +++ config/navigation.rb | 1 + config/routes.rb | 2 + config/sidekiq.yml | 4 ++ .../20210322164601_create_account_summaries.rb | 9 ++++ ...20210323114347_create_follow_recommendations.rb | 5 ++ ...13_create_follow_recommendation_suppressions.rb | 9 ++++ db/schema.rb | 63 +++++++++++++++++++--- db/views/account_summaries_v01.sql | 22 ++++++++ db/views/follow_recommendations_v01.sql | 38 +++++++++++++ ...follow_recommendation_suppression_fabricator.rb | 3 ++ .../follow_recommendation_suppression_spec.rb | 4 ++ 32 files changed, 560 insertions(+), 26 deletions(-) create mode 100644 app/controllers/admin/follow_recommendations_controller.rb create mode 100644 app/controllers/api/v2/suggestions_controller.rb create mode 100644 app/models/account_suggestions.rb create mode 100644 app/models/account_summary.rb create mode 100644 app/models/follow_recommendation.rb create mode 100644 app/models/follow_recommendation_filter.rb create mode 100644 app/models/follow_recommendation_suppression.rb create mode 100644 app/policies/follow_recommendation_policy.rb create mode 100644 app/serializers/rest/suggestion_serializer.rb create mode 100644 app/views/admin/follow_recommendations/_account.html.haml create mode 100644 app/views/admin/follow_recommendations/show.html.haml create mode 100644 app/workers/scheduler/follow_recommendations_scheduler.rb create mode 100644 db/migrate/20210322164601_create_account_summaries.rb create mode 100644 db/migrate/20210323114347_create_follow_recommendations.rb create mode 100644 db/migrate/20210324171613_create_follow_recommendation_suppressions.rb create mode 100644 db/views/account_summaries_v01.sql create mode 100644 db/views/follow_recommendations_v01.sql create mode 100644 spec/fabricators/follow_recommendation_suppression_fabricator.rb create mode 100644 spec/models/follow_recommendation_suppression_spec.rb (limited to 'config') diff --git a/app/controllers/admin/follow_recommendations_controller.rb b/app/controllers/admin/follow_recommendations_controller.rb new file mode 100644 index 000000000..e3eac62b3 --- /dev/null +++ b/app/controllers/admin/follow_recommendations_controller.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +module Admin + class FollowRecommendationsController < BaseController + before_action :set_language + + def show + authorize :follow_recommendation, :show? + + @form = Form::AccountBatch.new + @accounts = filtered_follow_recommendations + end + + def update + @form = Form::AccountBatch.new(form_account_batch_params.merge(current_account: current_account, action: action_from_button)) + @form.save + rescue ActionController::ParameterMissing + # Do nothing + ensure + redirect_to admin_follow_recommendations_path(filter_params) + end + + private + + def set_language + @language = follow_recommendation_filter.language + end + + def filtered_follow_recommendations + follow_recommendation_filter.results + end + + def follow_recommendation_filter + @follow_recommendation_filter ||= FollowRecommendationFilter.new(filter_params) + end + + def form_account_batch_params + params.require(:form_account_batch).permit(:action, account_ids: []) + end + + def filter_params + params.slice(*FollowRecommendationFilter::KEYS).permit(*FollowRecommendationFilter::KEYS) + end + + def action_from_button + if params[:suppress] + 'suppress_follow_recommendation' + elsif params[:unsuppress] + 'unsuppress_follow_recommendation' + end + end + end +end diff --git a/app/controllers/api/v1/suggestions_controller.rb b/app/controllers/api/v1/suggestions_controller.rb index 52054160d..b2788cc76 100644 --- a/app/controllers/api/v1/suggestions_controller.rb +++ b/app/controllers/api/v1/suggestions_controller.rb @@ -19,6 +19,6 @@ class Api::V1::SuggestionsController < Api::BaseController private def set_accounts - @accounts = PotentialFriendshipTracker.get(current_account.id, limit: limit_param(DEFAULT_ACCOUNTS_LIMIT)) + @accounts = PotentialFriendshipTracker.get(current_account, limit_param(DEFAULT_ACCOUNTS_LIMIT)) end end diff --git a/app/controllers/api/v2/suggestions_controller.rb b/app/controllers/api/v2/suggestions_controller.rb new file mode 100644 index 000000000..35eb276c0 --- /dev/null +++ b/app/controllers/api/v2/suggestions_controller.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class Api::V2::SuggestionsController < Api::BaseController + include Authorization + + before_action -> { doorkeeper_authorize! :read } + before_action :require_user! + before_action :set_suggestions + + def index + render json: @suggestions, each_serializer: REST::SuggestionSerializer + end + + private + + def set_suggestions + @suggestions = AccountSuggestions.get(current_account, limit_param(DEFAULT_ACCOUNTS_LIMIT)) + end +end diff --git a/app/javascript/mastodon/actions/suggestions.js b/app/javascript/mastodon/actions/suggestions.js index b15bd916b..0bf959017 100644 --- a/app/javascript/mastodon/actions/suggestions.js +++ b/app/javascript/mastodon/actions/suggestions.js @@ -11,8 +11,8 @@ export function fetchSuggestions() { return (dispatch, getState) => { dispatch(fetchSuggestionsRequest()); - api(getState).get('/api/v1/suggestions').then(response => { - dispatch(importFetchedAccounts(response.data)); + api(getState).get('/api/v2/suggestions').then(response => { + dispatch(importFetchedAccounts(response.data.map(x => x.account))); dispatch(fetchSuggestionsSuccess(response.data)); }).catch(error => dispatch(fetchSuggestionsFail(error))); }; @@ -25,10 +25,10 @@ export function fetchSuggestionsRequest() { }; }; -export function fetchSuggestionsSuccess(accounts) { +export function fetchSuggestionsSuccess(suggestions) { return { type: SUGGESTIONS_FETCH_SUCCESS, - accounts, + suggestions, skipLoading: true, }; }; diff --git a/app/javascript/mastodon/features/compose/components/search_results.js b/app/javascript/mastodon/features/compose/components/search_results.js index 4b4cdff74..c4e160b8a 100644 --- a/app/javascript/mastodon/features/compose/components/search_results.js +++ b/app/javascript/mastodon/features/compose/components/search_results.js @@ -51,13 +51,13 @@ class SearchResults extends ImmutablePureComponent { - {suggestions && suggestions.map(accountId => ( + {suggestions && suggestions.map(suggestion => ( ))} diff --git a/app/javascript/mastodon/reducers/suggestions.js b/app/javascript/mastodon/reducers/suggestions.js index 834be728f..1a6e66ee7 100644 --- a/app/javascript/mastodon/reducers/suggestions.js +++ b/app/javascript/mastodon/reducers/suggestions.js @@ -19,18 +19,18 @@ export default function suggestionsReducer(state = initialState, action) { return state.set('isLoading', true); case SUGGESTIONS_FETCH_SUCCESS: return state.withMutations(map => { - map.set('items', fromJS(action.accounts.map(x => x.id))); + map.set('items', fromJS(action.suggestions.map(x => ({ ...x, account: x.account.id })))); map.set('isLoading', false); }); case SUGGESTIONS_FETCH_FAIL: return state.set('isLoading', false); case SUGGESTIONS_DISMISS: - return state.update('items', list => list.filterNot(id => id === action.id)); + return state.update('items', list => list.filterNot(x => x.account === action.id)); case ACCOUNT_BLOCK_SUCCESS: case ACCOUNT_MUTE_SUCCESS: - return state.update('items', list => list.filterNot(id => id === action.relationship.id)); + return state.update('items', list => list.filterNot(x => x.account === action.relationship.id)); case DOMAIN_BLOCK_SUCCESS: - return state.update('items', list => list.filterNot(id => action.accounts.includes(id))); + return state.update('items', list => list.filterNot(x => action.accounts.includes(x.account))); default: return state; } diff --git a/app/lib/potential_friendship_tracker.rb b/app/lib/potential_friendship_tracker.rb index 188aa4a27..e72d454b6 100644 --- a/app/lib/potential_friendship_tracker.rb +++ b/app/lib/potential_friendship_tracker.rb @@ -28,10 +28,14 @@ class PotentialFriendshipTracker redis.zrem("interactions:#{account_id}", target_account_id) end - def get(account_id, limit: 20, offset: 0) - account_ids = redis.zrevrange("interactions:#{account_id}", offset, limit) - return [] if account_ids.empty? - Account.searchable.where(id: account_ids) + def get(account, limit) + account_ids = redis.zrevrange("interactions:#{account.id}", 0, limit) + + return [] if account_ids.empty? || limit < 1 + + accounts = Account.searchable.where(id: account_ids).index_by(&:id) + + account_ids.map { |id| accounts[id.to_i] }.compact end end end diff --git a/app/models/account.rb b/app/models/account.rb index d85fd1f6e..80689d4aa 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -110,6 +110,7 @@ class Account < ApplicationRecord scope :matches_domain, ->(value) { where(arel_table[:domain].matches("%#{value}%")) } scope :searchable, -> { without_suspended.where(moved_to_account_id: nil) } scope :discoverable, -> { searchable.without_silenced.where(discoverable: true).left_outer_joins(:account_stat) } + scope :followable_by, ->(account) { joins(arel_table.join(Follow.arel_table, Arel::Nodes::OuterJoin).on(arel_table[:id].eq(Follow.arel_table[:target_account_id]).and(Follow.arel_table[:account_id].eq(account.id))).join_sources).where(Follow.arel_table[:id].eq(nil)).joins(arel_table.join(FollowRequest.arel_table, Arel::Nodes::OuterJoin).on(arel_table[:id].eq(FollowRequest.arel_table[:target_account_id]).and(FollowRequest.arel_table[:account_id].eq(account.id))).join_sources).where(FollowRequest.arel_table[:id].eq(nil)) } scope :tagged_with, ->(tag) { joins(:accounts_tags).where(accounts_tags: { tag_id: tag }) } scope :by_recent_status, -> { order(Arel.sql('(case when account_stats.last_status_at is null then 1 else 0 end) asc, account_stats.last_status_at desc, accounts.id desc')) } scope :by_recent_sign_in, -> { order(Arel.sql('(case when users.current_sign_in_at is null then 1 else 0 end) asc, users.current_sign_in_at desc, accounts.id desc')) } @@ -363,7 +364,7 @@ class Account < ApplicationRecord end def excluded_from_timeline_account_ids - Rails.cache.fetch("exclude_account_ids_for:#{id}") { blocking.pluck(:target_account_id) + blocked_by.pluck(:account_id) + muting.pluck(:target_account_id) } + Rails.cache.fetch("exclude_account_ids_for:#{id}") { block_relationships.pluck(:target_account_id) + blocked_by_relationships.pluck(:account_id) + mute_relationships.pluck(:target_account_id) } end def excluded_from_timeline_domains diff --git a/app/models/account_suggestions.rb b/app/models/account_suggestions.rb new file mode 100644 index 000000000..7fe9d618e --- /dev/null +++ b/app/models/account_suggestions.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class AccountSuggestions + class Suggestion < ActiveModelSerializers::Model + attributes :account, :source + end + + def self.get(account, limit) + suggestions = PotentialFriendshipTracker.get(account, limit).map { |target_account| Suggestion.new(account: target_account, source: :past_interaction) } + suggestions.concat(FollowRecommendation.get(account, limit - suggestions.size, suggestions.map { |suggestion| suggestion.account.id }).map { |target_account| Suggestion.new(account: target_account, source: :global) }) if suggestions.size < limit + suggestions + end + + def self.remove(account, target_account_id) + PotentialFriendshipTracker.remove(account.id, target_account_id) + end +end diff --git a/app/models/account_summary.rb b/app/models/account_summary.rb new file mode 100644 index 000000000..6a7e17c6c --- /dev/null +++ b/app/models/account_summary.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true +# == Schema Information +# +# Table name: account_summaries +# +# account_id :bigint(8) primary key +# language :string +# sensitive :boolean +# + +class AccountSummary < ApplicationRecord + self.primary_key = :account_id + + scope :safe, -> { where(sensitive: false) } + scope :localized, ->(locale) { where(language: locale) } + scope :filtered, -> { joins(arel_table.join(FollowRecommendationSuppression.arel_table, Arel::Nodes::OuterJoin).on(arel_table[:account_id].eq(FollowRecommendationSuppression.arel_table[:account_id])).join_sources).where(FollowRecommendationSuppression.arel_table[:id].eq(nil)) } + + def self.refresh + Scenic.database.refresh_materialized_view(table_name, concurrently: true, cascade: false) + end + + def readonly? + true + end +end diff --git a/app/models/concerns/account_associations.rb b/app/models/concerns/account_associations.rb index 98849f8fc..aaf371ebd 100644 --- a/app/models/concerns/account_associations.rb +++ b/app/models/concerns/account_associations.rb @@ -63,5 +63,8 @@ module AccountAssociations # Account deletion requests has_one :deletion_request, class_name: 'AccountDeletionRequest', inverse_of: :account, dependent: :destroy + + # Follow recommendations + has_one :follow_recommendation_suppression, inverse_of: :account, dependent: :destroy end end diff --git a/app/models/follow_recommendation.rb b/app/models/follow_recommendation.rb new file mode 100644 index 000000000..c4355224d --- /dev/null +++ b/app/models/follow_recommendation.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true +# == Schema Information +# +# Table name: follow_recommendations +# +# account_id :bigint(8) primary key +# rank :decimal(, ) +# reason :text is an Array +# + +class FollowRecommendation < ApplicationRecord + self.primary_key = :account_id + + belongs_to :account_summary, foreign_key: :account_id + belongs_to :account, foreign_key: :account_id + + scope :safe, -> { joins(:account_summary).merge(AccountSummary.safe) } + scope :localized, ->(locale) { joins(:account_summary).merge(AccountSummary.localized(locale)) } + scope :filtered, -> { joins(:account_summary).merge(AccountSummary.filtered) } + + def readonly? + true + end + + def self.get(account, limit, exclude_account_ids = []) + account_ids = Redis.current.zrevrange("follow_recommendations:#{account.user_locale}", 0, -1).map(&:to_i) - exclude_account_ids - [account.id] + + return [] if account_ids.empty? || limit < 1 + + accounts = Account.followable_by(account) + .not_excluded_by_account(account) + .not_domain_blocked_by_account(account) + .where(id: account_ids) + .limit(limit) + .index_by(&:id) + + account_ids.map { |id| accounts[id] }.compact + end +end diff --git a/app/models/follow_recommendation_filter.rb b/app/models/follow_recommendation_filter.rb new file mode 100644 index 000000000..acf03cd84 --- /dev/null +++ b/app/models/follow_recommendation_filter.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +class FollowRecommendationFilter + KEYS = %i( + language + status + ).freeze + + attr_reader :params, :language + + def initialize(params) + @language = params.delete('language') || I18n.locale + @params = params + end + + def results + if params['status'] == 'suppressed' + Account.joins(:follow_recommendation_suppression).order(FollowRecommendationSuppression.arel_table[:id].desc).to_a + else + account_ids = Redis.current.zrevrange("follow_recommendations:#{@language}", 0, -1).map(&:to_i) + accounts = Account.where(id: account_ids).index_by(&:id) + + account_ids.map { |id| accounts[id] }.compact + end + end +end diff --git a/app/models/follow_recommendation_suppression.rb b/app/models/follow_recommendation_suppression.rb new file mode 100644 index 000000000..170506b85 --- /dev/null +++ b/app/models/follow_recommendation_suppression.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true +# == Schema Information +# +# Table name: follow_recommendation_suppressions +# +# id :bigint(8) not null, primary key +# account_id :bigint(8) not null +# created_at :datetime not null +# updated_at :datetime not null +# + +class FollowRecommendationSuppression < ApplicationRecord + include Redisable + + belongs_to :account + + after_commit :remove_follow_recommendations, on: :create + + private + + def remove_follow_recommendations + redis.pipelined do + I18n.available_locales.each do |locale| + redis.zrem("follow_recommendations:#{locale}", account_id) + end + end + end +end diff --git a/app/models/form/account_batch.rb b/app/models/form/account_batch.rb index 26d6d3abf..698933c9f 100644 --- a/app/models/form/account_batch.rb +++ b/app/models/form/account_batch.rb @@ -21,6 +21,10 @@ class Form::AccountBatch approve! when 'reject' reject! + when 'suppress_follow_recommendation' + suppress_follow_recommendation! + when 'unsuppress_follow_recommendation' + unsuppress_follow_recommendation! end end @@ -79,4 +83,18 @@ class Form::AccountBatch records.each { |account| authorize(account.user, :reject?) } .each { |account| DeleteAccountService.new.call(account, reserve_email: false, reserve_username: false) } end + + def suppress_follow_recommendation! + authorize(:follow_recommendation, :suppress?) + + accounts.each do |account| + FollowRecommendationSuppression.create(account: account) + end + end + + def unsuppress_follow_recommendation! + authorize(:follow_recommendation, :unsuppress?) + + FollowRecommendationSuppression.where(account_id: account_ids).destroy_all + end end diff --git a/app/policies/follow_recommendation_policy.rb b/app/policies/follow_recommendation_policy.rb new file mode 100644 index 000000000..68cd0e547 --- /dev/null +++ b/app/policies/follow_recommendation_policy.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class FollowRecommendationPolicy < ApplicationPolicy + def show? + staff? + end + + def suppress? + staff? + end + + def unsuppress? + staff? + end +end diff --git a/app/serializers/rest/suggestion_serializer.rb b/app/serializers/rest/suggestion_serializer.rb new file mode 100644 index 000000000..3d697fd9f --- /dev/null +++ b/app/serializers/rest/suggestion_serializer.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +class REST::SuggestionSerializer < ActiveModel::Serializer + attributes :source + + has_one :account, serializer: REST::AccountSerializer +end diff --git a/app/views/admin/follow_recommendations/_account.html.haml b/app/views/admin/follow_recommendations/_account.html.haml new file mode 100644 index 000000000..af5a4aaf7 --- /dev/null +++ b/app/views/admin/follow_recommendations/_account.html.haml @@ -0,0 +1,20 @@ +.batch-table__row + %label.batch-table__row__select.batch-table__row__select--aligned.batch-checkbox + = f.check_box :account_ids, { multiple: true, include_hidden: false }, account.id + .batch-table__row__content.batch-table__row__content--unpadded + %table.accounts-table + %tbody + %tr + %td= account_link_to account + %td.accounts-table__count.optional + = number_to_human account.statuses_count, strip_insignificant_zeros: true + %small= t('accounts.posts', count: account.statuses_count).downcase + %td.accounts-table__count.optional + = number_to_human account.followers_count, strip_insignificant_zeros: true + %small= t('accounts.followers', count: account.followers_count).downcase + %td.accounts-table__count + - if account.last_status_at.present? + %time.time-ago{ datetime: account.last_status_at.to_date.iso8601, title: l(account.last_status_at.to_date) }= l account.last_status_at + - else + \- + %small= t('accounts.last_active') diff --git a/app/views/admin/follow_recommendations/show.html.haml b/app/views/admin/follow_recommendations/show.html.haml new file mode 100644 index 000000000..1f050329a --- /dev/null +++ b/app/views/admin/follow_recommendations/show.html.haml @@ -0,0 +1,42 @@ +- content_for :page_title do + = t('admin.follow_recommendations.title') + +- content_for :header_tags do + = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' + +.simple_form + %p.hint= t('admin.follow_recommendations.description_html') + +%hr.spacer/ + += form_tag admin_follow_recommendations_path, method: 'GET', class: 'simple_form' do + .filters + .filter-subset.filter-subset--with-select + %strong= t('admin.follow_recommendations.language') + .input.select.optional + = select_tag :language, options_for_select(I18n.available_locales.map { |key| [human_locale(key), key]}, @language) + + .filter-subset + %strong= t('admin.follow_recommendations.status') + %ul + %li= filter_link_to t('admin.accounts.moderation.active'), status: nil + %li= filter_link_to t('admin.follow_recommendations.suppressed'), status: 'suppressed' + += form_for(@form, url: admin_follow_recommendations_path, method: :patch) do |f| + - RelationshipFilter::KEYS.each do |key| + = hidden_field_tag key, params[key] if params[key].present? + + .batch-table + .batch-table__toolbar + %label.batch-table__toolbar__select.batch-checkbox-all + = check_box_tag :batch_checkbox_all, nil, false + .batch-table__toolbar__actions + - if params[:status].blank? && can?(:suppress, :follow_recommendation) + = f.button safe_join([fa_icon('times'), t('admin.follow_recommendations.suppress')]), name: :suppress, class: 'table-action-link', type: :submit, data: { confirm: t('admin.reports.are_you_sure') } + - if params[:status] == 'suppressed' && can?(:unsuppress, :follow_recommendation) + = f.button safe_join([fa_icon('plus'), t('admin.follow_recommendations.unsuppress')]), name: :unsuppress, class: 'table-action-link', type: :submit + .batch-table__body + - if @accounts.empty? + = nothing_here 'nothing-here--under-tabs' + - else + = render partial: 'account', collection: @accounts, locals: { f: f } diff --git a/app/workers/scheduler/follow_recommendations_scheduler.rb b/app/workers/scheduler/follow_recommendations_scheduler.rb new file mode 100644 index 000000000..0a0286496 --- /dev/null +++ b/app/workers/scheduler/follow_recommendations_scheduler.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +class Scheduler::FollowRecommendationsScheduler + include Sidekiq::Worker + include Redisable + + sidekiq_options retry: 0 + + # The maximum number of accounts that can be requested in one page from the + # API is 80, and the suggestions API does not allow pagination. This number + # leaves some room for accounts being filtered during live access + SET_SIZE = 100 + + def perform + # Maintaining a materialized view speeds-up subsequent queries significantly + AccountSummary.refresh + + fallback_recommendations = FollowRecommendation.safe.filtered.limit(SET_SIZE).index_by(&:account_id) + + I18n.available_locales.each do |locale| + recommendations = begin + if AccountSummary.safe.filtered.localized(locale).exists? # We can skip the work if no accounts with that language exist + FollowRecommendation.safe.filtered.localized(locale).limit(SET_SIZE).index_by(&:account_id) + else + {} + end + end + + # Use language-agnostic results if there are not enough language-specific ones + missing = SET_SIZE - recommendations.keys.size + + if missing.positive? + added = 0 + + # Avoid duplicate results + fallback_recommendations.each_value do |recommendation| + next if recommendations.key?(recommendation.account_id) + + recommendations[recommendation.account_id] = recommendation + added += 1 + + break if added >= missing + end + end + + redis.pipelined do + redis.del(key(locale)) + + recommendations.each_value do |recommendation| + redis.zadd(key(locale), recommendation.rank, recommendation.account_id) + end + end + end + end + + private + + def key(locale) + "follow_recommendations:#{locale}" + end +end diff --git a/config/locales/en.yml b/config/locales/en.yml index 3387b4df6..afab6d9b5 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -440,6 +440,14 @@ en: create: Add domain title: Block new e-mail domain title: Blocked e-mail domains + follow_recommendations: + description_html: "Follow recommendations help new users quickly find interesting content. When a user has not interacted with others enough to form personalized follow recommendations, these accounts are recommended instead. They are re-calculated on a daily basis from a mix of accounts with the highest recent engagements and highest local follower counts for a given language." + language: For language + status: Status + suppress: Suppress follow recommendation + suppressed: Suppressed + title: Follow recommendations + unsuppress: Restore follow recommendation instances: by_domain: Domain delivery_available: Delivery is available diff --git a/config/navigation.rb b/config/navigation.rb index 3a82c7971..b3462c48d 100644 --- a/config/navigation.rb +++ b/config/navigation.rb @@ -39,6 +39,7 @@ SimpleNavigation::Configuration.run do |navigation| s.item :accounts, safe_join([fa_icon('users fw'), t('admin.accounts.title')]), admin_accounts_url, highlights_on: %r{/admin/accounts|/admin/pending_accounts} s.item :invites, safe_join([fa_icon('user-plus fw'), t('admin.invites.title')]), admin_invites_path s.item :tags, safe_join([fa_icon('hashtag fw'), t('admin.tags.title')]), admin_tags_path, highlights_on: %r{/admin/tags} + s.item :follow_recommendations, safe_join([fa_icon('user-plus fw'), t('admin.follow_recommendations.title')]), admin_follow_recommendations_path, highlights_on: %r{/admin/follow_recommendations} s.item :instances, safe_join([fa_icon('cloud fw'), t('admin.instances.title')]), admin_instances_url(limited: whitelist_mode? ? nil : '1'), highlights_on: %r{/admin/instances|/admin/domain_blocks|/admin/domain_allows}, if: -> { current_user.admin? } s.item :email_domain_blocks, safe_join([fa_icon('envelope fw'), t('admin.email_domain_blocks.title')]), admin_email_domain_blocks_url, highlights_on: %r{/admin/email_domain_blocks}, if: -> { current_user.admin? } s.item :ip_blocks, safe_join([fa_icon('ban fw'), t('admin.ip_blocks.title')]), admin_ip_blocks_url, highlights_on: %r{/admin/ip_blocks}, if: -> { current_user.admin? } diff --git a/config/routes.rb b/config/routes.rb index eedd0de69..4661a7c11 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -292,6 +292,7 @@ Rails.application.routes.draw do end resources :account_moderation_notes, only: [:create, :destroy] + resource :follow_recommendations, only: [:show, :update] resources :tags, only: [:index, :show, :update] do collection do @@ -507,6 +508,7 @@ Rails.application.routes.draw do namespace :v2 do resources :media, only: [:create] get '/search', to: 'search#index', as: :search + resources :suggestions, only: [:index] end namespace :web do diff --git a/config/sidekiq.yml b/config/sidekiq.yml index 010923717..a8e4c7feb 100644 --- a/config/sidekiq.yml +++ b/config/sidekiq.yml @@ -25,6 +25,10 @@ cron: '<%= Random.rand(0..59) %> <%= Random.rand(0..2) %> * * *' class: Scheduler::FeedCleanupScheduler queue: scheduler + follow_recommendations_scheduler: + cron: '<%= Random.rand(0..59) %> <%= Random.rand(6..9) %> * * *' + class: Scheduler::FollowRecommendationsScheduler + queue: scheduler doorkeeper_cleanup_scheduler: cron: '<%= Random.rand(0..59) %> <%= Random.rand(0..2) %> * * 0' class: Scheduler::DoorkeeperCleanupScheduler diff --git a/db/migrate/20210322164601_create_account_summaries.rb b/db/migrate/20210322164601_create_account_summaries.rb new file mode 100644 index 000000000..b9faf180d --- /dev/null +++ b/db/migrate/20210322164601_create_account_summaries.rb @@ -0,0 +1,9 @@ +class CreateAccountSummaries < ActiveRecord::Migration[5.2] + def change + create_view :account_summaries, materialized: true + + # To be able to refresh the view concurrently, + # at least one unique index is required + safety_assured { add_index :account_summaries, :account_id, unique: true } + end +end diff --git a/db/migrate/20210323114347_create_follow_recommendations.rb b/db/migrate/20210323114347_create_follow_recommendations.rb new file mode 100644 index 000000000..77e729032 --- /dev/null +++ b/db/migrate/20210323114347_create_follow_recommendations.rb @@ -0,0 +1,5 @@ +class CreateFollowRecommendations < ActiveRecord::Migration[5.2] + def change + create_view :follow_recommendations + end +end diff --git a/db/migrate/20210324171613_create_follow_recommendation_suppressions.rb b/db/migrate/20210324171613_create_follow_recommendation_suppressions.rb new file mode 100644 index 000000000..c17a0be63 --- /dev/null +++ b/db/migrate/20210324171613_create_follow_recommendation_suppressions.rb @@ -0,0 +1,9 @@ +class CreateFollowRecommendationSuppressions < ActiveRecord::Migration[6.1] + def change + create_table :follow_recommendation_suppressions do |t| + t.references :account, null: false, foreign_key: { on_delete: :cascade }, index: { unique: true } + + t.timestamps + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 4edaf5651..28f36abb1 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -2,15 +2,15 @@ # of editing this file, please use the migrations feature of Active Record to # incrementally modify your database, and then regenerate this schema definition. # -# Note that this schema.rb definition is the authoritative source for your -# database schema. If you need to create the application database on another -# system, you should be using db:schema:load, not running all the migrations -# from scratch. The latter is a flawed and unsustainable approach (the more migrations -# you'll amass, the slower it'll run and the greater likelihood for issues). +# This file is the source Rails uses to define your schema when running `bin/rails +# db:schema:load`. When creating a new database, `bin/rails db:schema:load` tends to +# be faster and is potentially less error prone than running all of your +# migrations from scratch. Old migrations may fail to apply correctly if those +# migrations use external dependencies or application code. # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2021_03_08_133107) do +ActiveRecord::Schema.define(version: 2021_03_24_171613) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -406,6 +406,13 @@ ActiveRecord::Schema.define(version: 2021_03_08_133107) do t.index ["tag_id"], name: "index_featured_tags_on_tag_id" end + create_table "follow_recommendation_suppressions", force: :cascade do |t| + t.bigint "account_id", null: false + t.datetime "created_at", precision: 6, null: false + t.datetime "updated_at", precision: 6, null: false + t.index ["account_id"], name: "index_follow_recommendation_suppressions_on_account_id", unique: true + end + create_table "follow_requests", force: :cascade do |t| t.datetime "created_at", null: false t.datetime "updated_at", null: false @@ -996,6 +1003,7 @@ ActiveRecord::Schema.define(version: 2021_03_08_133107) do add_foreign_key "favourites", "statuses", name: "fk_b0e856845e", on_delete: :cascade add_foreign_key "featured_tags", "accounts", on_delete: :cascade add_foreign_key "featured_tags", "tags", on_delete: :cascade + add_foreign_key "follow_recommendation_suppressions", "accounts", on_delete: :cascade add_foreign_key "follow_requests", "accounts", column: "target_account_id", name: "fk_9291ec025d", on_delete: :cascade add_foreign_key "follow_requests", "accounts", name: "fk_76d644b0e7", on_delete: :cascade add_foreign_key "follows", "accounts", column: "target_account_id", name: "fk_745ca29eac", on_delete: :cascade @@ -1079,4 +1087,47 @@ ActiveRecord::Schema.define(version: 2021_03_08_133107) do SQL add_index "instances", ["domain"], name: "index_instances_on_domain", unique: true + create_view "account_summaries", materialized: true, sql_definition: <<-SQL + SELECT accounts.id AS account_id, + mode() WITHIN GROUP (ORDER BY t0.language) AS language, + mode() WITHIN GROUP (ORDER BY t0.sensitive) AS sensitive + FROM (accounts + CROSS JOIN LATERAL ( SELECT statuses.account_id, + statuses.language, + statuses.sensitive + FROM statuses + WHERE ((statuses.account_id = accounts.id) AND (statuses.deleted_at IS NULL)) + ORDER BY statuses.id DESC + LIMIT 20) t0) + WHERE ((accounts.suspended_at IS NULL) AND (accounts.silenced_at IS NULL) AND (accounts.moved_to_account_id IS NULL) AND (accounts.discoverable = true) AND (accounts.locked = false)) + GROUP BY accounts.id; + SQL + add_index "account_summaries", ["account_id"], name: "index_account_summaries_on_account_id", unique: true + + create_view "follow_recommendations", sql_definition: <<-SQL + SELECT t0.account_id, + sum(t0.rank) AS rank, + array_agg(t0.reason) AS reason + FROM ( SELECT accounts.id AS account_id, + ((count(follows.id))::numeric / (1.0 + (count(follows.id))::numeric)) AS rank, + 'most_followed'::text AS reason + FROM ((follows + JOIN accounts ON ((accounts.id = follows.target_account_id))) + JOIN users ON ((users.account_id = follows.account_id))) + WHERE ((users.current_sign_in_at >= (now() - 'P30D'::interval)) AND (accounts.suspended_at IS NULL) AND (accounts.moved_to_account_id IS NULL) AND (accounts.silenced_at IS NULL) AND (accounts.locked = false) AND (accounts.discoverable = true)) + GROUP BY accounts.id + HAVING (count(follows.id) >= 5) + UNION ALL + SELECT accounts.id AS account_id, + (sum((status_stats.reblogs_count + status_stats.favourites_count)) / (1.0 + sum((status_stats.reblogs_count + status_stats.favourites_count)))) AS rank, + 'most_interactions'::text AS reason + FROM ((status_stats + JOIN statuses ON ((statuses.id = status_stats.status_id))) + JOIN accounts ON ((accounts.id = statuses.account_id))) + WHERE ((statuses.id >= (((date_part('epoch'::text, (now() - 'P30D'::interval)) * (1000)::double precision))::bigint << 16)) AND (accounts.suspended_at IS NULL) AND (accounts.moved_to_account_id IS NULL) AND (accounts.silenced_at IS NULL) AND (accounts.locked = false) AND (accounts.discoverable = true)) + GROUP BY accounts.id + HAVING (sum((status_stats.reblogs_count + status_stats.favourites_count)) >= (5)::numeric)) t0 + GROUP BY t0.account_id + ORDER BY (sum(t0.rank)) DESC; + SQL end diff --git a/db/views/account_summaries_v01.sql b/db/views/account_summaries_v01.sql new file mode 100644 index 000000000..5a632b622 --- /dev/null +++ b/db/views/account_summaries_v01.sql @@ -0,0 +1,22 @@ +SELECT + accounts.id AS account_id, + mode() WITHIN GROUP (ORDER BY language ASC) AS language, + mode() WITHIN GROUP (ORDER BY sensitive ASC) AS sensitive +FROM accounts +CROSS JOIN LATERAL ( + SELECT + statuses.account_id, + statuses.language, + statuses.sensitive + FROM statuses + WHERE statuses.account_id = accounts.id + AND statuses.deleted_at IS NULL + ORDER BY statuses.id DESC + LIMIT 20 +) t0 +WHERE accounts.suspended_at IS NULL + AND accounts.silenced_at IS NULL + AND accounts.moved_to_account_id IS NULL + AND accounts.discoverable = 't' + AND accounts.locked = 'f' +GROUP BY accounts.id diff --git a/db/views/follow_recommendations_v01.sql b/db/views/follow_recommendations_v01.sql new file mode 100644 index 000000000..799abeaee --- /dev/null +++ b/db/views/follow_recommendations_v01.sql @@ -0,0 +1,38 @@ +SELECT + account_id, + sum(rank) AS rank, + array_agg(reason) AS reason +FROM ( + SELECT + accounts.id AS account_id, + count(follows.id) / (1.0 + count(follows.id)) AS rank, + 'most_followed' AS reason + FROM follows + INNER JOIN accounts ON accounts.id = follows.target_account_id + INNER JOIN users ON users.account_id = follows.account_id + WHERE users.current_sign_in_at >= (now() - interval '30 days') + AND accounts.suspended_at IS NULL + AND accounts.moved_to_account_id IS NULL + AND accounts.silenced_at IS NULL + AND accounts.locked = 'f' + AND accounts.discoverable = 't' + GROUP BY accounts.id + HAVING count(follows.id) >= 5 + UNION ALL + SELECT accounts.id AS account_id, + sum(reblogs_count + favourites_count) / (1.0 + sum(reblogs_count + favourites_count)) AS rank, + 'most_interactions' AS reason + FROM status_stats + INNER JOIN statuses ON statuses.id = status_stats.status_id + INNER JOIN accounts ON accounts.id = statuses.account_id + WHERE statuses.id >= ((date_part('epoch', now() - interval '30 days') * 1000)::bigint << 16) + AND accounts.suspended_at IS NULL + AND accounts.moved_to_account_id IS NULL + AND accounts.silenced_at IS NULL + AND accounts.locked = 'f' + AND accounts.discoverable = 't' + GROUP BY accounts.id + HAVING sum(reblogs_count + favourites_count) >= 5 +) t0 +GROUP BY account_id +ORDER BY rank DESC diff --git a/spec/fabricators/follow_recommendation_suppression_fabricator.rb b/spec/fabricators/follow_recommendation_suppression_fabricator.rb new file mode 100644 index 000000000..4a6a07a66 --- /dev/null +++ b/spec/fabricators/follow_recommendation_suppression_fabricator.rb @@ -0,0 +1,3 @@ +Fabricator(:follow_recommendation_suppression) do + account +end diff --git a/spec/models/follow_recommendation_suppression_spec.rb b/spec/models/follow_recommendation_suppression_spec.rb new file mode 100644 index 000000000..39107a2b0 --- /dev/null +++ b/spec/models/follow_recommendation_suppression_spec.rb @@ -0,0 +1,4 @@ +require 'rails_helper' + +RSpec.describe FollowRecommendationSuppression, type: :model do +end -- cgit From 71f335c2fcac974af4672cb521751ba6cdee06a6 Mon Sep 17 00:00:00 2001 From: Claire Date: Tue, 13 Apr 2021 23:43:41 +0200 Subject: Add HTTP header to explicitly opt out of FLoC by default (#16036) Fixes #16034 --- config/environments/production.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'config') diff --git a/config/environments/production.rb b/config/environments/production.rb index 6df0a3365..22be14749 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -116,6 +116,7 @@ Rails.application.configure do 'X-Frame-Options' => 'DENY', 'X-Content-Type-Options' => 'nosniff', 'X-XSS-Protection' => '1; mode=block', + 'Permissions-Policy' => 'interest-cohort=()', } config.x.otp_secret = ENV.fetch('OTP_SECRET') -- cgit From 3d82a1de052ff3cf8698985eb3e8c1cd73c7eedd Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Thu, 15 Apr 2021 16:28:20 +0200 Subject: Change option labels on edit profile page (#16041) --- config/locales/simple_form.en.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'config') diff --git a/config/locales/simple_form.en.yml b/config/locales/simple_form.en.yml index 97d5b3122..7146adced 100644 --- a/config/locales/simple_form.en.yml +++ b/config/locales/simple_form.en.yml @@ -30,19 +30,19 @@ en: defaults: autofollow: People who sign up through the invite will automatically follow you avatar: PNG, GIF or JPG. At most %{size}. Will be downscaled to %{dimensions}px - bot: This account mainly performs automated actions and might not be monitored + bot: Signal to others that the account mainly performs automated actions and might not be monitored context: One or multiple contexts where the filter should apply current_password: For security purposes please enter the password of the current account current_username: To confirm, please enter the username of the current account digest: Only sent after a long period of inactivity and only if you have received any personal messages in your absence - discoverable: The profile directory is another way by which your account can reach a wider audience + discoverable: Allow your account to be discovered by strangers through recommendations and other features email: You will be sent a confirmation e-mail fields: You can have up to 4 items displayed as a table on your profile header: PNG, GIF or JPG. At most %{size}. Will be downscaled to %{dimensions}px inbox_url: Copy the URL from the frontpage of the relay you want to use irreversible: Filtered toots will disappear irreversibly, even if filter is later removed locale: The language of the user interface, e-mails and push notifications - locked: Requires you to manually approve followers + locked: Manually control who can follow you by approving follow requests password: Use at least 8 characters phrase: Will be matched regardless of casing in text or content warning of a toot scopes: Which APIs the application will be allowed to access. If you select a top-level scope, you don't need to select individual ones. @@ -51,7 +51,7 @@ en: setting_display_media_default: Hide media marked as sensitive setting_display_media_hide_all: Always hide media setting_display_media_show_all: Always show media - setting_hide_network: Who you follow and who follows you will not be shown on your profile + setting_hide_network: Who you follow and who follows you will be hidden on your profile setting_noindex: Affects your public profile and status pages setting_show_application: The application you use to toot will be displayed in the detailed view of your toots setting_use_blurhash: Gradients are based on the colors of the hidden visuals but obfuscate any details @@ -128,7 +128,7 @@ en: context: Filter contexts current_password: Current password data: Data - discoverable: List this account on the directory + discoverable: Suggest account to others display_name: Display name email: E-mail address expires_in: Expire after @@ -138,7 +138,7 @@ en: inbox_url: URL of the relay inbox irreversible: Drop instead of hide locale: Interface language - locked: Lock account + locked: Require follow requests max_uses: Max number of uses new_password: New password note: Bio @@ -160,7 +160,7 @@ en: setting_display_media_hide_all: Hide all setting_display_media_show_all: Show all setting_expand_spoilers: Always expand toots marked with content warnings - setting_hide_network: Hide your network + setting_hide_network: Hide your social graph setting_noindex: Opt-out of search engine indexing setting_reduce_motion: Reduce motion in animations setting_show_application: Disclose application used to send toots -- cgit From 3b8d085436fa38aed4d5fa3650e433fc7215b104 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Thu, 15 Apr 2021 16:28:43 +0200 Subject: Fix app name, website and redirect URIs not having a maximum length (#16042) Fix app scopes not being validated --- app/lib/application_extension.rb | 4 +- config/initializers/doorkeeper.rb | 5 ++ spec/controllers/api/v1/apps_controller_spec.rb | 78 ++++++++++++++++++++++--- 3 files changed, 77 insertions(+), 10 deletions(-) (limited to 'config') diff --git a/app/lib/application_extension.rb b/app/lib/application_extension.rb index 1d80b8c6d..e61cd0721 100644 --- a/app/lib/application_extension.rb +++ b/app/lib/application_extension.rb @@ -4,6 +4,8 @@ module ApplicationExtension extend ActiveSupport::Concern included do - validates :website, url: true, if: :website? + validates :name, length: { maximum: 60 } + validates :website, url: true, length: { maximum: 2_000 }, if: :website? + validates :redirect_uri, length: { maximum: 2_000 } end end diff --git a/config/initializers/doorkeeper.rb b/config/initializers/doorkeeper.rb index 63cff7c59..f78db8653 100644 --- a/config/initializers/doorkeeper.rb +++ b/config/initializers/doorkeeper.rb @@ -52,6 +52,11 @@ Doorkeeper.configure do # Issue access tokens with refresh token (disabled by default) # use_refresh_token + # Forbids creating/updating applications with arbitrary scopes that are + # not in configuration, i.e. `default_scopes` or `optional_scopes`. + # (Disabled by default) + enforce_configured_scopes + # Provide support for an owner to be assigned to each registered application (disabled by default) # Optional parameter :confirmation => true (default false) if you want to enforce ownership of # a registered application diff --git a/spec/controllers/api/v1/apps_controller_spec.rb b/spec/controllers/api/v1/apps_controller_spec.rb index 60a4c3b41..70cd62d48 100644 --- a/spec/controllers/api/v1/apps_controller_spec.rb +++ b/spec/controllers/api/v1/apps_controller_spec.rb @@ -4,23 +4,83 @@ RSpec.describe Api::V1::AppsController, type: :controller do render_views describe 'POST #create' do + let(:client_name) { 'Test app' } + let(:scopes) { nil } + let(:redirect_uris) { 'urn:ietf:wg:oauth:2.0:oob' } + let(:website) { nil } + + let(:app_params) do + { + client_name: client_name, + redirect_uris: redirect_uris, + scopes: scopes, + website: website, + } + end + before do - post :create, params: { client_name: 'Test app', redirect_uris: 'urn:ietf:wg:oauth:2.0:oob' } + post :create, params: app_params end - it 'returns http success' do - expect(response).to have_http_status(200) + context 'with valid params' do + it 'returns http success' do + expect(response).to have_http_status(200) + end + + it 'creates an OAuth app' do + expect(Doorkeeper::Application.find_by(name: client_name)).to_not be nil + end + + it 'returns client ID and client secret' do + json = body_as_json + + expect(json[:client_id]).to_not be_blank + expect(json[:client_secret]).to_not be_blank + end + end + + context 'with an unsupported scope' do + let(:scopes) { 'hoge' } + + it 'returns http unprocessable entity' do + expect(response).to have_http_status(422) + end end - it 'creates an OAuth app' do - expect(Doorkeeper::Application.find_by(name: 'Test app')).to_not be nil + context 'with many duplicate scopes' do + let(:scopes) { (%w(read) * 40).join(' ') } + + it 'returns http success' do + expect(response).to have_http_status(200) + end + + it 'only saves the scope once' do + expect(Doorkeeper::Application.find_by(name: client_name).scopes.to_s).to eq 'read' + end + end + + context 'with a too-long name' do + let(:client_name) { 'hoge' * 20 } + + it 'returns http unprocessable entity' do + expect(response).to have_http_status(422) + end + end + + context 'with a too-long website' do + let(:website) { 'https://foo.bar/' + ('hoge' * 2_000) } + + it 'returns http unprocessable entity' do + expect(response).to have_http_status(422) + end end - it 'returns client ID and client secret' do - json = body_as_json + context 'with a too-long redirect_uris' do + let(:redirect_uris) { 'https://foo.bar/' + ('hoge' * 2_000) } - expect(json[:client_id]).to_not be_blank - expect(json[:client_secret]).to_not be_blank + it 'returns http unprocessable entity' do + expect(response).to have_http_status(422) + end end end end -- cgit From 480d7c9478a9be3ac668b21402a269db41b10b18 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sat, 17 Apr 2021 11:12:49 +0200 Subject: Fix missing source strings and inconsistent lead text style in admin UI (#16052) --- app/helpers/application_helper.rb | 2 -- app/views/admin/follow_recommendations/show.html.haml | 3 +-- app/views/admin/rules/index.html.haml | 5 +++-- app/views/user_mailer/webauthn_enabled.text.erb | 4 ++-- config/locales/en.yml | 7 ++++++- 5 files changed, 12 insertions(+), 9 deletions(-) (limited to 'config') diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb index bf5742d34..fc2d2fea9 100644 --- a/app/helpers/application_helper.rb +++ b/app/helpers/application_helper.rb @@ -91,8 +91,6 @@ module ApplicationHelper fa_icon('unlock', title: I18n.t('statuses.visibilities.unlisted')) elsif status.private_visibility? || status.limited_visibility? fa_icon('lock', title: I18n.t('statuses.visibilities.private')) - elsif status.direct_visibility? - fa_icon('envelope', title: I18n.t('statuses.visibilities.direct')) end end diff --git a/app/views/admin/follow_recommendations/show.html.haml b/app/views/admin/follow_recommendations/show.html.haml index 1f050329a..5b949a165 100644 --- a/app/views/admin/follow_recommendations/show.html.haml +++ b/app/views/admin/follow_recommendations/show.html.haml @@ -4,8 +4,7 @@ - content_for :header_tags do = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous' -.simple_form - %p.hint= t('admin.follow_recommendations.description_html') +%p= t('admin.follow_recommendations.description_html') %hr.spacer/ diff --git a/app/views/admin/rules/index.html.haml b/app/views/admin/rules/index.html.haml index 3b069d083..4fb993ad0 100644 --- a/app/views/admin/rules/index.html.haml +++ b/app/views/admin/rules/index.html.haml @@ -1,8 +1,9 @@ - content_for :page_title do = t('admin.rules.title') -.simple_form - %p.hint= t('admin.rules.description') +%p= t('admin.rules.description_html') + +%hr.spacer/ - if can? :create, :rule = simple_form_for @rule, url: admin_rules_path do |f| diff --git a/app/views/user_mailer/webauthn_enabled.text.erb b/app/views/user_mailer/webauthn_enabled.text.erb index 4c233fefb..d4482a69b 100644 --- a/app/views/user_mailer/webauthn_enabled.text.erb +++ b/app/views/user_mailer/webauthn_enabled.text.erb @@ -1,7 +1,7 @@ -<%= t 'devise.mailer.webauthn_credentia.added.title' %> +<%= t 'devise.mailer.webauthn_credential.added.title' %> === -<%= t 'devise.mailer.webauthn_credentia.added.explanation' %> +<%= t 'devise.mailer.webauthn_credential.added.explanation' %> => <%= edit_user_registration_url %> diff --git a/config/locales/en.yml b/config/locales/en.yml index afab6d9b5..88acf3164 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -315,10 +315,12 @@ en: new: create: Create announcement title: New announcement + publish: Publish published_msg: Announcement successfully published! scheduled_for: Scheduled for %{time} scheduled_msg: Announcement scheduled for publication! title: Announcements + unpublish: Unpublish unpublished_msg: Announcement successfully unpublished! updated_msg: Announcement successfully updated! custom_emojis: @@ -552,8 +554,10 @@ en: updated_at: Updated rules: add_new: Add rule - description: While most claim to have read and agree to the terms of service, usually people do not read through until after a problem arises. Make it easier to see your server's rules at a glance by providing them in a flat bullet point list. Try to keep individual rules short and simple, but try not to split them up into many separate items either. + delete: Delete + description_html: While most claim to have read and agree to the terms of service, usually people do not read through until after a problem arises. Make it easier to see your server's rules at a glance by providing them in a flat bullet point list. Try to keep individual rules short and simple, but try not to split them up into many separate items either. edit: Edit rule + empty: No server rules have been defined yet. title: Server rules settings: activity_api_enabled: @@ -695,6 +699,7 @@ en: add_new: Add new delete: Delete edit_preset: Edit warning preset + empty: You haven't defined any warning presets yet. title: Manage warning presets admin_mailer: new_pending_account: -- cgit From 1efcbb9cfe36d4428ef6af988bb0d7b25ace3971 Mon Sep 17 00:00:00 2001 From: Claire Date: Mon, 19 Apr 2021 18:41:29 +0200 Subject: Add Message-ID header to outgoing emails (#16076) * Add Message-ID header to outgoing emails * Use email domain name from SMTP_FROM_ADDRESS, fallback on WEB_DOMAIN on failure * Use notifications@localhost as fallback for SMTP_FROM_ADDRESS, do not catch parse errors --- config/environments/production.rb | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'config') diff --git a/config/environments/production.rb b/config/environments/production.rb index 22be14749..df6b07d77 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -90,9 +90,12 @@ Rails.application.configure do config.action_mailer.perform_caching = false # E-mails + outgoing_email_address = ENV.fetch('SMTP_FROM_ADDRESS', 'notifications@localhost') + outgoing_mail_domain = Mail::Address.new(outgoing_email_address).domain config.action_mailer.default_options = { - from: ENV.fetch('SMTP_FROM_ADDRESS', 'notifications@localhost'), - reply_to: ENV['SMTP_REPLY_TO'] + from: outgoing_email_address, + reply_to: ENV['SMTP_REPLY_TO'], + 'Message-ID': -> { "<#{Mail.random_tag}@#{outgoing_mail_domain}>" }, } config.action_mailer.smtp_settings = { -- cgit