From f7117646afddb2676e9275d8efe90c3a20c59021 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Mon, 12 Apr 2021 12:37:14 +0200
Subject: Add cold-start follow recommendations (#15945)

---
 spec/models/follow_recommendation_suppression_spec.rb | 4 ++++
 1 file changed, 4 insertions(+)
 create mode 100644 spec/models/follow_recommendation_suppression_spec.rb

(limited to 'spec/models')

diff --git a/spec/models/follow_recommendation_suppression_spec.rb b/spec/models/follow_recommendation_suppression_spec.rb
new file mode 100644
index 000000000..39107a2b0
--- /dev/null
+++ b/spec/models/follow_recommendation_suppression_spec.rb
@@ -0,0 +1,4 @@
+require 'rails_helper'
+
+RSpec.describe FollowRecommendationSuppression, type: :model do
+end
-- 
cgit 


From ce2148c57111981be455a9953d3bb589cf53967f Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Thu, 15 Apr 2021 05:00:25 +0200
Subject: Add `policy` param to `POST /api/v1/push/subscriptions` (#16040)

With possible values `all`, `followed`, `follower`, and `none`,
control from whom notifications will generate a Web Push alert
---
 .../api/v1/push/subscriptions_controller.rb        | 28 +++----
 .../api/web/push_subscriptions_controller.rb       | 25 +++---
 app/models/web/push_subscription.rb                | 23 +++++-
 .../api/v1/push/subscriptions_controller_spec.rb   | 28 ++++---
 .../api/web/push_subscriptions_controller_spec.rb  | 23 ++++--
 spec/models/web/push_subscription_spec.rb          | 94 ++++++++++++++++++++--
 6 files changed, 170 insertions(+), 51 deletions(-)

(limited to 'spec/models')

diff --git a/app/controllers/api/v1/push/subscriptions_controller.rb b/app/controllers/api/v1/push/subscriptions_controller.rb
index 0918c61e9..47f2e6440 100644
--- a/app/controllers/api/v1/push/subscriptions_controller.rb
+++ b/app/controllers/api/v1/push/subscriptions_controller.rb
@@ -3,13 +3,13 @@
 class Api::V1::Push::SubscriptionsController < Api::BaseController
   before_action -> { doorkeeper_authorize! :push }
   before_action :require_user!
-  before_action :set_web_push_subscription
-  before_action :check_web_push_subscription, only: [:show, :update]
+  before_action :set_push_subscription
+  before_action :check_push_subscription, only: [:show, :update]
 
   def create
-    @web_subscription&.destroy!
+    @push_subscription&.destroy!
 
-    @web_subscription = ::Web::PushSubscription.create!(
+    @push_subscription = Web::PushSubscription.create!(
       endpoint: subscription_params[:endpoint],
       key_p256dh: subscription_params[:keys][:p256dh],
       key_auth: subscription_params[:keys][:auth],
@@ -18,31 +18,31 @@ class Api::V1::Push::SubscriptionsController < Api::BaseController
       access_token_id: doorkeeper_token.id
     )
 
-    render json: @web_subscription, serializer: REST::WebPushSubscriptionSerializer
+    render json: @push_subscription, serializer: REST::WebPushSubscriptionSerializer
   end
 
   def show
-    render json: @web_subscription, serializer: REST::WebPushSubscriptionSerializer
+    render json: @push_subscription, serializer: REST::WebPushSubscriptionSerializer
   end
 
   def update
-    @web_subscription.update!(data: data_params)
-    render json: @web_subscription, serializer: REST::WebPushSubscriptionSerializer
+    @push_subscription.update!(data: data_params)
+    render json: @push_subscription, serializer: REST::WebPushSubscriptionSerializer
   end
 
   def destroy
-    @web_subscription&.destroy!
+    @push_subscription&.destroy!
     render_empty
   end
 
   private
 
-  def set_web_push_subscription
-    @web_subscription = ::Web::PushSubscription.find_by(access_token_id: doorkeeper_token.id)
+  def set_push_subscription
+    @push_subscription = Web::PushSubscription.find_by(access_token_id: doorkeeper_token.id)
   end
 
-  def check_web_push_subscription
-    not_found if @web_subscription.nil?
+  def check_push_subscription
+    not_found if @push_subscription.nil?
   end
 
   def subscription_params
@@ -52,6 +52,6 @@ class Api::V1::Push::SubscriptionsController < Api::BaseController
   def data_params
     return {} if params[:data].blank?
 
-    params.require(:data).permit(alerts: [:follow, :follow_request, :favourite, :reblog, :mention, :poll, :status])
+    params.require(:data).permit(:policy, alerts: [:follow, :follow_request, :favourite, :reblog, :mention, :poll, :status])
   end
 end
diff --git a/app/controllers/api/web/push_subscriptions_controller.rb b/app/controllers/api/web/push_subscriptions_controller.rb
index 1dce3e70f..bed57fc54 100644
--- a/app/controllers/api/web/push_subscriptions_controller.rb
+++ b/app/controllers/api/web/push_subscriptions_controller.rb
@@ -2,6 +2,7 @@
 
 class Api::Web::PushSubscriptionsController < Api::Web::BaseController
   before_action :require_user!
+  before_action :set_push_subscription, only: :update
 
   def create
     active_session = current_session
@@ -15,9 +16,11 @@ class Api::Web::PushSubscriptionsController < Api::Web::BaseController
     alerts_enabled = active_session.detection.device.mobile? || active_session.detection.device.tablet?
 
     data = {
+      policy: 'all',
+
       alerts: {
         follow: alerts_enabled,
-        follow_request: false,
+        follow_request: alerts_enabled,
         favourite: alerts_enabled,
         reblog: alerts_enabled,
         mention: alerts_enabled,
@@ -28,7 +31,7 @@ class Api::Web::PushSubscriptionsController < Api::Web::BaseController
 
     data.deep_merge!(data_params) if params[:data]
 
-    web_subscription = ::Web::PushSubscription.create!(
+    push_subscription = ::Web::PushSubscription.create!(
       endpoint: subscription_params[:endpoint],
       key_p256dh: subscription_params[:keys][:p256dh],
       key_auth: subscription_params[:keys][:auth],
@@ -37,27 +40,27 @@ class Api::Web::PushSubscriptionsController < Api::Web::BaseController
       access_token_id: active_session.access_token_id
     )
 
-    active_session.update!(web_push_subscription: web_subscription)
+    active_session.update!(web_push_subscription: push_subscription)
 
-    render json: web_subscription, serializer: REST::WebPushSubscriptionSerializer
+    render json: push_subscription, serializer: REST::WebPushSubscriptionSerializer
   end
 
   def update
-    params.require([:id])
-
-    web_subscription = ::Web::PushSubscription.find(params[:id])
-    web_subscription.update!(data: data_params)
-
-    render json: web_subscription, serializer: REST::WebPushSubscriptionSerializer
+    @push_subscription.update!(data: data_params)
+    render json: @push_subscription, serializer: REST::WebPushSubscriptionSerializer
   end
 
   private
 
+  def set_push_subscription
+    @push_subscription = ::Web::PushSubscription.find(params[:id])
+  end
+
   def subscription_params
     @subscription_params ||= params.require(:subscription).permit(:endpoint, keys: [:auth, :p256dh])
   end
 
   def data_params
-    @data_params ||= params.require(:data).permit(alerts: [:follow, :follow_request, :favourite, :reblog, :mention, :poll, :status])
+    @data_params ||= params.require(:data).permit(:policy, alerts: [:follow, :follow_request, :favourite, :reblog, :mention, :poll, :status])
   end
 end
diff --git a/app/models/web/push_subscription.rb b/app/models/web/push_subscription.rb
index 7609b1bfc..6e46573ae 100644
--- a/app/models/web/push_subscription.rb
+++ b/app/models/web/push_subscription.rb
@@ -47,7 +47,7 @@ class Web::PushSubscription < ApplicationRecord
   end
 
   def pushable?(notification)
-    ActiveModel::Type::Boolean.new.cast(data&.dig('alerts', notification.type.to_s))
+    policy_allows_notification?(notification) && alert_enabled_for_notification_type?(notification)
   end
 
   def associated_user
@@ -100,4 +100,25 @@ class Web::PushSubscription < ApplicationRecord
   def contact_email
     @contact_email ||= ::Setting.site_contact_email
   end
+
+  def alert_enabled_for_notification_type?(notification)
+    truthy?(data&.dig('alerts', notification.type.to_s))
+  end
+
+  def policy_allows_notification?(notification)
+    case data&.dig('policy')
+    when nil, 'all'
+      true
+    when 'none'
+      false
+    when 'followed'
+      notification.account.following?(notification.from_account)
+    when 'follower'
+      notification.from_account.following?(notification.account)
+    end
+  end
+
+  def truthy?(val)
+    ActiveModel::Type::Boolean.new.cast(val)
+  end
 end
diff --git a/spec/controllers/api/v1/push/subscriptions_controller_spec.rb b/spec/controllers/api/v1/push/subscriptions_controller_spec.rb
index 01146294f..534d02879 100644
--- a/spec/controllers/api/v1/push/subscriptions_controller_spec.rb
+++ b/spec/controllers/api/v1/push/subscriptions_controller_spec.rb
@@ -27,20 +27,27 @@ describe Api::V1::Push::SubscriptionsController do
   let(:alerts_payload) do
     {
       data: {
+        policy: 'all',
+
         alerts: {
           follow: true,
+          follow_request: true,
           favourite: false,
           reblog: true,
           mention: false,
+          poll: true,
+          status: false,
         }
       }
     }.with_indifferent_access
   end
 
   describe 'POST #create' do
-    it 'saves push subscriptions' do
+    before do
       post :create, params: create_payload
+    end
 
+    it 'saves push subscriptions' do
       push_subscription = Web::PushSubscription.find_by(endpoint: create_payload[:subscription][:endpoint])
 
       expect(push_subscription.endpoint).to eq(create_payload[:subscription][:endpoint])
@@ -52,31 +59,34 @@ describe Api::V1::Push::SubscriptionsController do
 
     it 'replaces old subscription on repeat calls' do
       post :create, params: create_payload
-      post :create, params: create_payload
-
       expect(Web::PushSubscription.where(endpoint: create_payload[:subscription][:endpoint]).count).to eq 1
     end
   end
 
   describe 'PUT #update' do
-    it 'changes alert settings' do
+    before do
       post :create, params: create_payload
       put :update, params: alerts_payload
+    end
 
+    it 'changes alert settings' do
       push_subscription = Web::PushSubscription.find_by(endpoint: create_payload[:subscription][:endpoint])
 
-      expect(push_subscription.data.dig('alerts', 'follow')).to eq(alerts_payload[:data][:alerts][:follow].to_s)
-      expect(push_subscription.data.dig('alerts', 'favourite')).to eq(alerts_payload[:data][:alerts][:favourite].to_s)
-      expect(push_subscription.data.dig('alerts', 'reblog')).to eq(alerts_payload[:data][:alerts][:reblog].to_s)
-      expect(push_subscription.data.dig('alerts', 'mention')).to eq(alerts_payload[:data][:alerts][:mention].to_s)
+      expect(push_subscription.data['policy']).to eq(alerts_payload[:data][:policy])
+
+      %w(follow follow_request favourite reblog mention poll status).each do |type|
+        expect(push_subscription.data['alerts'][type]).to eq(alerts_payload[:data][:alerts][type.to_sym].to_s)
+      end
     end
   end
 
   describe 'DELETE #destroy' do
-    it 'removes the subscription' do
+    before do
       post :create, params: create_payload
       delete :destroy
+    end
 
+    it 'removes the subscription' do
       expect(Web::PushSubscription.find_by(endpoint: create_payload[:subscription][:endpoint])).to be_nil
     end
   end
diff --git a/spec/controllers/api/web/push_subscriptions_controller_spec.rb b/spec/controllers/api/web/push_subscriptions_controller_spec.rb
index 381cdeab9..bda4a7661 100644
--- a/spec/controllers/api/web/push_subscriptions_controller_spec.rb
+++ b/spec/controllers/api/web/push_subscriptions_controller_spec.rb
@@ -22,11 +22,16 @@ describe Api::Web::PushSubscriptionsController do
   let(:alerts_payload) do
     {
       data: {
+        policy: 'all',
+
         alerts: {
           follow: true,
+          follow_request: false,
           favourite: false,
           reblog: true,
           mention: false,
+          poll: true,
+          status: false,
         }
       }
     }
@@ -59,10 +64,11 @@ describe Api::Web::PushSubscriptionsController do
 
         push_subscription = Web::PushSubscription.find_by(endpoint: create_payload[:subscription][:endpoint])
 
-        expect(push_subscription.data['alerts']['follow']).to eq(alerts_payload[:data][:alerts][:follow].to_s)
-        expect(push_subscription.data['alerts']['favourite']).to eq(alerts_payload[:data][:alerts][:favourite].to_s)
-        expect(push_subscription.data['alerts']['reblog']).to eq(alerts_payload[:data][:alerts][:reblog].to_s)
-        expect(push_subscription.data['alerts']['mention']).to eq(alerts_payload[:data][:alerts][:mention].to_s)
+        expect(push_subscription.data['policy']).to eq 'all'
+
+        %w(follow follow_request favourite reblog mention poll status).each do |type|
+          expect(push_subscription.data['alerts'][type]).to eq(alerts_payload[:data][:alerts][type.to_sym].to_s)
+        end
       end
     end
   end
@@ -81,10 +87,11 @@ describe Api::Web::PushSubscriptionsController do
 
       push_subscription = Web::PushSubscription.find_by(endpoint: create_payload[:subscription][:endpoint])
 
-      expect(push_subscription.data['alerts']['follow']).to eq(alerts_payload[:data][:alerts][:follow].to_s)
-      expect(push_subscription.data['alerts']['favourite']).to eq(alerts_payload[:data][:alerts][:favourite].to_s)
-      expect(push_subscription.data['alerts']['reblog']).to eq(alerts_payload[:data][:alerts][:reblog].to_s)
-      expect(push_subscription.data['alerts']['mention']).to eq(alerts_payload[:data][:alerts][:mention].to_s)
+      expect(push_subscription.data['policy']).to eq 'all'
+
+      %w(follow follow_request favourite reblog mention poll status).each do |type|
+        expect(push_subscription.data['alerts'][type]).to eq(alerts_payload[:data][:alerts][type.to_sym].to_s)
+      end
     end
   end
 end
diff --git a/spec/models/web/push_subscription_spec.rb b/spec/models/web/push_subscription_spec.rb
index c6665611c..b44904369 100644
--- a/spec/models/web/push_subscription_spec.rb
+++ b/spec/models/web/push_subscription_spec.rb
@@ -1,16 +1,94 @@
 require 'rails_helper'
 
 RSpec.describe Web::PushSubscription, type: :model do
-  let(:alerts) { { mention: true, reblog: false, follow: true, follow_request: false, favourite: true } }
-  let(:push_subscription) { Web::PushSubscription.new(data: { alerts: alerts }) }
+  let(:account) { Fabricate(:account) }
+
+  let(:policy) { 'all' }
+
+  let(:data) do
+    {
+      policy: policy,
+
+      alerts: {
+        mention: true,
+        reblog: false,
+        follow: true,
+        follow_request: false,
+        favourite: true,
+      },
+    }
+  end
+
+  subject { described_class.new(data: data) }
 
   describe '#pushable?' do
-    it 'obeys alert settings' do
-      expect(push_subscription.send(:pushable?, Notification.new(activity_type: 'Mention'))).to eq true
-      expect(push_subscription.send(:pushable?, Notification.new(activity_type: 'Status'))).to eq false
-      expect(push_subscription.send(:pushable?, Notification.new(activity_type: 'Follow'))).to eq true
-      expect(push_subscription.send(:pushable?, Notification.new(activity_type: 'FollowRequest'))).to eq false
-      expect(push_subscription.send(:pushable?, Notification.new(activity_type: 'Favourite'))).to eq true
+    let(:notification_type) { :mention }
+    let(:notification) { Fabricate(:notification, account: account, type: notification_type) }
+
+    %i(mention reblog follow follow_request favourite).each do |type|
+      context "when notification is a #{type}" do
+        let(:notification_type) { type }
+
+        it "returns boolean corresonding to alert setting" do
+          expect(subject.pushable?(notification)).to eq data[:alerts][type]
+        end
+      end
+    end
+
+    context 'when policy is all' do
+      let(:policy) { 'all' }
+
+      it 'returns true' do
+        expect(subject.pushable?(notification)).to eq true
+      end
+    end
+
+    context 'when policy is none' do
+      let(:policy) { 'none' }
+
+      it 'returns false' do
+        expect(subject.pushable?(notification)).to eq false
+      end
+    end
+
+    context 'when policy is followed' do
+      let(:policy) { 'followed' }
+
+      context 'and notification is from someone you follow' do
+        before do
+          account.follow!(notification.from_account)
+        end
+
+        it 'returns true' do
+          expect(subject.pushable?(notification)).to eq true
+        end
+      end
+
+      context 'and notification is not from someone you follow' do
+        it 'returns false' do
+          expect(subject.pushable?(notification)).to eq false
+        end
+      end
+    end
+
+    context 'when policy is follower' do
+      let(:policy) { 'follower' }
+
+      context 'and notification is from someone who follows you' do
+        before do
+          notification.from_account.follow!(account)
+        end
+
+        it 'returns true' do
+          expect(subject.pushable?(notification)).to eq true
+        end
+      end
+
+      context 'and notification is not from someone who follows you' do
+        it 'returns false' do
+          expect(subject.pushable?(notification)).to eq false
+        end
+      end
     end
   end
 end
-- 
cgit 


From b3ceb3dcc4df62803aa967d7aecee686973a8996 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sat, 17 Apr 2021 03:14:25 +0200
Subject: Add canonical e-mail blocks for suspended accounts (#16049)

Prevent new accounts from being created using the same underlying
e-mail as a suspended account using extensions and period
permutations. Stores e-mails as a SHA256 hash
---
 app/helpers/email_helper.rb                        | 18 +++++++++
 app/models/account.rb                              | 14 +++++++
 app/models/canonical_email_block.rb                | 27 +++++++++++++
 app/validators/blacklisted_email_validator.rb      | 30 +++++++++-----
 ...20210416200740_create_canonical_email_blocks.rb | 10 +++++
 db/schema.rb                                       | 12 +++++-
 .../canonical_email_block_fabricator.rb            |  4 ++
 spec/models/canonical_email_block_spec.rb          | 47 ++++++++++++++++++++++
 .../validators/blacklisted_email_validator_spec.rb | 29 +++++++++----
 9 files changed, 171 insertions(+), 20 deletions(-)
 create mode 100644 app/helpers/email_helper.rb
 create mode 100644 app/models/canonical_email_block.rb
 create mode 100644 db/migrate/20210416200740_create_canonical_email_blocks.rb
 create mode 100644 spec/fabricators/canonical_email_block_fabricator.rb
 create mode 100644 spec/models/canonical_email_block_spec.rb

(limited to 'spec/models')

diff --git a/app/helpers/email_helper.rb b/app/helpers/email_helper.rb
new file mode 100644
index 000000000..360783c62
--- /dev/null
+++ b/app/helpers/email_helper.rb
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module EmailHelper
+  def self.included(base)
+    base.extend(self)
+  end
+
+  def email_to_canonical_email(str)
+    username, domain = str.downcase.split('@', 2)
+    username, = username.gsub('.', '').split('+', 2)
+
+    "#{username}@#{domain}"
+  end
+
+  def email_to_canonical_email_hash(str)
+    Digest::SHA2.new(256).hexdigest(email_to_canonical_email(str))
+  end
+end
diff --git a/app/models/account.rb b/app/models/account.rb
index 80689d4aa..a573365de 100644
--- a/app/models/account.rb
+++ b/app/models/account.rb
@@ -235,6 +235,7 @@ class Account < ApplicationRecord
     transaction do
       create_deletion_request!
       update!(suspended_at: date, suspension_origin: origin)
+      create_canonical_email_block!
     end
   end
 
@@ -242,6 +243,7 @@ class Account < ApplicationRecord
     transaction do
       deletion_request&.destroy!
       update!(suspended_at: nil, suspension_origin: nil)
+      destroy_canonical_email_block!
     end
   end
 
@@ -569,4 +571,16 @@ class Account < ApplicationRecord
   def clean_feed_manager
     FeedManager.instance.clean_feeds!(:home, [id])
   end
+
+  def create_canonical_email_block!
+    return unless local? && user_email.present?
+
+    CanonicalEmailBlock.create(reference_account: self, email: user_email)
+  end
+
+  def destroy_canonical_email_block!
+    return unless local?
+
+    CanonicalEmailBlock.where(reference_account: self).delete_all
+  end
 end
diff --git a/app/models/canonical_email_block.rb b/app/models/canonical_email_block.rb
new file mode 100644
index 000000000..a8546d65a
--- /dev/null
+++ b/app/models/canonical_email_block.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+# == Schema Information
+#
+# Table name: canonical_email_blocks
+#
+#  id                   :bigint(8)        not null, primary key
+#  canonical_email_hash :string           default(""), not null
+#  reference_account_id :bigint(8)        not null
+#  created_at           :datetime         not null
+#  updated_at           :datetime         not null
+#
+
+class CanonicalEmailBlock < ApplicationRecord
+  include EmailHelper
+
+  belongs_to :reference_account, class_name: 'Account'
+
+  validates :canonical_email_hash, presence: true
+
+  def email=(email)
+    self.canonical_email_hash = email_to_canonical_email_hash(email)
+  end
+
+  def self.block?(email)
+    where(canonical_email_hash: email_to_canonical_email_hash(email)).exists?
+  end
+end
diff --git a/app/validators/blacklisted_email_validator.rb b/app/validators/blacklisted_email_validator.rb
index 1ca73fdcc..eb66ad93d 100644
--- a/app/validators/blacklisted_email_validator.rb
+++ b/app/validators/blacklisted_email_validator.rb
@@ -6,26 +6,25 @@ class BlacklistedEmailValidator < ActiveModel::Validator
 
     @email = user.email
 
-    user.errors.add(:email, :blocked) if blocked_email?
+    user.errors.add(:email, :blocked) if blocked_email_provider?
+    user.errors.add(:email, :taken) if blocked_canonical_email?
   end
 
   private
 
-  def blocked_email?
-    on_blacklist? || not_on_whitelist?
+  def blocked_email_provider?
+    disallowed_through_email_domain_block? || disallowed_through_configuration? || not_allowed_through_configuration?
   end
 
-  def on_blacklist?
-    return true  if EmailDomainBlock.block?(@email)
-    return false if Rails.configuration.x.email_domains_blacklist.blank?
-
-    domains = Rails.configuration.x.email_domains_blacklist.gsub('.', '\.')
-    regexp  = Regexp.new("@(.+\\.)?(#{domains})", true)
+  def blocked_canonical_email?
+    CanonicalEmailBlock.block?(@email)
+  end
 
-    regexp.match?(@email)
+  def disallowed_through_email_domain_block?
+    EmailDomainBlock.block?(@email)
   end
 
-  def not_on_whitelist?
+  def not_allowed_through_configuration?
     return false if Rails.configuration.x.email_domains_whitelist.blank?
 
     domains = Rails.configuration.x.email_domains_whitelist.gsub('.', '\.')
@@ -33,4 +32,13 @@ class BlacklistedEmailValidator < ActiveModel::Validator
 
     @email !~ regexp
   end
+
+  def disallowed_through_configuration?
+    return false if Rails.configuration.x.email_domains_blacklist.blank?
+
+    domains = Rails.configuration.x.email_domains_blacklist.gsub('.', '\.')
+    regexp  = Regexp.new("@(.+\\.)?(#{domains})", true)
+
+    regexp.match?(@email)
+  end
 end
diff --git a/db/migrate/20210416200740_create_canonical_email_blocks.rb b/db/migrate/20210416200740_create_canonical_email_blocks.rb
new file mode 100644
index 000000000..a1f1660bf
--- /dev/null
+++ b/db/migrate/20210416200740_create_canonical_email_blocks.rb
@@ -0,0 +1,10 @@
+class CreateCanonicalEmailBlocks < ActiveRecord::Migration[6.1]
+  def change
+    create_table :canonical_email_blocks do |t|
+      t.string :canonical_email_hash, null: false, default: '', index: { unique: true }
+      t.belongs_to :reference_account, null: false, foreign_key: { on_cascade: :delete, to_table: 'accounts' }
+
+      t.timestamps
+    end
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 599cd873b..dcbbf4aea 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2021_03_24_171613) do
+ActiveRecord::Schema.define(version: 2021_04_16_200740) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -280,6 +280,15 @@ ActiveRecord::Schema.define(version: 2021_03_24_171613) do
     t.index ["status_id"], name: "index_bookmarks_on_status_id"
   end
 
+  create_table "canonical_email_blocks", force: :cascade do |t|
+    t.string "canonical_email_hash", default: "", null: false
+    t.bigint "reference_account_id", null: false
+    t.datetime "created_at", precision: 6, null: false
+    t.datetime "updated_at", precision: 6, null: false
+    t.index ["canonical_email_hash"], name: "index_canonical_email_blocks_on_canonical_email_hash", unique: true
+    t.index ["reference_account_id"], name: "index_canonical_email_blocks_on_reference_account_id"
+  end
+
   create_table "conversation_mutes", force: :cascade do |t|
     t.bigint "conversation_id", null: false
     t.bigint "account_id", null: false
@@ -991,6 +1000,7 @@ ActiveRecord::Schema.define(version: 2021_03_24_171613) do
   add_foreign_key "blocks", "accounts", name: "fk_4269e03e65", on_delete: :cascade
   add_foreign_key "bookmarks", "accounts", on_delete: :cascade
   add_foreign_key "bookmarks", "statuses", on_delete: :cascade
+  add_foreign_key "canonical_email_blocks", "accounts", column: "reference_account_id"
   add_foreign_key "conversation_mutes", "accounts", name: "fk_225b4212bb", on_delete: :cascade
   add_foreign_key "conversation_mutes", "conversations", on_delete: :cascade
   add_foreign_key "custom_filters", "accounts", on_delete: :cascade
diff --git a/spec/fabricators/canonical_email_block_fabricator.rb b/spec/fabricators/canonical_email_block_fabricator.rb
new file mode 100644
index 000000000..a0b6e0d22
--- /dev/null
+++ b/spec/fabricators/canonical_email_block_fabricator.rb
@@ -0,0 +1,4 @@
+Fabricator(:canonical_email_block) do
+  email "test@example.com"
+  reference_account { Fabricate(:account) }
+end
diff --git a/spec/models/canonical_email_block_spec.rb b/spec/models/canonical_email_block_spec.rb
new file mode 100644
index 000000000..8e0050d65
--- /dev/null
+++ b/spec/models/canonical_email_block_spec.rb
@@ -0,0 +1,47 @@
+require 'rails_helper'
+
+RSpec.describe CanonicalEmailBlock, type: :model do
+  describe '#email=' do
+    let(:target_hash) { '973dfe463ec85785f5f95af5ba3906eedb2d931c24e69824a89ea65dba4e813b' }
+
+    it 'sets canonical_email_hash' do
+      subject.email = 'test@example.com'
+      expect(subject.canonical_email_hash).to eq target_hash
+    end
+
+    it 'sets the same hash even with dot permutations' do
+      subject.email = 't.e.s.t@example.com'
+      expect(subject.canonical_email_hash).to eq target_hash
+    end
+
+    it 'sets the same hash even with extensions' do
+      subject.email = 'test+mastodon1@example.com'
+      expect(subject.canonical_email_hash).to eq target_hash
+    end
+
+    it 'sets the same hash with different casing' do
+      subject.email = 'Test@EXAMPLE.com'
+      expect(subject.canonical_email_hash).to eq target_hash
+    end
+  end
+
+  describe '.block?' do
+    let!(:canonical_email_block) { Fabricate(:canonical_email_block, email: 'foo@bar.com') }
+
+    it 'returns true for the same email' do
+      expect(described_class.block?('foo@bar.com')).to be true
+    end
+
+    it 'returns true for the same email with dots' do
+      expect(described_class.block?('f.oo@bar.com')).to be true
+    end
+
+    it 'returns true for the same email with extensions' do
+      expect(described_class.block?('foo+spam@bar.com')).to be true
+    end
+
+    it 'returns false for different email' do
+      expect(described_class.block?('hoge@bar.com')).to be false
+    end
+  end
+end
diff --git a/spec/validators/blacklisted_email_validator_spec.rb b/spec/validators/blacklisted_email_validator_spec.rb
index 53b355a57..f7d5e01bc 100644
--- a/spec/validators/blacklisted_email_validator_spec.rb
+++ b/spec/validators/blacklisted_email_validator_spec.rb
@@ -9,23 +9,36 @@ RSpec.describe BlacklistedEmailValidator, type: :validator do
 
     before do
       allow(user).to receive(:valid_invitation?) { false }
-      allow_any_instance_of(described_class).to receive(:blocked_email?) { blocked_email }
-      described_class.new.validate(user)
+      allow_any_instance_of(described_class).to receive(:blocked_email_provider?) { blocked_email }
     end
 
-    context 'blocked_email?' do
+    subject { described_class.new.validate(user); errors }
+
+    context 'when e-mail provider is blocked' do
       let(:blocked_email) { true }
 
-      it 'calls errors.add' do
-        expect(errors).to have_received(:add).with(:email, :blocked)
+      it 'adds error' do
+        expect(subject).to have_received(:add).with(:email, :blocked)
       end
     end
 
-    context '!blocked_email?' do
+    context 'when e-mail provider is not blocked' do
       let(:blocked_email) { false }
 
-      it 'not calls errors.add' do
-        expect(errors).not_to have_received(:add).with(:email, :blocked)
+      it 'does not add errors' do
+        expect(subject).not_to have_received(:add).with(:email, :blocked)
+      end
+
+      context 'when canonical e-mail is blocked' do
+        let(:other_user) { Fabricate(:user, email: 'i.n.f.o@mail.com') }
+
+        before do
+          other_user.account.suspend!
+        end
+
+        it 'adds error' do
+          expect(subject).to have_received(:add).with(:email, :taken)
+        end
       end
     end
   end
-- 
cgit 


From a6564d56d6af0467eb8cfa96444b4503f437e0a6 Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Fri, 23 Apr 2021 22:51:21 +0200
Subject: Fix edge case where accepted follow cannot be processed because of
 follow limit (#16098)

---
 app/models/follow_request.rb       | 2 +-
 spec/models/follow_request_spec.rb | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

(limited to 'spec/models')

diff --git a/app/models/follow_request.rb b/app/models/follow_request.rb
index 59fefcdf6..0b6f7629a 100644
--- a/app/models/follow_request.rb
+++ b/app/models/follow_request.rb
@@ -29,7 +29,7 @@ class FollowRequest < ApplicationRecord
   validates :account_id, uniqueness: { scope: :target_account_id }
 
   def authorize!
-    account.follow!(target_account, reblogs: show_reblogs, notify: notify, uri: uri)
+    account.follow!(target_account, reblogs: show_reblogs, notify: notify, uri: uri, bypass_limit: true)
     MergeWorker.perform_async(target_account.id, account.id) if account.local?
     destroy!
   end
diff --git a/spec/models/follow_request_spec.rb b/spec/models/follow_request_spec.rb
index cc484a5b9..36ce8ee60 100644
--- a/spec/models/follow_request_spec.rb
+++ b/spec/models/follow_request_spec.rb
@@ -7,7 +7,7 @@ RSpec.describe FollowRequest, type: :model do
     let(:target_account) { Fabricate(:account) }
 
     it 'calls Account#follow!, MergeWorker.perform_async, and #destroy!' do
-      expect(account).to        receive(:follow!).with(target_account, reblogs: true, notify: false, uri: follow_request.uri)
+      expect(account).to        receive(:follow!).with(target_account, reblogs: true, notify: false, uri: follow_request.uri, bypass_limit: true)
       expect(MergeWorker).to    receive(:perform_async).with(target_account.id, account.id)
       expect(follow_request).to receive(:destroy!)
       follow_request.authorize!
-- 
cgit 


From 7f0c49c58ab75bbd6b4dc44d47a8c4ab57db9d51 Mon Sep 17 00:00:00 2001
From: abcang <abcang1015@gmail.com>
Date: Sun, 25 Apr 2021 13:33:28 +0900
Subject: Improve tag search query (#16104)

---
 app/models/tag.rb                                          | 14 ++++++--------
 ...10421121431_add_case_insensitive_btree_index_to_tags.rb | 13 +++++++++++++
 db/schema.rb                                               |  4 ++--
 spec/models/tag_spec.rb                                    | 14 ++++++++++++++
 4 files changed, 35 insertions(+), 10 deletions(-)
 create mode 100644 db/migrate/20210421121431_add_case_insensitive_btree_index_to_tags.rb

(limited to 'spec/models')

diff --git a/app/models/tag.rb b/app/models/tag.rb
index bb93a52e2..efffc7eee 100644
--- a/app/models/tag.rb
+++ b/app/models/tag.rb
@@ -40,7 +40,8 @@ class Tag < ApplicationRecord
   scope :trendable, -> { Setting.trendable_by_default ? where(trendable: [true, nil]) : where(trendable: true) }
   scope :discoverable, -> { listable.joins(:account_tag_stat).where(AccountTagStat.arel_table[:accounts_count].gt(0)).order(Arel.sql('account_tag_stats.accounts_count desc')) }
   scope :recently_used, ->(account) { joins(:statuses).where(statuses: { id: account.statuses.select(:id).limit(1000) }).group(:id).order(Arel.sql('count(*) desc')) }
-  scope :matches_name, ->(value) { where(arel_table[:name].matches("#{value}%")) }
+  # Search with case-sensitive to use B-tree index.
+  scope :matches_name, ->(term) { where(arel_table[:name].lower.matches(arel_table.lower("#{sanitize_sql_like(Tag.normalize(term))}%"), nil, true)) }
 
   delegate :accounts_count,
            :accounts_count=,
@@ -126,10 +127,9 @@ class Tag < ApplicationRecord
     end
 
     def search_for(term, limit = 5, offset = 0, options = {})
-      normalized_term = normalize(term.strip)
-      pattern         = sanitize_sql_like(normalized_term) + '%'
-      query           = Tag.listable.where(arel_table[:name].lower.matches(pattern))
-      query           = query.where(arel_table[:name].lower.eq(normalized_term).or(arel_table[:reviewed_at].not_eq(nil))) if options[:exclude_unreviewed]
+      striped_term = term.strip
+      query = Tag.listable.matches_name(striped_term)
+      query = query.merge(matching_name(striped_term).or(where.not(reviewed_at: nil))) if options[:exclude_unreviewed]
 
       query.order(Arel.sql('length(name) ASC, name ASC'))
            .limit(limit)
@@ -145,7 +145,7 @@ class Tag < ApplicationRecord
     end
 
     def matching_name(name_or_names)
-      names = Array(name_or_names).map { |name| normalize(name).mb_chars.downcase.to_s }
+      names = Array(name_or_names).map { |name| arel_table.lower(normalize(name)) }
 
       if names.size == 1
         where(arel_table[:name].lower.eq(names.first))
@@ -154,8 +154,6 @@ class Tag < ApplicationRecord
       end
     end
 
-    private
-
     def normalize(str)
       str.gsub(/\A#/, '')
     end
diff --git a/db/migrate/20210421121431_add_case_insensitive_btree_index_to_tags.rb b/db/migrate/20210421121431_add_case_insensitive_btree_index_to_tags.rb
new file mode 100644
index 000000000..ed359e8cd
--- /dev/null
+++ b/db/migrate/20210421121431_add_case_insensitive_btree_index_to_tags.rb
@@ -0,0 +1,13 @@
+class AddCaseInsensitiveBtreeIndexToTags < ActiveRecord::Migration[5.2]
+  disable_ddl_transaction!
+
+  def up
+    safety_assured { execute 'CREATE UNIQUE INDEX CONCURRENTLY index_tags_on_name_lower_btree ON tags (lower(name) text_pattern_ops)' }
+    remove_index :tags, name: 'index_tags_on_name_lower'
+  end
+
+  def down
+    safety_assured { execute 'CREATE UNIQUE INDEX CONCURRENTLY index_tags_on_name_lower ON tags (lower(name))' }
+    remove_index :tags, name: 'index_tags_on_name_lower_btree'
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index dcbbf4aea..8dc0661fc 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2021_04_16_200740) do
+ActiveRecord::Schema.define(version: 2021_04_21_121431) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -862,7 +862,7 @@ ActiveRecord::Schema.define(version: 2021_04_16_200740) do
     t.datetime "last_status_at"
     t.float "max_score"
     t.datetime "max_score_at"
-    t.index "lower((name)::text)", name: "index_tags_on_name_lower", unique: true
+    t.index "lower((name)::text) text_pattern_ops", name: "index_tags_on_name_lower_btree", unique: true
   end
 
   create_table "tombstones", force: :cascade do |t|
diff --git a/spec/models/tag_spec.rb b/spec/models/tag_spec.rb
index df876593c..3949dbce5 100644
--- a/spec/models/tag_spec.rb
+++ b/spec/models/tag_spec.rb
@@ -96,6 +96,20 @@ RSpec.describe Tag, type: :model do
     end
   end
 
+  describe '.matches_name' do
+    it 'returns tags for multibyte case-insensitive names' do
+      upcase_string   = 'abcABCａｂｃＡＢＣやゆよ'
+      downcase_string = 'abcabcａｂｃａｂｃやゆよ';
+
+      tag = Fabricate(:tag, name: downcase_string)
+      expect(Tag.matches_name(upcase_string)).to eq [tag]
+    end
+
+    it 'uses the LIKE operator' do
+      expect(Tag.matches_name('100%abc').to_sql).to eq %q[SELECT "tags".* FROM "tags" WHERE LOWER("tags"."name") LIKE LOWER('100\\%abc%')]
+    end
+  end
+
   describe '.matching_name' do
     it 'returns tags for multibyte case-insensitive names' do
       upcase_string   = 'abcABCａｂｃＡＢＣやゆよ'
-- 
cgit 


From 566fc909134586d1746ad60ee455832dec6bc61a Mon Sep 17 00:00:00 2001
From: Claire <claire.github-309c@sitedethib.com>
Date: Thu, 6 May 2021 14:22:54 +0200
Subject: Add Ruby 3.0 support (#16046)

* Fix issues with POSIX::Spawn, Terrapin and Ruby 3.0

Also improve the Terrapin monkey-patch for the stderr/stdout issue.

* Fix keyword argument handling throughout the codebase

* Monkey-patch Paperclip to fix keyword arguments handling in validators

* Change validation_extensions to please CodeClimate

* Bump microformats from 4.2.1 to 4.3.1

* Allow Ruby 3.0

* Add Ruby 3.0 test target to CircleCI

* Add test for admin dashboard warnings

* Fix admin dashboard warnings on Ruby 3.0
---
 .circleci/config.yml                               | 27 +++++++
 Gemfile                                            |  2 +-
 Gemfile.lock                                       |  6 +-
 app/controllers/activitypub/outboxes_controller.rb |  2 +-
 app/controllers/api/v1/accounts_controller.rb      |  4 +-
 .../api/v1/follow_requests_controller.rb           |  2 +-
 app/models/session_activation.rb                   |  2 +-
 app/models/user.rb                                 | 19 +++--
 app/views/admin/dashboard/index.html.haml          |  2 +-
 app/workers/import/relationship_worker.rb          |  6 +-
 config/application.rb                              |  1 +
 config/initializers/session_store.rb               |  5 +-
 lib/paperclip/validation_extensions.rb             | 58 +++++++++++++++
 lib/terrapin/multi_pipe_extensions.rb              | 87 +++++++++++-----------
 .../controllers/admin/dashboard_controller_spec.rb | 12 ++-
 spec/mailers/notification_mailer_spec.rb           |  4 +-
 spec/mailers/user_mailer_spec.rb                   |  4 +-
 spec/models/session_activation_spec.rb             |  6 +-
 .../account_relationships_presenter_spec.rb        |  2 +-
 19 files changed, 177 insertions(+), 74 deletions(-)
 create mode 100644 lib/paperclip/validation_extensions.rb

(limited to 'spec/models')

diff --git a/.circleci/config.yml b/.circleci/config.yml
index 862fa126b..2f3860d7c 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -129,6 +129,13 @@ jobs:
         environment: *ruby_environment
     <<: *install_ruby_dependencies
 
+  install-ruby3.0:
+    <<: *defaults
+    docker:
+      - image: circleci/ruby:3.0-buster-node
+        environment: *ruby_environment
+    <<: *install_ruby_dependencies
+
   build:
     <<: *defaults
     steps:
@@ -187,6 +194,18 @@ jobs:
       - image: circleci/redis:5-alpine
     <<: *test_steps
 
+  test-ruby3.0:
+    <<: *defaults
+    docker:
+      - image: circleci/ruby:3.0-buster-node
+        environment: *ruby_environment
+      - image: circleci/postgres:12.2
+        environment:
+          POSTGRES_USER: root
+          POSTGRES_HOST_AUTH_METHOD: trust
+      - image: circleci/redis:5-alpine
+    <<: *test_steps
+
   test-webui:
     <<: *defaults
     docker:
@@ -227,6 +246,10 @@ workflows:
           requires:
             - install
             - install-ruby2.7
+      - install-ruby3.0:
+          requires:
+            - install
+            - install-ruby2.7
       - build:
           requires:
             - install-ruby2.7
@@ -241,6 +264,10 @@ workflows:
           requires:
             - install-ruby2.6
             - build
+      - test-ruby3.0:
+          requires:
+            - install-ruby3.0
+            - build
       - test-webui:
           requires:
             - install
diff --git a/Gemfile b/Gemfile
index 6ca0a81de..5a55d6b04 100644
--- a/Gemfile
+++ b/Gemfile
@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 source 'https://rubygems.org'
-ruby '>= 2.5.0', '< 3.0.0'
+ruby '>= 2.5.0', '< 3.1.0'
 
 gem 'pkg-config', '~> 1.4'
 
diff --git a/Gemfile.lock b/Gemfile.lock
index b1ae4fd22..980750b63 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -292,7 +292,7 @@ GEM
     ipaddress (0.8.3)
     iso-639 (0.3.5)
     jmespath (1.4.0)
-    json (2.3.1)
+    json (2.5.1)
     json-canonicalization (0.2.1)
     json-ld (3.1.9)
       htmlentities (~> 4.3)
@@ -344,7 +344,7 @@ GEM
       redis (>= 3.0.5)
     memory_profiler (1.0.0)
     method_source (1.0.0)
-    microformats (4.2.1)
+    microformats (4.3.1)
       json (~> 2.2)
       nokogiri (~> 1.10)
     mime-types (3.3.1)
@@ -354,7 +354,7 @@ GEM
       nokogiri (~> 1)
       rake
     mini_mime (1.0.3)
-    mini_portile2 (2.5.0)
+    mini_portile2 (2.5.1)
     minitest (5.14.4)
     msgpack (1.4.2)
     multi_json (1.15.0)
diff --git a/app/controllers/activitypub/outboxes_controller.rb b/app/controllers/activitypub/outboxes_controller.rb
index 5fd735ad6..111285036 100644
--- a/app/controllers/activitypub/outboxes_controller.rb
+++ b/app/controllers/activitypub/outboxes_controller.rb
@@ -20,7 +20,7 @@ class ActivityPub::OutboxesController < ActivityPub::BaseController
   def outbox_presenter
     if page_requested?
       ActivityPub::CollectionPresenter.new(
-        id: outbox_url(page_params),
+        id: outbox_url(**page_params),
         type: :ordered,
         part_of: outbox_url,
         prev: prev_page,
diff --git a/app/controllers/api/v1/accounts_controller.rb b/app/controllers/api/v1/accounts_controller.rb
index 996f1b79b..95869f554 100644
--- a/app/controllers/api/v1/accounts_controller.rb
+++ b/app/controllers/api/v1/accounts_controller.rb
@@ -35,7 +35,7 @@ class Api::V1::AccountsController < Api::BaseController
     follow  = FollowService.new.call(current_user.account, @account, reblogs: params.key?(:reblogs) ? truthy_param?(:reblogs) : nil, notify: params.key?(:notify) ? truthy_param?(:notify) : nil, with_rate_limit: true)
     options = @account.locked? || current_user.account.silenced? ? {} : { following_map: { @account.id => { reblogs: follow.show_reblogs?, notify: follow.notify? } }, requested_map: { @account.id => false } }
 
-    render json: @account, serializer: REST::RelationshipSerializer, relationships: relationships(options)
+    render json: @account, serializer: REST::RelationshipSerializer, relationships: relationships(**options)
   end
 
   def block
@@ -70,7 +70,7 @@ class Api::V1::AccountsController < Api::BaseController
   end
 
   def relationships(**options)
-    AccountRelationshipsPresenter.new([@account.id], current_user.account_id, options)
+    AccountRelationshipsPresenter.new([@account.id], current_user.account_id, **options)
   end
 
   def account_params
diff --git a/app/controllers/api/v1/follow_requests_controller.rb b/app/controllers/api/v1/follow_requests_controller.rb
index b34c76f29..f4b2a74d0 100644
--- a/app/controllers/api/v1/follow_requests_controller.rb
+++ b/app/controllers/api/v1/follow_requests_controller.rb
@@ -29,7 +29,7 @@ class Api::V1::FollowRequestsController < Api::BaseController
   end
 
   def relationships(**options)
-    AccountRelationshipsPresenter.new([params[:id]], current_user.account_id, options)
+    AccountRelationshipsPresenter.new([params[:id]], current_user.account_id, **options)
   end
 
   def load_accounts
diff --git a/app/models/session_activation.rb b/app/models/session_activation.rb
index b0ce9d112..3a59bad93 100644
--- a/app/models/session_activation.rb
+++ b/app/models/session_activation.rb
@@ -44,7 +44,7 @@ class SessionActivation < ApplicationRecord
     end
 
     def activate(**options)
-      activation = create!(options)
+      activation = create!(**options)
       purge_old
       activation
     end
diff --git a/app/models/user.rb b/app/models/user.rb
index 0440627c5..4973c68b6 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -370,15 +370,20 @@ class User < ApplicationRecord
 
   protected
 
-  def send_devise_notification(notification, *args)
+  def send_devise_notification(notification, *args, **kwargs)
     # This method can be called in `after_update` and `after_commit` hooks,
     # but we must make sure the mailer is actually called *after* commit,
     # otherwise it may work on stale data. To do this, figure out if we are
     # within a transaction.
+
+    # It seems like devise sends keyword arguments as a hash in the last
+    # positional argument
+    kwargs = args.pop if args.last.is_a?(Hash) && kwargs.empty?
+
     if ActiveRecord::Base.connection.current_transaction.try(:records)&.include?(self)
-      pending_devise_notifications << [notification, args]
+      pending_devise_notifications << [notification, args, kwargs]
     else
-      render_and_send_devise_message(notification, *args)
+      render_and_send_devise_message(notification, *args, **kwargs)
     end
   end
 
@@ -389,8 +394,8 @@ class User < ApplicationRecord
   end
 
   def send_pending_devise_notifications
-    pending_devise_notifications.each do |notification, args|
-      render_and_send_devise_message(notification, *args)
+    pending_devise_notifications.each do |notification, args, kwargs|
+      render_and_send_devise_message(notification, *args, **kwargs)
     end
 
     # Empty the pending notifications array because the
@@ -403,8 +408,8 @@ class User < ApplicationRecord
     @pending_devise_notifications ||= []
   end
 
-  def render_and_send_devise_message(notification, *args)
-    devise_mailer.send(notification, self, *args).deliver_later
+  def render_and_send_devise_message(notification, *args, **kwargs)
+    devise_mailer.send(notification, self, *args, **kwargs).deliver_later
   end
 
   def set_approved
diff --git a/app/views/admin/dashboard/index.html.haml b/app/views/admin/dashboard/index.html.haml
index 57a753e6b..e8a2b46fd 100644
--- a/app/views/admin/dashboard/index.html.haml
+++ b/app/views/admin/dashboard/index.html.haml
@@ -5,7 +5,7 @@
   .flash-message-stack
     - @system_checks.each do |message|
       .flash-message.warning
-        = t("admin.system_checks.#{message.key}.message_html", message.value ? { value: content_tag(:strong, message.value) } : {})
+        = t("admin.system_checks.#{message.key}.message_html", value: message.value ? content_tag(:strong, message.value) : nil)
         - if message.action
           = link_to t("admin.system_checks.#{message.key}.action"), message.action
 
diff --git a/app/workers/import/relationship_worker.rb b/app/workers/import/relationship_worker.rb
index 4a7100435..6791b15c3 100644
--- a/app/workers/import/relationship_worker.rb
+++ b/app/workers/import/relationship_worker.rb
@@ -5,7 +5,7 @@ class Import::RelationshipWorker
 
   sidekiq_options queue: 'pull', retry: 8, dead: false
 
-  def perform(account_id, target_account_uri, relationship, options = {})
+  def perform(account_id, target_account_uri, relationship, options)
     from_account   = Account.find(account_id)
     target_domain  = domain(target_account_uri)
     target_account = stoplight_wrap_request(target_domain) { ResolveAccountService.new.call(target_account_uri, { check_delivery_availability: true }) }
@@ -16,7 +16,7 @@ class Import::RelationshipWorker
     case relationship
     when 'follow'
       begin
-        FollowService.new.call(from_account, target_account, options)
+        FollowService.new.call(from_account, target_account, **options)
       rescue ActiveRecord::RecordInvalid
         raise if FollowLimitValidator.limit_for_account(from_account) < from_account.following_count
       end
@@ -27,7 +27,7 @@ class Import::RelationshipWorker
     when 'unblock'
       UnblockService.new.call(from_account, target_account)
     when 'mute'
-      MuteService.new.call(from_account, target_account, options)
+      MuteService.new.call(from_account, target_account, **options)
     when 'unmute'
       UnmuteService.new.call(from_account, target_account)
     end
diff --git a/config/application.rb b/config/application.rb
index 37a996224..08a4e4c97 100644
--- a/config/application.rb
+++ b/config/application.rb
@@ -10,6 +10,7 @@ require_relative '../lib/exceptions'
 require_relative '../lib/enumerable'
 require_relative '../lib/sanitize_ext/sanitize_config'
 require_relative '../lib/redis/namespace_extensions'
+require_relative '../lib/paperclip/validation_extensions'
 require_relative '../lib/paperclip/url_generator_extensions'
 require_relative '../lib/paperclip/attachment_extensions'
 require_relative '../lib/paperclip/media_type_spoof_detector_extensions'
diff --git a/config/initializers/session_store.rb b/config/initializers/session_store.rb
index e5d1be4c6..3d9bf96fd 100644
--- a/config/initializers/session_store.rb
+++ b/config/initializers/session_store.rb
@@ -1,7 +1,6 @@
 # Be sure to restart your server when you modify this file.
 
-Rails.application.config.session_store :cookie_store, {
+Rails.application.config.session_store :cookie_store,
   key: '_mastodon_session',
   secure: (Rails.env.production? || ENV['LOCAL_HTTPS'] == 'true'),
-  same_site: :lax,
-}
+  same_site: :lax
diff --git a/lib/paperclip/validation_extensions.rb b/lib/paperclip/validation_extensions.rb
new file mode 100644
index 000000000..0df0434f6
--- /dev/null
+++ b/lib/paperclip/validation_extensions.rb
@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+# Monkey-patch various Paperclip validators for Ruby 3.0 compatibility
+
+module Paperclip
+  module Validators
+    module AttachmentSizeValidatorExtensions
+      def validate_each(record, attr_name, _value)
+        base_attr_name = attr_name
+        attr_name = "#{attr_name}_file_size".to_sym
+        value = record.send(:read_attribute_for_validation, attr_name)
+
+        if value.present?
+          options.slice(*Paperclip::Validators::AttachmentSizeValidator::AVAILABLE_CHECKS).each do |option, option_value|
+            option_value = option_value.call(record) if option_value.is_a?(Proc)
+            option_value = extract_option_value(option, option_value)
+
+            next if value.send(Paperclip::Validators::AttachmentSizeValidator::CHECKS[option], option_value)
+
+            error_message_key = options[:in] ? :in_between : option
+            [attr_name, base_attr_name].each do |error_attr_name|
+              record.errors.add(error_attr_name, error_message_key, **filtered_options(value).merge(
+                min: min_value_in_human_size(record),
+                max: max_value_in_human_size(record),
+                count: human_size(option_value)
+              ))
+            end
+          end
+        end
+      end
+    end
+
+    module AttachmentContentTypeValidatorExtensions
+      def mark_invalid(record, attribute, types)
+        record.errors.add attribute, :invalid, **options.merge({ types: types.join(', ') })
+      end
+    end
+
+    module AttachmentPresenceValidatorExtensions
+      def validate_each(record, attribute, _value)
+        if record.send("#{attribute}_file_name").blank?
+          record.errors.add(attribute, :blank, **options)
+        end
+      end
+    end
+
+    module AttachmentFileNameValidatorExtensions
+      def mark_invalid(record, attribute, patterns)
+        record.errors.add attribute, :invalid, options.merge({ names: patterns.join(', ') })
+      end
+    end
+  end
+end
+
+Paperclip::Validators::AttachmentSizeValidator.prepend(Paperclip::Validators::AttachmentSizeValidatorExtensions)
+Paperclip::Validators::AttachmentContentTypeValidator.prepend(Paperclip::Validators::AttachmentContentTypeValidatorExtensions)
+Paperclip::Validators::AttachmentPresenceValidator.prepend(Paperclip::Validators::AttachmentPresenceValidatorExtensions)
+Paperclip::Validators::AttachmentFileNameValidator.prepend(Paperclip::Validators::AttachmentFileNameValidatorExtensions)
diff --git a/lib/terrapin/multi_pipe_extensions.rb b/lib/terrapin/multi_pipe_extensions.rb
index 51d7de37c..209f4ad6c 100644
--- a/lib/terrapin/multi_pipe_extensions.rb
+++ b/lib/terrapin/multi_pipe_extensions.rb
@@ -1,61 +1,64 @@
 # frozen_string_literal: false
-# Fix adapted from https://github.com/thoughtbot/terrapin/pull/5
+
+require 'fcntl'
 
 module Terrapin
   module MultiPipeExtensions
-    def read
-      read_streams(@stdout_in, @stderr_in)
-    end
+    def initialize
+      @stdout_in, @stdout_out = IO.pipe
+      @stderr_in, @stderr_out = IO.pipe
 
-    def close_read
-      begin
-        @stdout_in.close
-      rescue IOError
-        # Do nothing
-      end
-
-      begin
-        @stderr_in.close
-      rescue IOError
-        # Do nothing
-      end
+      clear_nonblocking_flags!
     end
 
-    def read_streams(output, error)
-      @stdout_output = ''
-      @stderr_output = ''
+    def pipe_options
+      # Add some flags to explicitly close the other end of the pipes
+      { out: @stdout_out, err: @stderr_out, @stdout_in => :close, @stderr_in => :close }
+    end
 
-      read_fds = [output, error]
+    def read
+      # While we are patching Terrapin, fix child process potentially getting stuck on writing
+      # to stderr.
 
-      until read_fds.empty?
-        to_read, = IO.select(read_fds)
+      @stdout_output = +''
+      @stderr_output = +''
 
-        if to_read.include?(output)
-          @stdout_output << read_stream(output)
-          read_fds.delete(output) if output.closed?
-        end
+      fds_to_read = [@stdout_in, @stderr_in]
+      until fds_to_read.empty?
+        rs, = IO.select(fds_to_read)
 
-        if to_read.include?(error)
-          @stderr_output << read_stream(error)
-          read_fds.delete(error) if error.closed?
-        end
+        read_nonblocking!(@stdout_in, @stdout_output, fds_to_read) if rs.include?(@stdout_in)
+        read_nonblocking!(@stderr_in, @stderr_output, fds_to_read) if rs.include?(@stderr_in)
       end
     end
 
-    def read_stream(io)
-      result = ''
-
-      begin
-        while (partial_result = io.read_nonblock(8192))
-          result << partial_result
-        end
-      rescue EOFError, Errno::EPIPE
-        io.close
-      rescue Errno::EINTR, Errno::EWOULDBLOCK, Errno::EAGAIN
-        # Do nothing
+    private
+
+    # @param [IO] io IO Stream to read until there is nothing to read
+    # @param [String] result Mutable string to which read values will be appended to
+    # @param [Array<IO>] fds_to_read Mutable array from which `io` should be removed on EOF
+    def read_nonblocking!(io, result, fds_to_read)
+      while (partial_result = io.read_nonblock(8192))
+        result << partial_result
       end
+    rescue IO::WaitReadable
+      # Do nothing
+    rescue EOFError
+      fds_to_read.delete(io)
+    end
+
+    def clear_nonblocking_flags!
+      # Ruby 3.0 sets pipes to non-blocking mode, and resets the flags as
+      # needed when calling fork/exec-related syscalls, but posix-spawn does
+      # not currently do that, so we need to do it manually for the time being
+      # so that the child process do not error out when the buffers are full.
+      stdout_flags = @stdout_out.fcntl(Fcntl::F_GETFL)
+      @stdout_out.fcntl(Fcntl::F_SETFL, stdout_flags & ~Fcntl::O_NONBLOCK) if stdout_flags & Fcntl::O_NONBLOCK
 
-      result
+      stderr_flags = @stderr_out.fcntl(Fcntl::F_GETFL)
+      @stderr_out.fcntl(Fcntl::F_SETFL, stderr_flags & ~Fcntl::O_NONBLOCK) if stderr_flags & Fcntl::O_NONBLOCK
+    rescue NameError, NotImplementedError, Errno::EINVAL
+      # Probably on windows, where pipes are blocking by default
     end
   end
 end
diff --git a/spec/controllers/admin/dashboard_controller_spec.rb b/spec/controllers/admin/dashboard_controller_spec.rb
index 73b50e721..7824854f9 100644
--- a/spec/controllers/admin/dashboard_controller_spec.rb
+++ b/spec/controllers/admin/dashboard_controller_spec.rb
@@ -3,9 +3,19 @@
 require 'rails_helper'
 
 describe Admin::DashboardController, type: :controller do
+  render_views
+
   describe 'GET #index' do
-    it 'returns 200' do
+    before do
+      allow(Admin::SystemCheck).to receive(:perform).and_return([
+        Admin::SystemCheck::Message.new(:database_schema_check),
+        Admin::SystemCheck::Message.new(:rules_check, nil, admin_rules_path),
+        Admin::SystemCheck::Message.new(:sidekiq_process_check, 'foo, bar'),
+      ])
       sign_in Fabricate(:user, admin: true)
+    end
+
+    it 'returns 200' do
       get :index
 
       expect(response).to have_http_status(200)
diff --git a/spec/mailers/notification_mailer_spec.rb b/spec/mailers/notification_mailer_spec.rb
index 3ae106218..9b645bad8 100644
--- a/spec/mailers/notification_mailer_spec.rb
+++ b/spec/mailers/notification_mailer_spec.rb
@@ -10,12 +10,12 @@ RSpec.describe NotificationMailer, type: :mailer do
     it 'renders subject localized for the locale of the receiver' do
       locale = %i(de en).sample
       receiver.update!(locale: locale)
-      expect(mail.subject).to eq I18n.t(*args, kwrest.merge(locale: locale))
+      expect(mail.subject).to eq I18n.t(*args, **kwrest.merge(locale: locale))
     end
 
     it 'renders subject localized for the default locale if the locale of the receiver is unavailable' do
       receiver.update!(locale: nil)
-      expect(mail.subject).to eq I18n.t(*args, kwrest.merge(locale: I18n.default_locale))
+      expect(mail.subject).to eq I18n.t(*args, **kwrest.merge(locale: I18n.default_locale))
     end
   end
 
diff --git a/spec/mailers/user_mailer_spec.rb b/spec/mailers/user_mailer_spec.rb
index 6b430b505..9c866788f 100644
--- a/spec/mailers/user_mailer_spec.rb
+++ b/spec/mailers/user_mailer_spec.rb
@@ -9,12 +9,12 @@ describe UserMailer, type: :mailer do
     it 'renders subject localized for the locale of the receiver' do
       locale = I18n.available_locales.sample
       receiver.update!(locale: locale)
-      expect(mail.subject).to eq I18n.t(*args, kwrest.merge(locale: locale))
+      expect(mail.subject).to eq I18n.t(*args, **kwrest.merge(locale: locale))
     end
 
     it 'renders subject localized for the default locale if the locale of the receiver is unavailable' do
       receiver.update!(locale: nil)
-      expect(mail.subject).to eq I18n.t(*args, kwrest.merge(locale: I18n.default_locale))
+      expect(mail.subject).to eq I18n.t(*args, **kwrest.merge(locale: I18n.default_locale))
     end
   end
 
diff --git a/spec/models/session_activation_spec.rb b/spec/models/session_activation_spec.rb
index 2aa695037..450dc1399 100644
--- a/spec/models/session_activation_spec.rb
+++ b/spec/models/session_activation_spec.rb
@@ -74,13 +74,13 @@ RSpec.describe SessionActivation, type: :model do
     let(:options) { { user: Fabricate(:user), session_id: '1' } }
 
     it 'calls create! and purge_old' do
-      expect(described_class).to receive(:create!).with(options)
+      expect(described_class).to receive(:create!).with(**options)
       expect(described_class).to receive(:purge_old)
-      described_class.activate(options)
+      described_class.activate(**options)
     end
 
     it 'returns an instance of SessionActivation' do
-      expect(described_class.activate(options)).to be_kind_of SessionActivation
+      expect(described_class.activate(**options)).to be_kind_of SessionActivation
     end
   end
 
diff --git a/spec/presenters/account_relationships_presenter_spec.rb b/spec/presenters/account_relationships_presenter_spec.rb
index f8b048d38..edfbbb354 100644
--- a/spec/presenters/account_relationships_presenter_spec.rb
+++ b/spec/presenters/account_relationships_presenter_spec.rb
@@ -13,7 +13,7 @@ RSpec.describe AccountRelationshipsPresenter do
       allow(Account).to receive(:domain_blocking_map).with(account_ids, current_account_id).and_return(default_map)
     end
 
-    let(:presenter)          { AccountRelationshipsPresenter.new(account_ids, current_account_id, options) }
+    let(:presenter)          { AccountRelationshipsPresenter.new(account_ids, current_account_id, **options) }
     let(:current_account_id) { Fabricate(:account).id }
     let(:account_ids)        { [Fabricate(:account).id] }
     let(:default_map)        { { 1 => true } }
-- 
cgit 


From 74081433d0078784b7c2139f6caaa812740632b2 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Fri, 7 May 2021 14:33:43 +0200
Subject: Change trending hashtags to be affected be reblogs (#16164)

If a status with a hashtag becomes very popular, it stands to
reason that the hashtag should have a chance at trending

Fix no stats being recorded for hashtags that are not allowed
to trend, and stop ignoring bots

Remove references to hashtags in profile directory from the code
and the admin UI
---
 app/controllers/directories_controller.rb          | 10 ------
 app/lib/activitypub/activity/announce.rb           |  4 +++
 app/lib/activitypub/activity/create.rb             |  2 +-
 app/models/account.rb                              | 11 ++-----
 app/models/account_tag_stat.rb                     | 24 --------------
 app/models/tag.rb                                  | 38 +++++-----------------
 app/models/tag_filter.rb                           |  2 --
 app/models/trending_tags.rb                        | 12 ++++---
 app/services/process_hashtags_service.rb           |  3 +-
 app/services/reblog_service.rb                     | 11 +++++++
 app/views/admin/tags/_tag.html.haml                |  2 --
 app/views/admin/tags/index.html.haml               |  9 ++---
 app/views/admin/tags/show.html.haml                | 13 ++------
 config/locales/en.yml                              |  7 ++--
 config/locales/simple_form.en.yml                  |  2 +-
 config/routes.rb                                   |  2 --
 .../20210502233513_drop_account_tag_stats.rb       | 13 ++++++++
 db/schema.rb                                       | 10 ------
 spec/models/account_tag_stat_spec.rb               | 38 ----------------------
 spec/models/trending_tags_spec.rb                  |  6 ++--
 20 files changed, 59 insertions(+), 160 deletions(-)
 delete mode 100644 app/models/account_tag_stat.rb
 create mode 100644 db/post_migrate/20210502233513_drop_account_tag_stats.rb
 delete mode 100644 spec/models/account_tag_stat_spec.rb

(limited to 'spec/models')

diff --git a/app/controllers/directories_controller.rb b/app/controllers/directories_controller.rb
index f198ad5ba..f28c5b2af 100644
--- a/app/controllers/directories_controller.rb
+++ b/app/controllers/directories_controller.rb
@@ -6,7 +6,6 @@ class DirectoriesController < ApplicationController
   before_action :authenticate_user!, if: :whitelist_mode?
   before_action :require_enabled!
   before_action :set_instance_presenter
-  before_action :set_tag, only: :show
   before_action :set_accounts
 
   skip_before_action :require_functional!, unless: :whitelist_mode?
@@ -15,23 +14,14 @@ class DirectoriesController < ApplicationController
     render :index
   end
 
-  def show
-    render :index
-  end
-
   private
 
   def require_enabled!
     return not_found unless Setting.profile_directory
   end
 
-  def set_tag
-    @tag = Tag.discoverable.find_normalized!(params[:id])
-  end
-
   def set_accounts
     @accounts = Account.local.discoverable.by_recent_status.page(params[:page]).per(20).tap do |query|
-      query.merge!(Account.tagged_with(@tag.id)) if @tag
       query.merge!(Account.not_excluded_by_account(current_account)) if current_account
     end
   end
diff --git a/app/lib/activitypub/activity/announce.rb b/app/lib/activitypub/activity/announce.rb
index a1081522e..9f778ffb9 100644
--- a/app/lib/activitypub/activity/announce.rb
+++ b/app/lib/activitypub/activity/announce.rb
@@ -22,6 +22,10 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity
         visibility: visibility_from_audience
       )
 
+      original_status.tags.each do |tag|
+        tag.use!(@account)
+      end
+
       distribute(@status)
     end
 
diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index c7a655d9d..e46361c14 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -164,7 +164,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
   def attach_tags(status)
     @tags.each do |tag|
       status.tags << tag
-      TrendingTags.record_use!(tag, status.account, status.created_at) if status.public_visibility?
+      tag.use!(@account, status: status, at_time: status.created_at) if status.public_visibility?
     end
 
     @mentions.each do |mention|
diff --git a/app/models/account.rb b/app/models/account.rb
index a573365de..994459338 100644
--- a/app/models/account.rb
+++ b/app/models/account.rb
@@ -111,7 +111,6 @@ class Account < ApplicationRecord
   scope :searchable, -> { without_suspended.where(moved_to_account_id: nil) }
   scope :discoverable, -> { searchable.without_silenced.where(discoverable: true).left_outer_joins(:account_stat) }
   scope :followable_by, ->(account) { joins(arel_table.join(Follow.arel_table, Arel::Nodes::OuterJoin).on(arel_table[:id].eq(Follow.arel_table[:target_account_id]).and(Follow.arel_table[:account_id].eq(account.id))).join_sources).where(Follow.arel_table[:id].eq(nil)).joins(arel_table.join(FollowRequest.arel_table, Arel::Nodes::OuterJoin).on(arel_table[:id].eq(FollowRequest.arel_table[:target_account_id]).and(FollowRequest.arel_table[:account_id].eq(account.id))).join_sources).where(FollowRequest.arel_table[:id].eq(nil)) }
-  scope :tagged_with, ->(tag) { joins(:accounts_tags).where(accounts_tags: { tag_id: tag }) }
   scope :by_recent_status, -> { order(Arel.sql('(case when account_stats.last_status_at is null then 1 else 0 end) asc, account_stats.last_status_at desc, accounts.id desc')) }
   scope :by_recent_sign_in, -> { order(Arel.sql('(case when users.current_sign_in_at is null then 1 else 0 end) asc, users.current_sign_in_at desc, accounts.id desc')) }
   scope :popular, -> { order('account_stats.followers_count desc') }
@@ -279,19 +278,13 @@ class Account < ApplicationRecord
       if hashtags_map.key?(tag.name)
         hashtags_map.delete(tag.name)
       else
-        transaction do
-          tags.delete(tag)
-          tag.decrement_count!(:accounts_count)
-        end
+        tags.delete(tag)
       end
     end
 
     # Add hashtags that were so far missing
     hashtags_map.each_value do |tag|
-      transaction do
-        tags << tag
-        tag.increment_count!(:accounts_count)
-      end
+      tags << tag
     end
   end
 
diff --git a/app/models/account_tag_stat.rb b/app/models/account_tag_stat.rb
deleted file mode 100644
index 3c36c155a..000000000
--- a/app/models/account_tag_stat.rb
+++ /dev/null
@@ -1,24 +0,0 @@
-# frozen_string_literal: true
-# == Schema Information
-#
-# Table name: account_tag_stats
-#
-#  id             :bigint(8)        not null, primary key
-#  tag_id         :bigint(8)        not null
-#  accounts_count :bigint(8)        default(0), not null
-#  hidden         :boolean          default(FALSE), not null
-#  created_at     :datetime         not null
-#  updated_at     :datetime         not null
-#
-
-class AccountTagStat < ApplicationRecord
-  belongs_to :tag, inverse_of: :account_tag_stat
-
-  def increment_count!(key)
-    update(key => public_send(key) + 1)
-  end
-
-  def decrement_count!(key)
-    update(key => [public_send(key) - 1, 0].max)
-  end
-end
diff --git a/app/models/tag.rb b/app/models/tag.rb
index efffc7eee..735c30608 100644
--- a/app/models/tag.rb
+++ b/app/models/tag.rb
@@ -20,10 +20,8 @@
 class Tag < ApplicationRecord
   has_and_belongs_to_many :statuses
   has_and_belongs_to_many :accounts
-  has_and_belongs_to_many :sample_accounts, -> { local.discoverable.popular.limit(3) }, class_name: 'Account'
 
   has_many :featured_tags, dependent: :destroy, inverse_of: :tag
-  has_one :account_tag_stat, dependent: :destroy
 
   HASHTAG_SEPARATORS = "_\u00B7\u200c"
   HASHTAG_NAME_RE    = "([[:word:]_][[:word:]#{HASHTAG_SEPARATORS}]*[[:alpha:]#{HASHTAG_SEPARATORS}][[:word:]#{HASHTAG_SEPARATORS}]*[[:word:]_])|([[:word:]_]*[[:alpha:]][[:word:]_]*)"
@@ -38,29 +36,11 @@ class Tag < ApplicationRecord
   scope :usable, -> { where(usable: [true, nil]) }
   scope :listable, -> { where(listable: [true, nil]) }
   scope :trendable, -> { Setting.trendable_by_default ? where(trendable: [true, nil]) : where(trendable: true) }
-  scope :discoverable, -> { listable.joins(:account_tag_stat).where(AccountTagStat.arel_table[:accounts_count].gt(0)).order(Arel.sql('account_tag_stats.accounts_count desc')) }
   scope :recently_used, ->(account) { joins(:statuses).where(statuses: { id: account.statuses.select(:id).limit(1000) }).group(:id).order(Arel.sql('count(*) desc')) }
-  # Search with case-sensitive to use B-tree index.
-  scope :matches_name, ->(term) { where(arel_table[:name].lower.matches(arel_table.lower("#{sanitize_sql_like(Tag.normalize(term))}%"), nil, true)) }
-
-  delegate :accounts_count,
-           :accounts_count=,
-           :increment_count!,
-           :decrement_count!,
-           to: :account_tag_stat
-
-  after_save :save_account_tag_stat
+  scope :matches_name, ->(term) { where(arel_table[:name].lower.matches(arel_table.lower("#{sanitize_sql_like(Tag.normalize(term))}%"), nil, true)) } # Search with case-sensitive to use B-tree index
 
   update_index('tags#tag', :self)
 
-  def account_tag_stat
-    super || build_account_tag_stat
-  end
-
-  def cached_sample_accounts
-    Rails.cache.fetch("#{cache_key}/sample_accounts", expires_in: 12.hours) { sample_accounts }
-  end
-
   def to_param
     name
   end
@@ -95,6 +75,10 @@ class Tag < ApplicationRecord
     requested_review_at.present?
   end
 
+  def use!(account, status: nil, at_time: Time.now.utc)
+    TrendingTags.record_use!(self, account, status: status, at_time: at_time)
+  end
+
   def trending?
     TrendingTags.trending?(self)
   end
@@ -127,9 +111,10 @@ class Tag < ApplicationRecord
     end
 
     def search_for(term, limit = 5, offset = 0, options = {})
-      striped_term = term.strip
-      query = Tag.listable.matches_name(striped_term)
-      query = query.merge(matching_name(striped_term).or(where.not(reviewed_at: nil))) if options[:exclude_unreviewed]
+      stripped_term = term.strip
+
+      query = Tag.listable.matches_name(stripped_term)
+      query = query.merge(matching_name(stripped_term).or(where.not(reviewed_at: nil))) if options[:exclude_unreviewed]
 
       query.order(Arel.sql('length(name) ASC, name ASC'))
            .limit(limit)
@@ -161,11 +146,6 @@ class Tag < ApplicationRecord
 
   private
 
-  def save_account_tag_stat
-    return unless account_tag_stat&.changed?
-    account_tag_stat.save
-  end
-
   def validate_name_change
     errors.add(:name, I18n.t('tags.does_not_match_previous_name')) unless name_was.mb_chars.casecmp(name.mb_chars).zero?
   end
diff --git a/app/models/tag_filter.rb b/app/models/tag_filter.rb
index a9ff5b703..85bfcbea5 100644
--- a/app/models/tag_filter.rb
+++ b/app/models/tag_filter.rb
@@ -33,8 +33,6 @@ class TagFilter
 
   def scope_for(key, value)
     case key.to_s
-    when 'directory'
-      Tag.discoverable
     when 'reviewed'
       Tag.reviewed.order(reviewed_at: :desc)
     when 'unreviewed'
diff --git a/app/models/trending_tags.rb b/app/models/trending_tags.rb
index 9c2aa0ee8..31890b082 100644
--- a/app/models/trending_tags.rb
+++ b/app/models/trending_tags.rb
@@ -13,19 +13,23 @@ class TrendingTags
   class << self
     include Redisable
 
-    def record_use!(tag, account, at_time = Time.now.utc)
-      return if account.silenced? || account.bot? || !tag.usable? || !(tag.trendable? || tag.requires_review?)
+    def record_use!(tag, account, status: nil, at_time: Time.now.utc)
+      return unless tag.usable? && !account.silenced?
 
+      # Even if a tag is not allowed to trend, we still need to
+      # record the stats since they can be displayed in other places
       increment_historical_use!(tag.id, at_time)
       increment_unique_use!(tag.id, account.id, at_time)
       increment_use!(tag.id, at_time)
 
-      tag.update(last_status_at: Time.now.utc) if tag.last_status_at.nil? || tag.last_status_at < 12.hours.ago
+      # Only update when the tag was last used once every 12 hours
+      # and only if a status is given (lets use ignore reblogs)
+      tag.update(last_status_at: at_time) if status.present? && (tag.last_status_at.nil? || (tag.last_status_at < at_time && tag.last_status_at < 12.hours.ago))
     end
 
     def update!(at_time = Time.now.utc)
       tag_ids = redis.smembers("#{KEY}:used:#{at_time.beginning_of_day.to_i}") + redis.zrange(KEY, 0, -1)
-      tags    = Tag.where(id: tag_ids.uniq)
+      tags    = Tag.trendable.where(id: tag_ids.uniq)
 
       # First pass to calculate scores and update the set
 
diff --git a/app/services/process_hashtags_service.rb b/app/services/process_hashtags_service.rb
index e8e139b05..c42b79db8 100644
--- a/app/services/process_hashtags_service.rb
+++ b/app/services/process_hashtags_service.rb
@@ -8,8 +8,7 @@ class ProcessHashtagsService < BaseService
     Tag.find_or_create_by_names(tags) do |tag|
       status.tags << tag
       records << tag
-
-      TrendingTags.record_use!(tag, status.account, status.created_at) if status.public_visibility?
+      tag.use!(status.account, status: status, at_time: status.created_at) if status.public_visibility?
     end
 
     return unless status.distributable?
diff --git a/app/services/reblog_service.rb b/app/services/reblog_service.rb
index 5032397b3..744bdf567 100644
--- a/app/services/reblog_service.rb
+++ b/app/services/reblog_service.rb
@@ -35,6 +35,7 @@ class ReblogService < BaseService
 
     create_notification(reblog)
     bump_potential_friendship(account, reblog)
+    record_use(account, reblog)
 
     reblog
   end
@@ -59,6 +60,16 @@ class ReblogService < BaseService
     PotentialFriendshipTracker.record(account.id, reblog.reblog.account_id, :reblog)
   end
 
+  def record_use(account, reblog)
+    return unless reblog.public_visibility?
+
+    original_status = reblog.reblog
+
+    original_status.tags.each do |tag|
+      tag.use!(account)
+    end
+  end
+
   def build_json(reblog)
     Oj.dump(serialize_payload(ActivityPub::ActivityPresenter.from_status(reblog), ActivityPub::ActivitySerializer, signer: reblog.account))
   end
diff --git a/app/views/admin/tags/_tag.html.haml b/app/views/admin/tags/_tag.html.haml
index 287d28e53..adf4ca7b2 100644
--- a/app/views/admin/tags/_tag.html.haml
+++ b/app/views/admin/tags/_tag.html.haml
@@ -10,8 +10,6 @@
         = tag.name
 
         %small
-          = t('admin.tags.in_directory', count: tag.accounts_count)
-          &bull;
           = t('admin.tags.unique_uses_today', count: tag.history.first[:accounts])
 
           - if tag.trending?
diff --git a/app/views/admin/tags/index.html.haml b/app/views/admin/tags/index.html.haml
index d7719d45d..d78f3c6d1 100644
--- a/app/views/admin/tags/index.html.haml
+++ b/app/views/admin/tags/index.html.haml
@@ -5,12 +5,6 @@
   = javascript_pack_tag 'admin', async: true, crossorigin: 'anonymous'
 
 .filters
-  .filter-subset
-    %strong= t('admin.tags.context')
-    %ul
-      %li= filter_link_to t('generic.all'), directory: nil
-      %li= filter_link_to t('admin.tags.directory'), directory: '1'
-
   .filter-subset
     %strong= t('admin.tags.review')
     %ul
@@ -23,8 +17,9 @@
     %strong= t('generic.order_by')
     %ul
       %li= filter_link_to t('admin.tags.most_recent'), popular: nil, active: nil
-      %li= filter_link_to t('admin.tags.most_popular'), popular: '1', active: nil
       %li= filter_link_to t('admin.tags.last_active'), active: '1', popular: nil
+      %li= filter_link_to t('admin.tags.most_popular'), popular: '1', active: nil
+
 
 = form_tag admin_tags_url, method: 'GET', class: 'simple_form' do
   .fields-group
diff --git a/app/views/admin/tags/show.html.haml b/app/views/admin/tags/show.html.haml
index c9a147587..c4caffda1 100644
--- a/app/views/admin/tags/show.html.haml
+++ b/app/views/admin/tags/show.html.haml
@@ -10,15 +10,6 @@
     %div
       .dashboard__counters__num= number_with_delimiter @accounts_week
       .dashboard__counters__label= t 'admin.tags.accounts_week'
-  %div
-    - if @tag.accounts_count > 0
-      = link_to explore_hashtag_path(@tag) do
-        .dashboard__counters__num= number_with_delimiter @tag.accounts_count
-        .dashboard__counters__label= t 'admin.tags.directory'
-    - else
-      %div
-        .dashboard__counters__num= number_with_delimiter @tag.accounts_count
-        .dashboard__counters__label= t 'admin.tags.directory'
 
 %hr.spacer/
 
@@ -30,8 +21,8 @@
 
   .fields-group
     = f.input :usable, as: :boolean, wrapper: :with_label
-    = f.input :trendable, as: :boolean, wrapper: :with_label, disabled: !Setting.trends
-    = f.input :listable, as: :boolean, wrapper: :with_label, disabled: !Setting.profile_directory
+    = f.input :trendable, as: :boolean, wrapper: :with_label
+    = f.input :listable, as: :boolean, wrapper: :with_label
 
   .actions
     = f.button :button, t('generic.save_changes'), type: :submit
diff --git a/config/locales/en.yml b/config/locales/en.yml
index bfa489817..d8ad5bd84 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -698,12 +698,9 @@ en:
       accounts_today: Unique uses today
       accounts_week: Unique uses this week
       breakdown: Breakdown of today's usage by source
-      context: Context
-      directory: In directory
-      in_directory: "%{count} in directory"
-      last_active: Last active
+      last_active: Recently used
       most_popular: Most popular
-      most_recent: Most recent
+      most_recent: Recently created
       name: Hashtag
       review: Review status
       reviewed: Reviewed
diff --git a/config/locales/simple_form.en.yml b/config/locales/simple_form.en.yml
index 8ff880ebc..c4388ffc5 100644
--- a/config/locales/simple_form.en.yml
+++ b/config/locales/simple_form.en.yml
@@ -208,7 +208,7 @@ en:
       rule:
         text: Rule
       tag:
-        listable: Allow this hashtag to appear in searches and on the profile directory
+        listable: Allow this hashtag to appear in searches and suggestions
         name: Hashtag
         trendable: Allow this hashtag to appear under trends
         usable: Allow posts to use this hashtag
diff --git a/config/routes.rb b/config/routes.rb
index 8ca7fccdd..2373d8a51 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -97,8 +97,6 @@ Rails.application.routes.draw do
   post '/interact/:id', to: 'remote_interaction#create'
 
   get '/explore', to: 'directories#index', as: :explore
-  get '/explore/:id', to: 'directories#show', as: :explore_hashtag
-
   get '/settings', to: redirect('/settings/profile')
 
   namespace :settings do
diff --git a/db/post_migrate/20210502233513_drop_account_tag_stats.rb b/db/post_migrate/20210502233513_drop_account_tag_stats.rb
new file mode 100644
index 000000000..80adadcab
--- /dev/null
+++ b/db/post_migrate/20210502233513_drop_account_tag_stats.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class DropAccountTagStats < ActiveRecord::Migration[5.2]
+  disable_ddl_transaction!
+
+  def up
+    drop_table :account_tag_stats
+  end
+
+  def down
+    raise ActiveRecord::IrreversibleMigration
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 88e906079..19b1afe00 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -115,15 +115,6 @@ ActiveRecord::Schema.define(version: 2021_05_05_174616) do
     t.index ["account_id"], name: "index_account_stats_on_account_id", unique: true
   end
 
-  create_table "account_tag_stats", force: :cascade do |t|
-    t.bigint "tag_id", null: false
-    t.bigint "accounts_count", default: 0, null: false
-    t.boolean "hidden", default: false, null: false
-    t.datetime "created_at", null: false
-    t.datetime "updated_at", null: false
-    t.index ["tag_id"], name: "index_account_tag_stats_on_tag_id", unique: true
-  end
-
   create_table "account_warning_presets", force: :cascade do |t|
     t.text "text", default: "", null: false
     t.datetime "created_at", null: false
@@ -985,7 +976,6 @@ ActiveRecord::Schema.define(version: 2021_05_05_174616) do
   add_foreign_key "account_pins", "accounts", column: "target_account_id", on_delete: :cascade
   add_foreign_key "account_pins", "accounts", on_delete: :cascade
   add_foreign_key "account_stats", "accounts", on_delete: :cascade
-  add_foreign_key "account_tag_stats", "tags", on_delete: :cascade
   add_foreign_key "account_warnings", "accounts", column: "target_account_id", on_delete: :cascade
   add_foreign_key "account_warnings", "accounts", on_delete: :nullify
   add_foreign_key "accounts", "accounts", column: "moved_to_account_id", on_delete: :nullify
diff --git a/spec/models/account_tag_stat_spec.rb b/spec/models/account_tag_stat_spec.rb
deleted file mode 100644
index 6d3057f35..000000000
--- a/spec/models/account_tag_stat_spec.rb
+++ /dev/null
@@ -1,38 +0,0 @@
-# frozen_string_literal: true
-
-require 'rails_helper'
-
-RSpec.describe AccountTagStat, type: :model do
-  key = 'accounts_count'
-  let(:account_tag_stat) { Fabricate(:tag).account_tag_stat }
-
-  describe '#increment_count!' do
-    it 'calls #update' do
-      args = { key => account_tag_stat.public_send(key) + 1 }
-      expect(account_tag_stat).to receive(:update).with(args)
-      account_tag_stat.increment_count!(key)
-    end
-
-    it 'increments value by 1' do
-      expect do
-        account_tag_stat.increment_count!(key)
-      end.to change { account_tag_stat.accounts_count }.by(1)
-    end
-  end
-
-  describe '#decrement_count!' do
-    it 'calls #update' do
-      args = { key => [account_tag_stat.public_send(key) - 1, 0].max }
-      expect(account_tag_stat).to receive(:update).with(args)
-      account_tag_stat.decrement_count!(key)
-    end
-
-    it 'decrements value by 1' do
-      account_tag_stat.update(key => 1)
-
-      expect do
-        account_tag_stat.decrement_count!(key)
-      end.to change { account_tag_stat.accounts_count }.by(-1)
-    end
-  end
-end
diff --git a/spec/models/trending_tags_spec.rb b/spec/models/trending_tags_spec.rb
index b6122c994..dfbc7d6f8 100644
--- a/spec/models/trending_tags_spec.rb
+++ b/spec/models/trending_tags_spec.rb
@@ -7,9 +7,9 @@ RSpec.describe TrendingTags do
 
   describe '.update!' do
     let!(:at_time) { Time.now.utc }
-    let!(:tag1) { Fabricate(:tag, name: 'Catstodon') }
-    let!(:tag2) { Fabricate(:tag, name: 'DogsOfMastodon') }
-    let!(:tag3) { Fabricate(:tag, name: 'OCs') }
+    let!(:tag1) { Fabricate(:tag, name: 'Catstodon', trendable: true) }
+    let!(:tag2) { Fabricate(:tag, name: 'DogsOfMastodon', trendable: true) }
+    let!(:tag3) { Fabricate(:tag, name: 'OCs', trendable: true) }
 
     before do
       allow(Redis.current).to receive(:pfcount) do |key|
-- 
cgit