From e64e6a03dd1e0978fee48f0596dcfbc7fd29958f Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Fri, 28 Jun 2019 15:54:10 +0200
Subject: Add categories for custom emojis (#11196)

Fix #7940
---
 db/migrate/20190627222225_create_custom_emoji_categories.rb   | 9 +++++++++
 db/migrate/20190627222826_add_category_id_to_custom_emojis.rb | 5 +++++
 2 files changed, 14 insertions(+)
 create mode 100644 db/migrate/20190627222225_create_custom_emoji_categories.rb
 create mode 100644 db/migrate/20190627222826_add_category_id_to_custom_emojis.rb

(limited to 'db/migrate')

diff --git a/db/migrate/20190627222225_create_custom_emoji_categories.rb b/db/migrate/20190627222225_create_custom_emoji_categories.rb
new file mode 100644
index 000000000..4713793e6
--- /dev/null
+++ b/db/migrate/20190627222225_create_custom_emoji_categories.rb
@@ -0,0 +1,9 @@
+class CreateCustomEmojiCategories < ActiveRecord::Migration[5.2]
+  def change
+    create_table :custom_emoji_categories do |t|
+      t.string :name, index: { unique: true }
+
+      t.timestamps
+    end
+  end
+end
diff --git a/db/migrate/20190627222826_add_category_id_to_custom_emojis.rb b/db/migrate/20190627222826_add_category_id_to_custom_emojis.rb
new file mode 100644
index 000000000..873b4d05f
--- /dev/null
+++ b/db/migrate/20190627222826_add_category_id_to_custom_emojis.rb
@@ -0,0 +1,5 @@
+class AddCategoryIdToCustomEmojis < ActiveRecord::Migration[5.2]
+  def change
+    add_column :custom_emojis, :category_id, :bigint
+  end
+end
-- 
cgit 


From 27ad4c1501eb391b56e89bdab52624b953fde786 Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Wed, 10 Jul 2019 17:09:10 +0200
Subject: Fix old migration script depending on the StreamEntry model (#11278)

---
 db/migrate/20180528141303_fix_accounts_unique_index.rb | 5 +++++
 1 file changed, 5 insertions(+)

(limited to 'db/migrate')

diff --git a/db/migrate/20180528141303_fix_accounts_unique_index.rb b/db/migrate/20180528141303_fix_accounts_unique_index.rb
index bd4e158b7..bbbf28d81 100644
--- a/db/migrate/20180528141303_fix_accounts_unique_index.rb
+++ b/db/migrate/20180528141303_fix_accounts_unique_index.rb
@@ -12,6 +12,11 @@ class FixAccountsUniqueIndex < ActiveRecord::Migration[5.2]
     end
   end
 
+  class StreamEntry < ApplicationRecord
+    # Dummy class, to make migration possible across version changes
+    belongs_to :account, inverse_of: :stream_entries
+  end
+
   disable_ddl_transaction!
 
   def up
-- 
cgit 


From 6ff67be0f6e79ec403e08c69717ee8c89451c70e Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sat, 13 Jul 2019 16:45:50 +0200
Subject: Add a spam check (#11217)

* Add a spam check

* Use Nilsimsa to generate locality-sensitive hashes and compare using Levenshtein distance

* Add more tests

* Add exemption when the message is a reply to something that mentions the sender

* Use Nilsimsa Compare Value instead of Levenshtein distance

* Use MD5 for messages shorter than 10 characters

* Add message to automated report, do not add non-public statuses to
automated report, add trust level to accounts and make unsilencing
raise the trust level to prevent repeated spam checks on that account

* Expire spam check data after 3 months

* Add support for local statuses, reduce expiration to 1 week, always create a report

* Add content warnings to the spam check and exempt empty statuses

* Change Nilsimsa threshold to 95 and make sure removed statuses are removed from the spam check

* Add all matched statuses into automatic report
---
 Gemfile                                            |   1 +
 Gemfile.lock                                       |   8 +
 app/lib/activitypub/activity/create.rb             |  13 ++
 app/lib/spam_check.rb                              | 169 +++++++++++++++++++++
 app/models/account.rb                              |  18 ++-
 app/services/remove_status_service.rb              |   5 +
 config/locales/en.yml                              |   2 +
 .../20190701022101_add_trust_level_to_accounts.rb  |   5 +
 db/schema.rb                                       |   1 +
 spec/lib/spam_check_spec.rb                        | 160 +++++++++++++++++++
 10 files changed, 377 insertions(+), 5 deletions(-)
 create mode 100644 app/lib/spam_check.rb
 create mode 100644 db/migrate/20190701022101_add_trust_level_to_accounts.rb
 create mode 100644 spec/lib/spam_check_spec.rb

(limited to 'db/migrate')

diff --git a/Gemfile b/Gemfile
index 613515628..15334678b 100644
--- a/Gemfile
+++ b/Gemfile
@@ -58,6 +58,7 @@ gem 'idn-ruby', require: 'idn'
 gem 'kaminari', '~> 1.1'
 gem 'link_header', '~> 0.0'
 gem 'mime-types', '~> 3.2', require: 'mime/types/columnar'
+gem 'nilsimsa', git: 'https://github.com/witgo/nilsimsa', ref: 'fd184883048b922b176939f851338d0a4971a532'
 gem 'nokogiri', '~> 1.10'
 gem 'nsa', '~> 0.2'
 gem 'oj', '~> 3.7'
diff --git a/Gemfile.lock b/Gemfile.lock
index 340bbcdd8..c3198b7d9 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -12,6 +12,13 @@ GIT
   specs:
     http_parser.rb (0.6.1)
 
+GIT
+  remote: https://github.com/witgo/nilsimsa
+  revision: fd184883048b922b176939f851338d0a4971a532
+  ref: fd184883048b922b176939f851338d0a4971a532
+  specs:
+    nilsimsa (1.1.2)
+
 GEM
   remote: https://rubygems.org/
   specs:
@@ -704,6 +711,7 @@ DEPENDENCIES
   microformats (~> 4.1)
   mime-types (~> 3.2)
   net-ldap (~> 0.10)
+  nilsimsa!
   nokogiri (~> 1.10)
   nsa (~> 0.2)
   oj (~> 3.7)
diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index 5849c20d7..56c24680a 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -41,6 +41,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
 
     resolve_thread(@status)
     fetch_replies(@status)
+    check_for_spam
     distribute(@status)
     forward_for_reply if @status.distributable?
   end
@@ -406,6 +407,18 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     Account.local.where(username: local_usernames).exists?
   end
 
+  def check_for_spam
+    spam_check = SpamCheck.new(@status)
+
+    return if spam_check.skip?
+
+    if spam_check.spam?
+      spam_check.flag!
+    else
+      spam_check.remember!
+    end
+  end
+
   def forward_for_reply
     return unless @json['signature'].present? && reply_to_local?
     ActivityPub::RawDistributionWorker.perform_async(Oj.dump(@json), replied_to_status.account_id, [@account.preferred_inbox_url])
diff --git a/app/lib/spam_check.rb b/app/lib/spam_check.rb
new file mode 100644
index 000000000..923d48a02
--- /dev/null
+++ b/app/lib/spam_check.rb
@@ -0,0 +1,169 @@
+# frozen_string_literal: true
+
+class SpamCheck
+  include Redisable
+  include ActionView::Helpers::TextHelper
+
+  NILSIMSA_COMPARE_THRESHOLD = 95
+  NILSIMSA_MIN_SIZE          = 10
+  EXPIRE_SET_AFTER           = 1.week.seconds
+
+  def initialize(status)
+    @account = status.account
+    @status  = status
+  end
+
+  def skip?
+    already_flagged? || trusted? || no_unsolicited_mentions? || solicited_reply?
+  end
+
+  def spam?
+    if insufficient_data?
+      false
+    elsif nilsimsa?
+      any_other_digest?('nilsimsa') { |_, other_digest| nilsimsa_compare_value(digest, other_digest) >= NILSIMSA_COMPARE_THRESHOLD }
+    else
+      any_other_digest?('md5') { |_, other_digest| other_digest == digest }
+    end
+  end
+
+  def flag!
+    auto_silence_account!
+    auto_report_status!
+  end
+
+  def remember!
+    # The scores in sorted sets don't actually have enough bits to hold an exact
+    # value of our snowflake IDs, so we use it only for its ordering property. To
+    # get the correct status ID back, we have to save it in the string value
+
+    redis.zadd(redis_key, @status.id, digest_with_algorithm)
+    redis.zremrangebyrank(redis_key, '0', '-10')
+    redis.expire(redis_key, EXPIRE_SET_AFTER)
+  end
+
+  def reset!
+    redis.del(redis_key)
+  end
+
+  def hashable_text
+    return @hashable_text if defined?(@hashable_text)
+
+    @hashable_text = @status.text
+    @hashable_text = remove_mentions(@hashable_text)
+    @hashable_text = strip_tags(@hashable_text) unless @status.local?
+    @hashable_text = normalize_unicode(@status.spoiler_text + ' ' + @hashable_text)
+    @hashable_text = remove_whitespace(@hashable_text)
+  end
+
+  def insufficient_data?
+    hashable_text.blank?
+  end
+
+  def digest
+    @digest ||= begin
+      if nilsimsa?
+        Nilsimsa.new(hashable_text).hexdigest
+      else
+        Digest::MD5.hexdigest(hashable_text)
+      end
+    end
+  end
+
+  def digest_with_algorithm
+    if nilsimsa?
+      ['nilsimsa', digest, @status.id].join(':')
+    else
+      ['md5', digest, @status.id].join(':')
+    end
+  end
+
+  private
+
+  def remove_mentions(text)
+    return text.gsub(Account::MENTION_RE, '') if @status.local?
+
+    Nokogiri::HTML.fragment(text).tap do |html|
+      mentions = @status.mentions.map { |mention| ActivityPub::TagManager.instance.url_for(mention.account) }
+
+      html.traverse do |element|
+        element.unlink if element.name == 'a' && mentions.include?(element['href'])
+      end
+    end.to_s
+  end
+
+  def normalize_unicode(text)
+    text.unicode_normalize(:nfkc).downcase
+  end
+
+  def remove_whitespace(text)
+    text.gsub(/\s+/, ' ').strip
+  end
+
+  def auto_silence_account!
+    @account.silence!
+  end
+
+  def auto_report_status!
+    status_ids = Status.where(visibility: %i(public unlisted)).where(id: matching_status_ids).pluck(:id) + [@status.id] if @status.distributable?
+    ReportService.new.call(Account.representative, @account, status_ids: status_ids, comment: I18n.t('spam_check.spam_detected_and_silenced'))
+  end
+
+  def already_flagged?
+    @account.silenced?
+  end
+
+  def trusted?
+    @account.trust_level > Account::TRUST_LEVELS[:untrusted]
+  end
+
+  def no_unsolicited_mentions?
+    @status.mentions.all? { |mention| mention.silent? || (!@account.local? && !mention.account.local?) || mention.account.following?(@account) }
+  end
+
+  def solicited_reply?
+    !@status.thread.nil? && @status.thread.mentions.where(account: @account).exists?
+  end
+
+  def nilsimsa_compare_value(first, second)
+    first  = [first].pack('H*')
+    second = [second].pack('H*')
+    bits   = 0
+
+    0.upto(31) do |i|
+      bits += Nilsimsa::POPC[255 & (first[i].ord ^ second[i].ord)].ord
+    end
+
+    128 - bits # -128 <= Nilsimsa Compare Value <= 128
+  end
+
+  def nilsimsa?
+    hashable_text.size > NILSIMSA_MIN_SIZE
+  end
+
+  def other_digests
+    redis.zrange(redis_key, 0, -1)
+  end
+
+  def any_other_digest?(filter_algorithm)
+    other_digests.any? do |record|
+      algorithm, other_digest, status_id = record.split(':')
+
+      next unless algorithm == filter_algorithm
+
+      yield algorithm, other_digest, status_id
+    end
+  end
+
+  def matching_status_ids
+    if nilsimsa?
+      other_digests.select { |record| record.start_with?('nilsimsa') && nilsimsa_compare_value(digest, record.split(':')[1]) >= NILSIMSA_COMPARE_THRESHOLD }.map { |record| record.split(':')[2] }.compact
+    else
+      other_digests.select { |record| record.start_with?('md5') && record.split(':')[1] == digest }.map { |record| record.split(':')[2] }.compact
+    end
+  end
+
+  def redis_key
+    @redis_key ||= "spam_check:#{@account.id}"
+  end
+end
diff --git a/app/models/account.rb b/app/models/account.rb
index d6772eb98..a22b7fd7c 100644
--- a/app/models/account.rb
+++ b/app/models/account.rb
@@ -45,6 +45,7 @@
 #  also_known_as           :string           is an Array
 #  silenced_at             :datetime
 #  suspended_at            :datetime
+#  trust_level             :integer
 #
 
 class Account < ApplicationRecord
@@ -62,6 +63,11 @@ class Account < ApplicationRecord
   include AccountCounters
   include DomainNormalizable
 
+  TRUST_LEVELS = {
+    untrusted: 0,
+    trusted: 1,
+  }.freeze
+
   enum protocol: [:ostatus, :activitypub]
 
   validates :username, presence: true
@@ -163,6 +169,10 @@ class Account < ApplicationRecord
     last_webfingered_at.nil? || last_webfingered_at <= 1.day.ago
   end
 
+  def trust_level
+    self[:trust_level] || 0
+  end
+
   def refresh!
     ResolveAccountService.new.call(acct) unless local?
   end
@@ -171,21 +181,19 @@ class Account < ApplicationRecord
     silenced_at.present?
   end
 
-  def silence!(date = nil)
-    date ||= Time.now.utc
+  def silence!(date = Time.now.utc)
     update!(silenced_at: date)
   end
 
   def unsilence!
-    update!(silenced_at: nil)
+    update!(silenced_at: nil, trust_level: trust_level == TRUST_LEVELS[:untrusted] ? TRUST_LEVELS[:trusted] : trust_level)
   end
 
   def suspended?
     suspended_at.present?
   end
 
-  def suspend!(date = nil)
-    date ||= Time.now.utc
+  def suspend!(date = Time.now.utc)
     transaction do
       user&.disable! if local?
       update!(suspended_at: date)
diff --git a/app/services/remove_status_service.rb b/app/services/remove_status_service.rb
index 6311971ff..a69fce8b8 100644
--- a/app/services/remove_status_service.rb
+++ b/app/services/remove_status_service.rb
@@ -23,6 +23,7 @@ class RemoveStatusService < BaseService
         remove_from_hashtags
         remove_from_public
         remove_from_media if status.media_attachments.any?
+        remove_from_spam_check
 
         @status.destroy!
       else
@@ -142,6 +143,10 @@ class RemoveStatusService < BaseService
     redis.publish('timeline:public:local:media', @payload) if @status.local?
   end
 
+  def remove_from_spam_check
+    redis.zremrangebyscore("spam_check:#{@status.account_id}", @status.id, @status.id)
+  end
+
   def lock_options
     { redis: Redis.current, key: "distribute:#{@status.id}" }
   end
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 00b7d1dbe..89251ad40 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -875,6 +875,8 @@ en:
     profile: Profile
     relationships: Follows and followers
     two_factor_authentication: Two-factor Auth
+  spam_check:
+    spam_detected_and_silenced: This is an automated report. Spam has been detected and the sender has been silenced automatically. If this is a mistake, please unsilence the account.
   statuses:
     attached:
       description: 'Attached: %{attached}'
diff --git a/db/migrate/20190701022101_add_trust_level_to_accounts.rb b/db/migrate/20190701022101_add_trust_level_to_accounts.rb
new file mode 100644
index 000000000..917486d2e
--- /dev/null
+++ b/db/migrate/20190701022101_add_trust_level_to_accounts.rb
@@ -0,0 +1,5 @@
+class AddTrustLevelToAccounts < ActiveRecord::Migration[5.2]
+  def change
+    add_column :accounts, :trust_level, :integer
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 2e38fb1f2..c7b6b9be6 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -148,6 +148,7 @@ ActiveRecord::Schema.define(version: 2019_07_06_233204) do
     t.string "also_known_as", array: true
     t.datetime "silenced_at"
     t.datetime "suspended_at"
+    t.integer "trust_level"
     t.index "(((setweight(to_tsvector('simple'::regconfig, (display_name)::text), 'A'::\"char\") || setweight(to_tsvector('simple'::regconfig, (username)::text), 'B'::\"char\")) || setweight(to_tsvector('simple'::regconfig, (COALESCE(domain, ''::character varying))::text), 'C'::\"char\")))", name: "search_index", using: :gin
     t.index "lower((username)::text), lower((domain)::text)", name: "index_accounts_on_username_and_domain_lower", unique: true
     t.index ["moved_to_account_id"], name: "index_accounts_on_moved_to_account_id"
diff --git a/spec/lib/spam_check_spec.rb b/spec/lib/spam_check_spec.rb
new file mode 100644
index 000000000..c722dc642
--- /dev/null
+++ b/spec/lib/spam_check_spec.rb
@@ -0,0 +1,160 @@
+require 'rails_helper'
+
+RSpec.describe SpamCheck do
+  let!(:sender) { Fabricate(:account) }
+  let!(:alice) { Fabricate(:account, username: 'alice') }
+  let!(:bob) { Fabricate(:account, username: 'bob') }
+
+  def status_with_html(text, options = {})
+    status = PostStatusService.new.call(sender, { text: text }.merge(options))
+    status.update_columns(text: Formatter.instance.format(status), local: false)
+    status
+  end
+
+  describe '#hashable_text' do
+    it 'removes mentions from HTML for remote statuses' do
+      status = status_with_html('@alice Hello')
+      expect(described_class.new(status).hashable_text).to eq 'hello'
+    end
+
+    it 'removes mentions from text for local statuses' do
+      status = PostStatusService.new.call(alice, text: "Hey @#{sender.username}, how are you?")
+      expect(described_class.new(status).hashable_text).to eq 'hey , how are you?'
+    end
+  end
+
+  describe '#insufficient_data?' do
+    it 'returns true when there is no text' do
+      status = status_with_html('@alice')
+      expect(described_class.new(status).insufficient_data?).to be true
+    end
+
+    it 'returns false when there is text' do
+      status = status_with_html('@alice h')
+      expect(described_class.new(status).insufficient_data?).to be false
+    end
+  end
+
+  describe '#digest' do
+    it 'returns a string' do
+      status = status_with_html('@alice Hello world')
+      expect(described_class.new(status).digest).to be_a String
+    end
+  end
+
+  describe '#spam?' do
+    it 'returns false for a unique status' do
+      status = status_with_html('@alice Hello')
+      expect(described_class.new(status).spam?).to be false
+    end
+
+    it 'returns false for different statuses to the same recipient' do
+      status1 = status_with_html('@alice Hello')
+      described_class.new(status1).remember!
+      status2 = status_with_html('@alice Are you available to talk?')
+      expect(described_class.new(status2).spam?).to be false
+    end
+
+    it 'returns false for statuses with different content warnings' do
+      status1 = status_with_html('@alice Are you available to talk?')
+      described_class.new(status1).remember!
+      status2 = status_with_html('@alice Are you available to talk?', spoiler_text: 'This is a completely different matter than what I was talking about previously, I swear!')
+      expect(described_class.new(status2).spam?).to be false
+    end
+
+    it 'returns false for different statuses to different recipients' do
+      status1 = status_with_html('@alice How is it going?')
+      described_class.new(status1).remember!
+      status2 = status_with_html('@bob Are you okay?')
+      expect(described_class.new(status2).spam?).to be false
+    end
+
+    it 'returns false for very short different statuses to different recipients' do
+      status1 = status_with_html('@alice 🙄')
+      described_class.new(status1).remember!
+      status2 = status_with_html('@bob Huh?')
+      expect(described_class.new(status2).spam?).to be false
+    end
+
+    it 'returns false for statuses with no text' do
+      status1 = status_with_html('@alice')
+      described_class.new(status1).remember!
+      status2 = status_with_html('@bob')
+      expect(described_class.new(status2).spam?).to be false
+    end
+
+    it 'returns true for duplicate statuses to the same recipient' do
+      status1 = status_with_html('@alice Hello')
+      described_class.new(status1).remember!
+      status2 = status_with_html('@alice Hello')
+      expect(described_class.new(status2).spam?).to be true
+    end
+
+    it 'returns true for duplicate statuses to different recipients' do
+      status1 = status_with_html('@alice Hello')
+      described_class.new(status1).remember!
+      status2 = status_with_html('@bob Hello')
+      expect(described_class.new(status2).spam?).to be true
+    end
+
+    it 'returns true for nearly identical statuses with random numbers' do
+      source_text = 'Sodium, atomic number 11, was first isolated by Humphry Davy in 1807. A chemical component of salt, he named it Na in honor of the saltiest region on earth, North America.'
+      status1 = status_with_html('@alice ' + source_text + ' 1234')
+      described_class.new(status1).remember!
+      status2 = status_with_html('@bob ' + source_text + ' 9568')
+      expect(described_class.new(status2).spam?).to be true
+    end
+  end
+
+  describe '#skip?' do
+    it 'returns true when the sender is already silenced' do
+      status = status_with_html('@alice Hello')
+      sender.silence!
+      expect(described_class.new(status).skip?).to be true
+    end
+
+    it 'returns true when the mentioned person follows the sender' do
+      status = status_with_html('@alice Hello')
+      alice.follow!(sender)
+      expect(described_class.new(status).skip?).to be true
+    end
+
+    it 'returns false when even one mentioned person doesn\'t follow the sender' do
+      status = status_with_html('@alice @bob Hello')
+      alice.follow!(sender)
+      expect(described_class.new(status).skip?).to be false
+    end
+
+    it 'returns true when the sender is replying to a status that mentions the sender' do
+      parent = PostStatusService.new.call(alice, text: "Hey @#{sender.username}, how are you?")
+      status = status_with_html('@alice @bob Hello', thread: parent)
+      expect(described_class.new(status).skip?).to be true
+    end
+  end
+
+  describe '#remember!' do
+    pending
+  end
+
+  describe '#flag!' do
+    let!(:status1) { status_with_html('@alice General Kenobi you are a bold one') }
+    let!(:status2) { status_with_html('@alice @bob General Kenobi, you are a bold one') }
+
+    before do
+      described_class.new(status1).remember!
+      described_class.new(status2).flag!
+    end
+
+    it 'silences the account' do
+      expect(sender.silenced?).to be true
+    end
+
+    it 'creates a report about the account' do
+      expect(sender.targeted_reports.unresolved.count).to eq 1
+    end
+
+    it 'attaches both matching statuses to the report' do
+      expect(sender.targeted_reports.first.status_ids).to include(status1.id, status2.id)
+    end
+  end
+end
-- 
cgit 


From 730c4053d642024b9949d72c8a9f1873532c6212 Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Fri, 19 Jul 2019 01:44:42 +0200
Subject: Add ActivityPub actor representing the entire server (#11321)

* Add support for an instance actor

* Skip username validation for local Application accounts

* Add migration script to create instance actor

* Make Codeclimate happy

* Switch to id -99 for instance actor

* Remove unused `icon` and `image` attributes from instance actor

* Use if/elsif/else instead of return + ternary operator

* Add instance actor to fresh installs

* Use instance actor as instance representative

Use instance actor for forwarding reports, relay operations, and spam
auto-reporting.

* Seed database in test environment

* Fix single-user mode

* Fix tests

* Fix specs to accomodate for an extra `Account`

* Auto-reject follows on instance actor

Following an instance actor might make sense, but we are not handling that
right now, so auto-reject.

* Fix webfinger lookup and serialization for instance actor

* Rename instance actor

* Make it clear in the HTML view that the instance actor should not be blocked

* Raise cache time for instance actor as there's no dynamic content

* Re-use /about/more with a flash message for instance actor profile
---
 app/controllers/about_controller.rb                |  4 +-
 app/controllers/application_controller.rb          |  2 +-
 app/controllers/home_controller.rb                 |  2 +-
 app/controllers/instance_actors_controller.rb      | 20 ++++++++
 app/javascript/styles/mastodon/containers.scss     |  4 ++
 app/lib/activitypub/activity/follow.rb             |  2 +-
 app/lib/activitypub/tag_manager.rb                 |  5 +-
 app/lib/webfinger_resource.rb                      |  6 +++
 app/models/account.rb                              |  8 ++-
 app/models/concerns/account_finder_concern.rb      |  2 +-
 app/serializers/activitypub/actor_serializer.rb    | 14 ++++--
 app/serializers/webfinger_serializer.rb            | 25 +++++++---
 app/views/about/more.html.haml                     |  2 +
 app/views/well_known/webfinger/show.xml.ruby       | 57 ++++++++++++++--------
 config/locales/en.yml                              |  3 ++
 config/routes.rb                                   |  4 ++
 db/migrate/20190715164535_add_instance_actor.rb    |  9 ++++
 db/schema.rb                                       |  2 +-
 db/seeds.rb                                        |  4 +-
 spec/models/account_spec.rb                        | 12 ++---
 spec/services/fetch_remote_account_service_spec.rb |  1 -
 spec/services/fetch_resource_service_spec.rb       |  4 +-
 spec/spec_helper.rb                                |  1 +
 23 files changed, 141 insertions(+), 52 deletions(-)
 create mode 100644 app/controllers/instance_actors_controller.rb
 create mode 100644 db/migrate/20190715164535_add_instance_actor.rb

(limited to 'db/migrate')

diff --git a/app/controllers/about_controller.rb b/app/controllers/about_controller.rb
index 52fb1dc1b..33bac9bbc 100644
--- a/app/controllers/about_controller.rb
+++ b/app/controllers/about_controller.rb
@@ -11,7 +11,9 @@ class AboutController < ApplicationController
 
   def show; end
 
-  def more; end
+  def more
+    flash.now[:notice] = I18n.t('about.instance_actor_flash') if params[:instance_actor]
+  end
 
   def terms; end
 
diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb
index 26f3b1def..51e9764d4 100644
--- a/app/controllers/application_controller.rb
+++ b/app/controllers/application_controller.rb
@@ -91,7 +91,7 @@ class ApplicationController < ActionController::Base
   end
 
   def single_user_mode?
-    @single_user_mode ||= Rails.configuration.x.single_user_mode && Account.exists?
+    @single_user_mode ||= Rails.configuration.x.single_user_mode && Account.where('id > 0').exists?
   end
 
   def use_seamless_external_login?
diff --git a/app/controllers/home_controller.rb b/app/controllers/home_controller.rb
index d1c525134..42493cd78 100644
--- a/app/controllers/home_controller.rb
+++ b/app/controllers/home_controller.rb
@@ -58,7 +58,7 @@ class HomeController < ApplicationController
     if request.path.start_with?('/web')
       new_user_session_path
     elsif single_user_mode?
-      short_account_path(Account.local.without_suspended.first)
+      short_account_path(Account.local.without_suspended.where('id > 0').first)
     else
       about_path
     end
diff --git a/app/controllers/instance_actors_controller.rb b/app/controllers/instance_actors_controller.rb
new file mode 100644
index 000000000..41f33602e
--- /dev/null
+++ b/app/controllers/instance_actors_controller.rb
@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+class InstanceActorsController < ApplicationController
+  include AccountControllerConcern
+
+  def show
+    expires_in 10.minutes, public: true
+    render json: @account, content_type: 'application/activity+json', serializer: ActivityPub::ActorSerializer, adapter: ActivityPub::Adapter, fields: restrict_fields_to
+  end
+
+  private
+
+  def set_account
+    @account = Account.find(-99)
+  end
+
+  def restrict_fields_to
+    %i(id type preferred_username inbox public_key endpoints url manually_approves_followers)
+  end
+end
diff --git a/app/javascript/styles/mastodon/containers.scss b/app/javascript/styles/mastodon/containers.scss
index 3564bf07b..2b6794ee2 100644
--- a/app/javascript/styles/mastodon/containers.scss
+++ b/app/javascript/styles/mastodon/containers.scss
@@ -145,6 +145,10 @@
     min-height: 100%;
   }
 
+  .flash-message {
+    margin-bottom: 10px;
+  }
+
   @media screen and (max-width: 738px) {
     grid-template-columns: minmax(0, 50%) minmax(0, 50%);
 
diff --git a/app/lib/activitypub/activity/follow.rb b/app/lib/activitypub/activity/follow.rb
index 3eb88339a..28f1da19f 100644
--- a/app/lib/activitypub/activity/follow.rb
+++ b/app/lib/activitypub/activity/follow.rb
@@ -8,7 +8,7 @@ class ActivityPub::Activity::Follow < ActivityPub::Activity
 
     return if target_account.nil? || !target_account.local? || delete_arrived_first?(@json['id']) || @account.requested?(target_account)
 
-    if target_account.blocking?(@account) || target_account.domain_blocking?(@account.domain) || target_account.moved?
+    if target_account.blocking?(@account) || target_account.domain_blocking?(@account.domain) || target_account.moved? || target_account.instance_actor?
       reject_follow_request!(target_account)
       return
     end
diff --git a/app/lib/activitypub/tag_manager.rb b/app/lib/activitypub/tag_manager.rb
index 4d452f290..512272dbe 100644
--- a/app/lib/activitypub/tag_manager.rb
+++ b/app/lib/activitypub/tag_manager.rb
@@ -17,7 +17,7 @@ class ActivityPub::TagManager
 
     case target.object_type
     when :person
-      short_account_url(target)
+      target.instance_actor? ? about_more_url(instance_actor: true) : short_account_url(target)
     when :note, :comment, :activity
       return activity_account_status_url(target.account, target) if target.reblog?
       short_account_status_url(target.account, target)
@@ -29,7 +29,7 @@ class ActivityPub::TagManager
 
     case target.object_type
     when :person
-      account_url(target)
+      target.instance_actor? ? instance_actor_url : account_url(target)
     when :note, :comment, :activity
       return activity_account_status_url(target.account, target) if target.reblog?
       account_status_url(target.account, target)
@@ -119,6 +119,7 @@ class ActivityPub::TagManager
 
   def uri_to_local_id(uri, param = :id)
     path_params = Rails.application.routes.recognize_path(uri)
+    path_params[:username] = Rails.configuration.x.local_domain if path_params[:controller] == 'instance_actors'
     path_params[param]
   end
 
diff --git a/app/lib/webfinger_resource.rb b/app/lib/webfinger_resource.rb
index a54a702a2..22d78874a 100644
--- a/app/lib/webfinger_resource.rb
+++ b/app/lib/webfinger_resource.rb
@@ -23,11 +23,17 @@ class WebfingerResource
   def username_from_url
     if account_show_page?
       path_params[:username]
+    elsif instance_actor_page?
+      Rails.configuration.x.local_domain
     else
       raise ActiveRecord::RecordNotFound
     end
   end
 
+  def instance_actor_page?
+    path_params[:controller] == 'instance_actors'
+  end
+
   def account_show_page?
     path_params[:controller] == 'accounts' && path_params[:action] == 'show'
   end
diff --git a/app/models/account.rb b/app/models/account.rb
index adf4586fa..ccd116d6e 100644
--- a/app/models/account.rb
+++ b/app/models/account.rb
@@ -77,7 +77,7 @@ class Account < ApplicationRecord
   validates :username, format: { with: /\A#{USERNAME_RE}\z/i }, if: -> { !local? && will_save_change_to_username? }
 
   # Local user validations
-  validates :username, format: { with: /\A[a-z0-9_]+\z/i }, length: { maximum: 30 }, if: -> { local? && will_save_change_to_username? }
+  validates :username, format: { with: /\A[a-z0-9_]+\z/i }, length: { maximum: 30 }, if: -> { local? && will_save_change_to_username? && actor_type != 'Application' }
   validates_with UniqueUsernameValidator, if: -> { local? && will_save_change_to_username? }
   validates_with UnreservedUsernameValidator, if: -> { local? && will_save_change_to_username? }
   validates :display_name, length: { maximum: 30 }, if: -> { local? && will_save_change_to_display_name? }
@@ -139,6 +139,10 @@ class Account < ApplicationRecord
     %w(Application Service).include? actor_type
   end
 
+  def instance_actor?
+    id == -99
+  end
+
   alias bot bot?
 
   def bot=(val)
@@ -498,7 +502,7 @@ class Account < ApplicationRecord
   end
 
   def generate_keys
-    return unless local? && !Rails.env.test?
+    return unless local? && private_key.blank? && public_key.blank?
 
     keypair = OpenSSL::PKey::RSA.new(2048)
     self.private_key = keypair.to_pem
diff --git a/app/models/concerns/account_finder_concern.rb b/app/models/concerns/account_finder_concern.rb
index ccd7bfa12..a54c2174d 100644
--- a/app/models/concerns/account_finder_concern.rb
+++ b/app/models/concerns/account_finder_concern.rb
@@ -13,7 +13,7 @@ module AccountFinderConcern
     end
 
     def representative
-      find_local(Setting.site_contact_username.strip.gsub(/\A@/, '')) || Account.local.without_suspended.first
+      Account.find(-99)
     end
 
     def find_local(username)
diff --git a/app/serializers/activitypub/actor_serializer.rb b/app/serializers/activitypub/actor_serializer.rb
index 0644219fb..0bd7aed2e 100644
--- a/app/serializers/activitypub/actor_serializer.rb
+++ b/app/serializers/activitypub/actor_serializer.rb
@@ -39,11 +39,17 @@ class ActivityPub::ActorSerializer < ActivityPub::Serializer
   delegate :moved?, to: :object
 
   def id
-    account_url(object)
+    object.instance_actor? ? instance_actor_url : account_url(object)
   end
 
   def type
-    object.bot? ? 'Service' : 'Person'
+    if object.instance_actor?
+      'Application'
+    elsif object.bot?
+      'Service'
+    else
+      'Person'
+    end
   end
 
   def following
@@ -55,7 +61,7 @@ class ActivityPub::ActorSerializer < ActivityPub::Serializer
   end
 
   def inbox
-    account_inbox_url(object)
+    object.instance_actor? ? instance_actor_inbox_url : account_inbox_url(object)
   end
 
   def outbox
@@ -95,7 +101,7 @@ class ActivityPub::ActorSerializer < ActivityPub::Serializer
   end
 
   def url
-    short_account_url(object)
+    object.instance_actor? ? about_more_url(instance_actor: true) : short_account_url(object)
   end
 
   def avatar_exists?
diff --git a/app/serializers/webfinger_serializer.rb b/app/serializers/webfinger_serializer.rb
index f4af21551..008d0c182 100644
--- a/app/serializers/webfinger_serializer.rb
+++ b/app/serializers/webfinger_serializer.rb
@@ -10,15 +10,26 @@ class WebfingerSerializer < ActiveModel::Serializer
   end
 
   def aliases
-    [short_account_url(object), account_url(object)]
+    if object.instance_actor?
+      [instance_actor_url]
+    else
+      [short_account_url(object), account_url(object)]
+    end
   end
 
   def links
-    [
-      { rel: 'http://webfinger.net/rel/profile-page', type: 'text/html', href: short_account_url(object) },
-      { rel: 'http://schemas.google.com/g/2010#updates-from', type: 'application/atom+xml', href: account_url(object, format: 'atom') },
-      { rel: 'self', type: 'application/activity+json', href: account_url(object) },
-      { rel: 'http://ostatus.org/schema/1.0/subscribe', template: "#{authorize_interaction_url}?uri={uri}" },
-    ]
+    if object.instance_actor?
+      [
+        { rel: 'http://webfinger.net/rel/profile-page', type: 'text/html', href: about_more_url(instance_actor: true) },
+        { rel: 'self', type: 'application/activity+json', href: instance_actor_url },
+      ]
+    else
+      [
+        { rel: 'http://webfinger.net/rel/profile-page', type: 'text/html', href: short_account_url(object) },
+        { rel: 'http://schemas.google.com/g/2010#updates-from', type: 'application/atom+xml', href: account_url(object, format: 'atom') },
+        { rel: 'self', type: 'application/activity+json', href: account_url(object) },
+        { rel: 'http://ostatus.org/schema/1.0/subscribe', template: "#{authorize_interaction_url}?uri={uri}" },
+      ]
+    end
   end
 end
diff --git a/app/views/about/more.html.haml b/app/views/about/more.html.haml
index b248ed1d2..21431ef8e 100644
--- a/app/views/about/more.html.haml
+++ b/app/views/about/more.html.haml
@@ -43,5 +43,7 @@
           = mail_to @instance_presenter.site_contact_email, nil, title: @instance_presenter.site_contact_email
 
   .column-3
+    = render 'application/flashes'
+
     .box-widget
       .rich-formatting= @instance_presenter.site_extended_description.html_safe.presence || t('about.extended_description_html')
diff --git a/app/views/well_known/webfinger/show.xml.ruby b/app/views/well_known/webfinger/show.xml.ruby
index ae80df9d2..f5a54052a 100644
--- a/app/views/well_known/webfinger/show.xml.ruby
+++ b/app/views/well_known/webfinger/show.xml.ruby
@@ -4,30 +4,47 @@ doc << Ox::Element.new('XRD').tap do |xrd|
   xrd['xmlns'] = 'http://docs.oasis-open.org/ns/xri/xrd-1.0'
 
   xrd << (Ox::Element.new('Subject') << @account.to_webfinger_s)
-  xrd << (Ox::Element.new('Alias') << short_account_url(@account))
-  xrd << (Ox::Element.new('Alias') << account_url(@account))
 
-  xrd << Ox::Element.new('Link').tap do |link|
-    link['rel']      = 'http://webfinger.net/rel/profile-page'
-    link['type']     = 'text/html'
-    link['href']     = short_account_url(@account)
-  end
+  if @account.instance_actor?
+    xrd << (Ox::Element.new('Alias') << instance_actor_url)
 
-  xrd << Ox::Element.new('Link').tap do |link|
-    link['rel']      = 'http://schemas.google.com/g/2010#updates-from'
-    link['type']     = 'application/atom+xml'
-    link['href']     = account_url(@account, format: 'atom')
-  end
+    xrd << Ox::Element.new('Link').tap do |link|
+      link['rel']      = 'http://webfinger.net/rel/profile-page'
+      link['type']     = 'text/html'
+      link['href']     = about_more_url(instance_actor: true)
+    end
 
-  xrd << Ox::Element.new('Link').tap do |link|
-    link['rel']      = 'self'
-    link['type']     = 'application/activity+json'
-    link['href']     = account_url(@account)
-  end
+    xrd << Ox::Element.new('Link').tap do |link|
+      link['rel']      = 'self'
+      link['type']     = 'application/activity+json'
+      link['href']     = instance_actor_url
+    end
+  else
+    xrd << (Ox::Element.new('Alias') << short_account_url(@account))
+    xrd << (Ox::Element.new('Alias') << account_url(@account))
+
+    xrd << Ox::Element.new('Link').tap do |link|
+      link['rel']      = 'http://webfinger.net/rel/profile-page'
+      link['type']     = 'text/html'
+      link['href']     = short_account_url(@account)
+    end
+
+    xrd << Ox::Element.new('Link').tap do |link|
+      link['rel']      = 'http://schemas.google.com/g/2010#updates-from'
+      link['type']     = 'application/atom+xml'
+      link['href']     = account_url(@account, format: 'atom')
+    end
+
+    xrd << Ox::Element.new('Link').tap do |link|
+      link['rel']      = 'self'
+      link['type']     = 'application/activity+json'
+      link['href']     = account_url(@account)
+    end
 
-  xrd << Ox::Element.new('Link').tap do |link|
-    link['rel']      = 'http://ostatus.org/schema/1.0/subscribe'
-    link['template'] = "#{authorize_interaction_url}?acct={uri}"
+    xrd << Ox::Element.new('Link').tap do |link|
+      link['rel']      = 'http://ostatus.org/schema/1.0/subscribe'
+      link['template'] = "#{authorize_interaction_url}?acct={uri}"
+    end
   end
 end
 
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 4e252945f..89c52b84a 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -24,6 +24,9 @@ en:
     generic_description: "%{domain} is one server in the network"
     get_apps: Try a mobile app
     hosted_on: Mastodon hosted on %{domain}
+    instance_actor_flash: |
+      This account is a virtual actor used to represent the server itself and not any individual user.
+      It is used for federation purposes and should not be blocked unless you want to block the whole instance, in which case you should use a domain block.
     learn_more: Learn more
     privacy_policy: Privacy policy
     see_whats_happening: See what's happening
diff --git a/config/routes.rb b/config/routes.rb
index 95f8a39ad..27b536641 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -28,6 +28,10 @@ Rails.application.routes.draw do
   get 'intent', to: 'intents#show'
   get 'custom.css', to: 'custom_css#show', as: :custom_css
 
+  resource :instance_actor, path: 'actor', only: [:show] do
+    resource :inbox, only: [:create], module: :activitypub
+  end
+
   devise_scope :user do
     get '/invite/:invite_code', to: 'auth/registrations#new', as: :public_invite
     match '/auth/finish_signup' => 'auth/confirmations#finish_signup', via: [:get, :patch], as: :finish_signup
diff --git a/db/migrate/20190715164535_add_instance_actor.rb b/db/migrate/20190715164535_add_instance_actor.rb
new file mode 100644
index 000000000..a26d54949
--- /dev/null
+++ b/db/migrate/20190715164535_add_instance_actor.rb
@@ -0,0 +1,9 @@
+class AddInstanceActor < ActiveRecord::Migration[5.2]
+  def up
+    Account.create!(id: -99, actor_type: 'Application', locked: true, username: Rails.configuration.x.local_domain)
+  end
+
+  def down
+    Account.find_by(id: -99, actor_type: 'Application').destroy!
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index c7b6b9be6..a6a14827b 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_07_06_233204) do
+ActiveRecord::Schema.define(version: 2019_07_15_164535) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
diff --git a/db/seeds.rb b/db/seeds.rb
index 9a6e9dd78..5f43fbac8 100644
--- a/db/seeds.rb
+++ b/db/seeds.rb
@@ -1,7 +1,9 @@
 Doorkeeper::Application.create!(name: 'Web', superapp: true, redirect_uri: Doorkeeper.configuration.native_redirect_uri, scopes: 'read write follow')
 
+domain = ENV['LOCAL_DOMAIN'] || Rails.configuration.x.local_domain
+Account.create!(id: -99, actor_type: 'Application', locked: true, username: domain)
+
 if Rails.env.development?
-  domain = ENV['LOCAL_DOMAIN'] || Rails.configuration.x.local_domain
   admin  = Account.where(username: 'admin').first_or_initialize(username: 'admin')
   admin.save(validate: false)
   User.where(email: "admin@#{domain}").first_or_initialize(email: "admin@#{domain}", password: 'mastodonadmin', password_confirmation: 'mastodonadmin', confirmed_at: Time.now.utc, admin: true, account: admin, agreement: true, approved: true).save!
diff --git a/spec/models/account_spec.rb b/spec/models/account_spec.rb
index ce9ea250d..6495a6193 100644
--- a/spec/models/account_spec.rb
+++ b/spec/models/account_spec.rb
@@ -450,7 +450,7 @@ RSpec.describe Account, type: :model do
   describe '.domains' do
     it 'returns domains' do
       Fabricate(:account, domain: 'domain')
-      expect(Account.domains).to match_array(['domain'])
+      expect(Account.remote.domains).to match_array(['domain'])
     end
   end
 
@@ -665,7 +665,7 @@ RSpec.describe Account, type: :model do
           { username: 'b', domain: 'b' },
         ].map(&method(:Fabricate).curry(2).call(:account))
 
-        expect(Account.alphabetic).to eq matches
+        expect(Account.where('id > 0').alphabetic).to eq matches
       end
     end
 
@@ -732,7 +732,7 @@ RSpec.describe Account, type: :model do
         2.times { Fabricate(:account, domain: 'example.com') }
         Fabricate(:account, domain: 'example2.com')
 
-        results = Account.by_domain_accounts
+        results = Account.where('id > 0').by_domain_accounts
         expect(results.length).to eq 2
         expect(results.first.domain).to eq 'example.com'
         expect(results.first.accounts_count).to eq 2
@@ -745,7 +745,7 @@ RSpec.describe Account, type: :model do
       it 'returns an array of accounts who do not have a domain' do
         account_1 = Fabricate(:account, domain: nil)
         account_2 = Fabricate(:account, domain: 'example.com')
-        expect(Account.local).to match_array([account_1])
+        expect(Account.where('id > 0').local).to match_array([account_1])
       end
     end
 
@@ -756,14 +756,14 @@ RSpec.describe Account, type: :model do
           matches[index] = Fabricate(:account, domain: matches[index])
         end
 
-        expect(Account.partitioned).to match_array(matches)
+        expect(Account.where('id > 0').partitioned).to match_array(matches)
       end
     end
 
     describe 'recent' do
       it 'returns a relation of accounts sorted by recent creation' do
         matches = 2.times.map { Fabricate(:account) }
-        expect(Account.recent).to match_array(matches)
+        expect(Account.where('id > 0').recent).to match_array(matches)
       end
     end
 
diff --git a/spec/services/fetch_remote_account_service_spec.rb b/spec/services/fetch_remote_account_service_spec.rb
index b37445861..ee7325be2 100644
--- a/spec/services/fetch_remote_account_service_spec.rb
+++ b/spec/services/fetch_remote_account_service_spec.rb
@@ -4,7 +4,6 @@ RSpec.describe FetchRemoteAccountService, type: :service do
   let(:url) { 'https://example.com/alice' }
   let(:prefetched_body) { nil }
   let(:protocol) { :ostatus }
-  let!(:representative) { Fabricate(:account) }
 
   subject { FetchRemoteAccountService.new.call(url, prefetched_body, protocol) }
 
diff --git a/spec/services/fetch_resource_service_spec.rb b/spec/services/fetch_resource_service_spec.rb
index 98630966b..f836147d3 100644
--- a/spec/services/fetch_resource_service_spec.rb
+++ b/spec/services/fetch_resource_service_spec.rb
@@ -1,8 +1,6 @@
 require 'rails_helper'
 
 RSpec.describe FetchResourceService, type: :service do
-  let!(:representative) { Fabricate(:account) }
-
   describe '#call' do
     let(:url) { 'http://example.com' }
 
@@ -60,7 +58,7 @@ RSpec.describe FetchResourceService, type: :service do
 
       it 'signs request' do
         subject
-        expect(a_request(:get, url).with(headers: { 'Signature' => /keyId="#{Regexp.escape(ActivityPub::TagManager.instance.uri_for(representative) + '#main-key')}"/ })).to have_been_made
+        expect(a_request(:get, url).with(headers: { 'Signature' => /keyId="#{Regexp.escape(ActivityPub::TagManager.instance.uri_for(Account.representative) + '#main-key')}"/ })).to have_been_made
       end
 
       context 'when content type is application/atom+xml' do
diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb
index 0cd1f91d0..45ba1bbd9 100644
--- a/spec/spec_helper.rb
+++ b/spec/spec_helper.rb
@@ -27,6 +27,7 @@ RSpec.configure do |config|
   end
 
   config.before :suite do
+    Rails.application.load_seed
     Chewy.strategy(:bypass)
   end
 
-- 
cgit 


From cd68714393b4429defc3d1df2451c98e6784d0c1 Mon Sep 17 00:00:00 2001
From: Daigo 3 Dango <zunda@users.noreply.github.com>
Date: Mon, 22 Jul 2019 23:08:11 -1000
Subject: List columns within the method (#11377)

To avoid the exception:

NoMethodError: undefined method `perform' for nil:NilClass
.../vendor/bundle/ruby/2.6.0/gems/strong_migrations-0.4.1/lib/strong_migrations/migration.rb:14:in `method_missing'
.../vendor/bundle/ruby/2.6.0/gems/activerecord-5.2.3/lib/active_record/migration.rb:604:in `method_missing'
.../db/migrate/20170918125918_ids_to_bigints.rb:69:in `<class:IdsToBigints>'
.../db/migrate/20170918125918_ids_to_bigints.rb:3:in `<top (required)>'
---
 db/migrate/20170918125918_ids_to_bigints.rb | 130 ++++++++++++++--------------
 1 file changed, 65 insertions(+), 65 deletions(-)

(limited to 'db/migrate')

diff --git a/db/migrate/20170918125918_ids_to_bigints.rb b/db/migrate/20170918125918_ids_to_bigints.rb
index c6feed8f9..8e19468db 100644
--- a/db/migrate/20170918125918_ids_to_bigints.rb
+++ b/db/migrate/20170918125918_ids_to_bigints.rb
@@ -5,70 +5,70 @@ class IdsToBigints < ActiveRecord::Migration[5.1]
 
   disable_ddl_transaction!
 
-  INCLUDED_COLUMNS = [
-    [:account_domain_blocks, :account_id],
-    [:account_domain_blocks, :id],
-    [:accounts, :id],
-    [:blocks, :account_id],
-    [:blocks, :id],
-    [:blocks, :target_account_id],
-    [:conversation_mutes, :account_id],
-    [:conversation_mutes, :id],
-    [:domain_blocks, :id],
-    [:favourites, :account_id],
-    [:favourites, :id],
-    [:favourites, :status_id],
-    [:follow_requests, :account_id],
-    [:follow_requests, :id],
-    [:follow_requests, :target_account_id],
-    [:follows, :account_id],
-    [:follows, :id],
-    [:follows, :target_account_id],
-    [:imports, :account_id],
-    [:imports, :id],
-    [:media_attachments, :account_id],
-    [:media_attachments, :id],
-    [:mentions, :account_id],
-    [:mentions, :id],
-    [:mutes, :account_id],
-    [:mutes, :id],
-    [:mutes, :target_account_id],
-    [:notifications, :account_id],
-    [:notifications, :from_account_id],
-    [:notifications, :id],
-    [:oauth_access_grants, :application_id],
-    [:oauth_access_grants, :id],
-    [:oauth_access_grants, :resource_owner_id],
-    [:oauth_access_tokens, :application_id],
-    [:oauth_access_tokens, :id],
-    [:oauth_access_tokens, :resource_owner_id],
-    [:oauth_applications, :id],
-    [:oauth_applications, :owner_id],
-    [:reports, :account_id],
-    [:reports, :action_taken_by_account_id],
-    [:reports, :id],
-    [:reports, :target_account_id],
-    [:session_activations, :access_token_id],
-    [:session_activations, :user_id],
-    [:session_activations, :web_push_subscription_id],
-    [:settings, :id],
-    [:settings, :thing_id],
-    [:statuses, :account_id],
-    [:statuses, :application_id],
-    [:statuses, :in_reply_to_account_id],
-    [:stream_entries, :account_id],
-    [:stream_entries, :id],
-    [:subscriptions, :account_id],
-    [:subscriptions, :id],
-    [:tags, :id],
-    [:users, :account_id],
-    [:users, :id],
-    [:web_settings, :id],
-    [:web_settings, :user_id],
-  ]
-  INCLUDED_COLUMNS << [:deprecated_preview_cards, :id] if table_exists?(:deprecated_preview_cards)
-
   def migrate_columns(to_type)
+    included_columns = [
+      [:account_domain_blocks, :account_id],
+      [:account_domain_blocks, :id],
+      [:accounts, :id],
+      [:blocks, :account_id],
+      [:blocks, :id],
+      [:blocks, :target_account_id],
+      [:conversation_mutes, :account_id],
+      [:conversation_mutes, :id],
+      [:domain_blocks, :id],
+      [:favourites, :account_id],
+      [:favourites, :id],
+      [:favourites, :status_id],
+      [:follow_requests, :account_id],
+      [:follow_requests, :id],
+      [:follow_requests, :target_account_id],
+      [:follows, :account_id],
+      [:follows, :id],
+      [:follows, :target_account_id],
+      [:imports, :account_id],
+      [:imports, :id],
+      [:media_attachments, :account_id],
+      [:media_attachments, :id],
+      [:mentions, :account_id],
+      [:mentions, :id],
+      [:mutes, :account_id],
+      [:mutes, :id],
+      [:mutes, :target_account_id],
+      [:notifications, :account_id],
+      [:notifications, :from_account_id],
+      [:notifications, :id],
+      [:oauth_access_grants, :application_id],
+      [:oauth_access_grants, :id],
+      [:oauth_access_grants, :resource_owner_id],
+      [:oauth_access_tokens, :application_id],
+      [:oauth_access_tokens, :id],
+      [:oauth_access_tokens, :resource_owner_id],
+      [:oauth_applications, :id],
+      [:oauth_applications, :owner_id],
+      [:reports, :account_id],
+      [:reports, :action_taken_by_account_id],
+      [:reports, :id],
+      [:reports, :target_account_id],
+      [:session_activations, :access_token_id],
+      [:session_activations, :user_id],
+      [:session_activations, :web_push_subscription_id],
+      [:settings, :id],
+      [:settings, :thing_id],
+      [:statuses, :account_id],
+      [:statuses, :application_id],
+      [:statuses, :in_reply_to_account_id],
+      [:stream_entries, :account_id],
+      [:stream_entries, :id],
+      [:subscriptions, :account_id],
+      [:subscriptions, :id],
+      [:tags, :id],
+      [:users, :account_id],
+      [:users, :id],
+      [:web_settings, :id],
+      [:web_settings, :user_id],
+    ]
+    included_columns << [:deprecated_preview_cards, :id] if table_exists?(:deprecated_preview_cards)
+
     # Print out a warning that this will probably take a while.
     say ''
     say 'WARNING: This migration may take a *long* time for large instances'
@@ -86,7 +86,7 @@ class IdsToBigints < ActiveRecord::Migration[5.1]
       sleep 1
     end
 
-    tables = INCLUDED_COLUMNS.map(&:first).uniq
+    tables = included_columns.map(&:first).uniq
     table_sizes = {}
 
     # Sort tables by their size
@@ -94,7 +94,7 @@ class IdsToBigints < ActiveRecord::Migration[5.1]
       table_sizes[table] = estimate_rows_in_table(table)
     end
 
-    ordered_columns = INCLUDED_COLUMNS.sort_by do |col_parts|
+    ordered_columns = included_columns.sort_by do |col_parts|
       [-table_sizes[col_parts.first], col_parts.last]
     end
 
-- 
cgit 


From f371b32137ccd7e74ca29d25af2072fb79654b15 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sun, 28 Jul 2019 05:59:51 +0200
Subject: Change hashtags to preserve first-used casing (#11416)

---
 app/lib/activitypub/activity/create.rb             |  9 ++----
 app/models/tag.rb                                  | 34 +++++++++++++++++++---
 app/services/hashtag_query_service.rb              |  4 +--
 app/services/process_hashtags_service.rb           |  4 +--
 ...726175042_add_case_insensitive_index_to_tags.rb | 15 ++++++++++
 db/schema.rb                                       |  5 ++--
 6 files changed, 53 insertions(+), 18 deletions(-)
 create mode 100644 db/migrate/20190726175042_add_case_insensitive_index_to_tags.rb

(limited to 'db/migrate')

diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index 56c24680a..000b77df5 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -148,12 +148,9 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
   def process_hashtag(tag)
     return if tag['name'].blank?
 
-    hashtag = tag['name'].gsub(/\A#/, '').mb_chars.downcase
-    hashtag = Tag.where(name: hashtag).first_or_create!(name: hashtag)
-
-    return if @tags.include?(hashtag)
-
-    @tags << hashtag
+    Tag.find_or_create_by_names(tag['name']) do |hashtag|
+      @tags << hashtag unless @tags.include?(hashtag)
+    end
   rescue ActiveRecord::RecordInvalid
     nil
   end
diff --git a/app/models/tag.rb b/app/models/tag.rb
index b371d59c1..972242064 100644
--- a/app/models/tag.rb
+++ b/app/models/tag.rb
@@ -20,7 +20,7 @@ class Tag < ApplicationRecord
   HASHTAG_NAME_RE = '([[:word:]_][[:word:]_·]*[[:alpha:]_·][[:word:]_·]*[[:word:]_])|([[:word:]_]*[[:alpha:]][[:word:]_]*)'
   HASHTAG_RE = /(?:^|[^\/\)\w])#(#{HASHTAG_NAME_RE})/i
 
-  validates :name, presence: true, uniqueness: true, format: { with: /\A(#{HASHTAG_NAME_RE})\z/i }
+  validates :name, presence: true, format: { with: /\A(#{HASHTAG_NAME_RE})\z/i }
 
   scope :discoverable, -> { joins(:account_tag_stat).where(AccountTagStat.arel_table[:accounts_count].gt(0)).where(account_tag_stats: { hidden: false }).order(Arel.sql('account_tag_stats.accounts_count desc')) }
   scope :hidden, -> { where(account_tag_stats: { hidden: true }) }
@@ -64,22 +64,48 @@ class Tag < ApplicationRecord
   end
 
   class << self
+    def find_or_create_by_names(name_or_names)
+      Array(name_or_names).map(&method(:normalize)).uniq.map do |normalized_name|
+        tag = matching_name(normalized_name).first || create(name: normalized_name)
+
+        yield tag if block_given?
+
+        tag
+      end
+    end
+
     def search_for(term, limit = 5, offset = 0)
-      pattern = sanitize_sql_like(term.strip) + '%'
+      pattern = sanitize_sql_like(normalize(term.strip)) + '%'
 
-      Tag.where('lower(name) like lower(?)', pattern)
+      Tag.where(arel_table[:name].lower.matches(pattern.downcase))
          .order(:name)
          .limit(limit)
          .offset(offset)
     end
 
     def find_normalized(name)
-      find_by(name: name.mb_chars.downcase.to_s)
+      matching_name(name).first
     end
 
     def find_normalized!(name)
       find_normalized(name) || raise(ActiveRecord::RecordNotFound)
     end
+
+    def matching_name(name_or_names)
+      names = Array(name_or_names).map { |name| normalize(name).downcase }
+
+      if names.size == 1
+        where(arel_table[:name].lower.eq(names.first))
+      else
+        where(arel_table[:name].lower.in(names))
+      end
+    end
+
+    private
+
+    def normalize(str)
+      str.gsub(/\A#/, '').mb_chars.to_s
+    end
   end
 
   private
diff --git a/app/services/hashtag_query_service.rb b/app/services/hashtag_query_service.rb
index 5773d78c6..282821710 100644
--- a/app/services/hashtag_query_service.rb
+++ b/app/services/hashtag_query_service.rb
@@ -14,7 +14,7 @@ class HashtagQueryService < BaseService
 
   private
 
-  def tags_for(tags)
-    Tag.where(name: tags.map(&:downcase)) if tags.presence
+  def tags_for(names)
+    Tag.matching_name(names) if names.presence
   end
 end
diff --git a/app/services/process_hashtags_service.rb b/app/services/process_hashtags_service.rb
index b6974e598..e8e139b05 100644
--- a/app/services/process_hashtags_service.rb
+++ b/app/services/process_hashtags_service.rb
@@ -5,9 +5,7 @@ class ProcessHashtagsService < BaseService
     tags    = Extractor.extract_hashtags(status.text) if status.local?
     records = []
 
-    tags.map { |str| str.mb_chars.downcase }.uniq(&:to_s).each do |name|
-      tag = Tag.where(name: name).first_or_create(name: name)
-
+    Tag.find_or_create_by_names(tags) do |tag|
       status.tags << tag
       records << tag
 
diff --git a/db/migrate/20190726175042_add_case_insensitive_index_to_tags.rb b/db/migrate/20190726175042_add_case_insensitive_index_to_tags.rb
new file mode 100644
index 000000000..6fa8c0ec4
--- /dev/null
+++ b/db/migrate/20190726175042_add_case_insensitive_index_to_tags.rb
@@ -0,0 +1,15 @@
+class AddCaseInsensitiveIndexToTags < ActiveRecord::Migration[5.2]
+  disable_ddl_transaction!
+
+  def up
+    safety_assured { execute 'CREATE UNIQUE INDEX CONCURRENTLY index_tags_on_name_lower ON tags (lower(name))' }
+    remove_index :tags, name: 'index_tags_on_name'
+    remove_index :tags, name: 'hashtag_search_index'
+  end
+
+  def down
+    add_index :tags, :name, unique: true, algorithm: :concurrently
+    safety_assured { execute 'CREATE INDEX CONCURRENTLY hashtag_search_index ON tags (name text_pattern_ops)' }
+    remove_index :tags, name: 'index_tags_on_name_lower'
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 6319dd932..1847305c7 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_07_15_164535) do
+ActiveRecord::Schema.define(version: 2019_07_26_175042) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -652,8 +652,7 @@ ActiveRecord::Schema.define(version: 2019_07_15_164535) do
     t.string "name", default: "", null: false
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
-    t.index "lower((name)::text) text_pattern_ops", name: "hashtag_search_index"
-    t.index ["name"], name: "index_tags_on_name", unique: true
+    t.index "lower((name)::text)", name: "index_tags_on_name_lower", unique: true
   end
 
   create_table "tombstones", force: :cascade do |t|
-- 
cgit 


From e136112ab76a37cbde5c1bcfeb562c8e0a5b5116 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Mon, 29 Jul 2019 20:40:21 +0200
Subject: Fix tag normalization and migration not removing duplicate tags
 (#11441)

Fix #11428
---
 app/models/tag.rb                                  |  8 ++---
 ...726175042_add_case_insensitive_index_to_tags.rb | 13 +++++++++
 spec/models/tag_spec.rb                            | 34 ++++++++++++++++++++++
 3 files changed, 51 insertions(+), 4 deletions(-)

(limited to 'db/migrate')

diff --git a/app/models/tag.rb b/app/models/tag.rb
index 972242064..46e3a3ec0 100644
--- a/app/models/tag.rb
+++ b/app/models/tag.rb
@@ -65,7 +65,7 @@ class Tag < ApplicationRecord
 
   class << self
     def find_or_create_by_names(name_or_names)
-      Array(name_or_names).map(&method(:normalize)).uniq.map do |normalized_name|
+      Array(name_or_names).map(&method(:normalize)).uniq { |str| str.mb_chars.downcase.to_s }.map do |normalized_name|
         tag = matching_name(normalized_name).first || create(name: normalized_name)
 
         yield tag if block_given?
@@ -77,7 +77,7 @@ class Tag < ApplicationRecord
     def search_for(term, limit = 5, offset = 0)
       pattern = sanitize_sql_like(normalize(term.strip)) + '%'
 
-      Tag.where(arel_table[:name].lower.matches(pattern.downcase))
+      Tag.where(arel_table[:name].lower.matches(pattern.mb_chars.downcase.to_s))
          .order(:name)
          .limit(limit)
          .offset(offset)
@@ -92,7 +92,7 @@ class Tag < ApplicationRecord
     end
 
     def matching_name(name_or_names)
-      names = Array(name_or_names).map { |name| normalize(name).downcase }
+      names = Array(name_or_names).map { |name| normalize(name).mb_chars.downcase.to_s }
 
       if names.size == 1
         where(arel_table[:name].lower.eq(names.first))
@@ -104,7 +104,7 @@ class Tag < ApplicationRecord
     private
 
     def normalize(str)
-      str.gsub(/\A#/, '').mb_chars.to_s
+      str.gsub(/\A#/, '')
     end
   end
 
diff --git a/db/migrate/20190726175042_add_case_insensitive_index_to_tags.rb b/db/migrate/20190726175042_add_case_insensitive_index_to_tags.rb
index 6fa8c0ec4..057fc86ba 100644
--- a/db/migrate/20190726175042_add_case_insensitive_index_to_tags.rb
+++ b/db/migrate/20190726175042_add_case_insensitive_index_to_tags.rb
@@ -2,6 +2,19 @@ class AddCaseInsensitiveIndexToTags < ActiveRecord::Migration[5.2]
   disable_ddl_transaction!
 
   def up
+    Tag.connection.select_all('SELECT string_agg(id::text, \',\') AS ids FROM tags GROUP BY lower(name) HAVING count(*) > 1').to_hash.each do |row|
+      canonical_tag_id  = row['ids'].split(',').first
+      redundant_tag_ids = row['ids'].split(',')[1..-1]
+
+      safety_assured do
+        execute "UPDATE accounts_tags AS t0 SET tag_id = #{canonical_tag_id} WHERE tag_id IN (#{redundant_tag_ids.join(', ')}) AND NOT EXISTS (SELECT t1.tag_id FROM accounts_tags AS t1 WHERE t1.tag_id = #{canonical_tag_id} AND t1.account_id = t0.account_id)"
+        execute "UPDATE statuses_tags AS t0 SET tag_id = #{canonical_tag_id} WHERE tag_id IN (#{redundant_tag_ids.join(', ')}) AND NOT EXISTS (SELECT t1.tag_id FROM statuses_tags AS t1 WHERE t1.tag_id = #{canonical_tag_id} AND t1.status_id = t0.status_id)"
+        execute "UPDATE featured_tags AS t0 SET tag_id = #{canonical_tag_id} WHERE tag_id IN (#{redundant_tag_ids.join(', ')})  AND NOT EXISTS (SELECT t1.tag_id FROM featured_tags AS t1 WHERE t1.tag_id = #{canonical_tag_id} AND t1.account_id = t0.account_id)"
+      end
+
+      Tag.where(id: redundant_tag_ids).in_batches.delete_all
+    end
+
     safety_assured { execute 'CREATE UNIQUE INDEX CONCURRENTLY index_tags_on_name_lower ON tags (lower(name))' }
     remove_index :tags, name: 'index_tags_on_name'
     remove_index :tags, name: 'hashtag_search_index'
diff --git a/spec/models/tag_spec.rb b/spec/models/tag_spec.rb
index 9a30ceaa5..5f07fd618 100644
--- a/spec/models/tag_spec.rb
+++ b/spec/models/tag_spec.rb
@@ -82,6 +82,40 @@ RSpec.describe Tag, type: :model do
     end
   end
 
+  describe '.find_normalized' do
+    it 'returns tag for a multibyte case-insensitive name' do
+      upcase_string   = 'abcABCａｂｃＡＢＣやゆよ'
+      downcase_string = 'abcabcａｂｃａｂｃやゆよ';
+
+      tag = Fabricate(:tag, name: downcase_string)
+      expect(Tag.find_normalized(upcase_string)).to eq tag
+    end
+  end
+
+  describe '.matching_name' do
+    it 'returns tags for multibyte case-insensitive names' do
+      upcase_string   = 'abcABCａｂｃＡＢＣやゆよ'
+      downcase_string = 'abcabcａｂｃａｂｃやゆよ';
+
+      tag = Fabricate(:tag, name: downcase_string)
+      expect(Tag.matching_name(upcase_string)).to eq [tag]
+    end
+  end
+
+  describe '.find_or_create_by_names' do
+    it 'runs a passed block once per tag regardless of duplicates' do
+      upcase_string   = 'abcABCａｂｃＡＢＣやゆよ'
+      downcase_string = 'abcabcａｂｃａｂｃやゆよ';
+      count           = 0
+
+      Tag.find_or_create_by_names([upcase_string, downcase_string]) do |tag|
+        count += 1
+      end
+
+      expect(count).to eq 1
+    end
+  end
+
   describe '.search_for' do
     it 'finds tag records with matching names' do
       tag = Fabricate(:tag, name: "match")
-- 
cgit 


From 24552b5160a5090e7d6056fb69a209aa48fe4fce Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Tue, 30 Jul 2019 11:10:46 +0200
Subject: Add whitelist mode (#11291)

---
 app/controllers/about_controller.rb                |  5 +++
 app/controllers/activitypub/base_controller.rb     |  2 ++
 app/controllers/activitypub/inboxes_controller.rb  |  2 +-
 app/controllers/admin/domain_allows_controller.rb  | 40 ++++++++++++++++++++++
 app/controllers/admin/instances_controller.rb      | 28 +++++++++++++--
 app/controllers/api/base_controller.rb             |  9 +++++
 app/controllers/api/v1/accounts_controller.rb      |  2 ++
 app/controllers/api/v1/apps_controller.rb          |  2 ++
 .../api/v1/instances/activity_controller.rb        |  3 +-
 .../api/v1/instances/peers_controller.rb           |  3 +-
 app/controllers/api/v1/instances_controller.rb     |  1 +
 app/controllers/application_controller.rb          |  4 ++-
 app/controllers/concerns/account_owned_concern.rb  |  1 +
 app/controllers/directories_controller.rb          |  5 +--
 app/controllers/home_controller.rb                 |  2 +-
 app/controllers/media_controller.rb                |  1 +
 app/controllers/media_proxy_controller.rb          |  2 ++
 app/controllers/public_timelines_controller.rb     |  5 +--
 app/controllers/remote_interaction_controller.rb   |  1 +
 app/controllers/tags_controller.rb                 |  1 +
 app/helpers/domain_control_helper.rb               | 10 +++++-
 app/models/domain_allow.rb                         | 33 ++++++++++++++++++
 app/models/instance.rb                             |  3 +-
 app/models/instance_filter.rb                      |  4 +++
 app/policies/domain_allow_policy.rb                | 11 ++++++
 app/services/concerns/payloadable.rb               |  2 +-
 app/services/unallow_domain_service.rb             | 11 ++++++
 app/views/admin/domain_allows/new.html.haml        | 14 ++++++++
 app/views/admin/instances/index.html.haml          | 35 ++++++++++++-------
 app/views/admin/instances/show.html.haml           |  4 ++-
 app/views/admin/settings/edit.html.haml            | 28 ++++++++-------
 app/views/auth/registrations/new.html.haml         |  2 +-
 app/views/layouts/public.html.haml                 |  9 +++--
 config/initializers/2_whitelist_mode.rb            |  5 +++
 config/locales/en.yml                              |  7 ++++
 config/locales/simple_form.en.yml                  |  2 ++
 config/navigation.rb                               |  2 +-
 config/routes.rb                                   |  1 +
 db/migrate/20190705002136_create_domain_allows.rb  |  9 +++++
 db/schema.rb                                       |  9 ++++-
 lib/mastodon/domains_cli.rb                        | 22 ++++++++++--
 spec/fabricators/domain_allow_fabricator.rb        |  3 ++
 spec/models/domain_allow_spec.rb                   |  5 +++
 streaming/index.js                                 |  5 +--
 44 files changed, 302 insertions(+), 53 deletions(-)
 create mode 100644 app/controllers/admin/domain_allows_controller.rb
 create mode 100644 app/models/domain_allow.rb
 create mode 100644 app/policies/domain_allow_policy.rb
 create mode 100644 app/services/unallow_domain_service.rb
 create mode 100644 app/views/admin/domain_allows/new.html.haml
 create mode 100644 config/initializers/2_whitelist_mode.rb
 create mode 100644 db/migrate/20190705002136_create_domain_allows.rb
 create mode 100644 spec/fabricators/domain_allow_fabricator.rb
 create mode 100644 spec/models/domain_allow_spec.rb

(limited to 'db/migrate')

diff --git a/app/controllers/about_controller.rb b/app/controllers/about_controller.rb
index 31cf17710..d276e8fe5 100644
--- a/app/controllers/about_controller.rb
+++ b/app/controllers/about_controller.rb
@@ -3,6 +3,7 @@
 class AboutController < ApplicationController
   layout 'public'
 
+  before_action :require_open_federation!, only: [:show, :more]
   before_action :set_body_classes, only: :show
   before_action :set_instance_presenter
   before_action :set_expires_in
@@ -19,6 +20,10 @@ class AboutController < ApplicationController
 
   private
 
+  def require_open_federation!
+    not_found if whitelist_mode?
+  end
+
   def new_user
     User.new.tap do |user|
       user.build_account
diff --git a/app/controllers/activitypub/base_controller.rb b/app/controllers/activitypub/base_controller.rb
index a3b5c4dfa..0c2591e97 100644
--- a/app/controllers/activitypub/base_controller.rb
+++ b/app/controllers/activitypub/base_controller.rb
@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 class ActivityPub::BaseController < Api::BaseController
+  skip_before_action :require_authenticated_user!
+
   private
 
   def set_cache_headers
diff --git a/app/controllers/activitypub/inboxes_controller.rb b/app/controllers/activitypub/inboxes_controller.rb
index 7cfd9a25e..bcfc1e6d4 100644
--- a/app/controllers/activitypub/inboxes_controller.rb
+++ b/app/controllers/activitypub/inboxes_controller.rb
@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-class ActivityPub::InboxesController < Api::BaseController
+class ActivityPub::InboxesController < ActivityPub::BaseController
   include SignatureVerification
   include JsonLdHelper
   include AccountOwnedConcern
diff --git a/app/controllers/admin/domain_allows_controller.rb b/app/controllers/admin/domain_allows_controller.rb
new file mode 100644
index 000000000..31be1978b
--- /dev/null
+++ b/app/controllers/admin/domain_allows_controller.rb
@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+class Admin::DomainAllowsController < Admin::BaseController
+  before_action :set_domain_allow, only: [:destroy]
+
+  def new
+    authorize :domain_allow, :create?
+
+    @domain_allow = DomainAllow.new(domain: params[:_domain])
+  end
+
+  def create
+    authorize :domain_allow, :create?
+
+    @domain_allow = DomainAllow.new(resource_params)
+
+    if @domain_allow.save
+      log_action :create, @domain_allow
+      redirect_to admin_instances_path, notice: I18n.t('admin.domain_allows.created_msg')
+    else
+      render :new
+    end
+  end
+
+  def destroy
+    authorize @domain_allow, :destroy?
+    UnallowDomainService.new.call(@domain_allow)
+    redirect_to admin_instances_path, notice: I18n.t('admin.domain_allows.destroyed_msg')
+  end
+
+  private
+
+  def set_domain_allow
+    @domain_allow = DomainAllow.find(params[:id])
+  end
+
+  def resource_params
+    params.require(:domain_allow).permit(:domain)
+  end
+end
diff --git a/app/controllers/admin/instances_controller.rb b/app/controllers/admin/instances_controller.rb
index 7888e844f..d4f201807 100644
--- a/app/controllers/admin/instances_controller.rb
+++ b/app/controllers/admin/instances_controller.rb
@@ -2,6 +2,10 @@
 
 module Admin
   class InstancesController < BaseController
+    before_action :set_domain_block, only: :show
+    before_action :set_domain_allow, only: :show
+    before_action :set_instance, only: :show
+
     def index
       authorize :instance, :index?
 
@@ -11,20 +15,38 @@ module Admin
     def show
       authorize :instance, :show?
 
-      @instance        = Instance.new(Account.by_domain_accounts.find_by(domain: params[:id]) || DomainBlock.find_by!(domain: params[:id]))
       @following_count = Follow.where(account: Account.where(domain: params[:id])).count
       @followers_count = Follow.where(target_account: Account.where(domain: params[:id])).count
       @reports_count   = Report.where(target_account: Account.where(domain: params[:id])).count
       @blocks_count    = Block.where(target_account: Account.where(domain: params[:id])).count
       @available       = DeliveryFailureTracker.available?(Account.select(:shared_inbox_url).where(domain: params[:id]).first&.shared_inbox_url)
       @media_storage   = MediaAttachment.where(account: Account.where(domain: params[:id])).sum(:file_file_size)
-      @domain_block    = DomainBlock.rule_for(params[:id])
     end
 
     private
 
+    def set_domain_block
+      @domain_block = DomainBlock.rule_for(params[:id])
+    end
+
+    def set_domain_allow
+      @domain_allow = DomainAllow.rule_for(params[:id])
+    end
+
+    def set_instance
+      resource   = Account.by_domain_accounts.find_by(domain: params[:id])
+      resource ||= @domain_block
+      resource ||= @domain_allow
+
+      if resource
+        @instance = Instance.new(resource)
+      else
+        not_found
+      end
+    end
+
     def filtered_instances
-      InstanceFilter.new(filter_params).results
+      InstanceFilter.new(whitelist_mode? ? { allowed: true } : filter_params).results
     end
 
     def paginated_instances
diff --git a/app/controllers/api/base_controller.rb b/app/controllers/api/base_controller.rb
index 6f33a1ea9..109e38ffa 100644
--- a/app/controllers/api/base_controller.rb
+++ b/app/controllers/api/base_controller.rb
@@ -9,6 +9,7 @@ class Api::BaseController < ApplicationController
   skip_before_action :store_current_location
   skip_before_action :require_functional!
 
+  before_action :require_authenticated_user!, if: :disallow_unauthenticated_api_access?
   before_action :set_cache_headers
 
   protect_from_forgery with: :null_session
@@ -69,6 +70,10 @@ class Api::BaseController < ApplicationController
     nil
   end
 
+  def require_authenticated_user!
+    render json: { error: 'This API requires an authenticated user' }, status: 401 unless current_user
+  end
+
   def require_user!
     if !current_user
       render json: { error: 'This method requires an authenticated user' }, status: 422
@@ -94,4 +99,8 @@ class Api::BaseController < ApplicationController
   def set_cache_headers
     response.headers['Cache-Control'] = 'no-cache, no-store, max-age=0, must-revalidate'
   end
+
+  def disallow_unauthenticated_api_access?
+    authorized_fetch_mode?
+  end
 end
diff --git a/app/controllers/api/v1/accounts_controller.rb b/app/controllers/api/v1/accounts_controller.rb
index b0c62778e..b306e8e8c 100644
--- a/app/controllers/api/v1/accounts_controller.rb
+++ b/app/controllers/api/v1/accounts_controller.rb
@@ -12,6 +12,8 @@ class Api::V1::AccountsController < Api::BaseController
   before_action :check_account_suspension, only: [:show]
   before_action :check_enabled_registrations, only: [:create]
 
+  skip_before_action :require_authenticated_user!, only: :create
+
   respond_to :json
 
   def show
diff --git a/app/controllers/api/v1/apps_controller.rb b/app/controllers/api/v1/apps_controller.rb
index e9f7a7291..97177547a 100644
--- a/app/controllers/api/v1/apps_controller.rb
+++ b/app/controllers/api/v1/apps_controller.rb
@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 class Api::V1::AppsController < Api::BaseController
+  skip_before_action :require_authenticated_user!
+
   def create
     @app = Doorkeeper::Application.create!(application_options)
     render json: @app, serializer: REST::ApplicationSerializer
diff --git a/app/controllers/api/v1/instances/activity_controller.rb b/app/controllers/api/v1/instances/activity_controller.rb
index d0080c5c2..4fb5a69d8 100644
--- a/app/controllers/api/v1/instances/activity_controller.rb
+++ b/app/controllers/api/v1/instances/activity_controller.rb
@@ -2,6 +2,7 @@
 
 class Api::V1::Instances::ActivityController < Api::BaseController
   before_action :require_enabled_api!
+
   skip_before_action :set_cache_headers
 
   respond_to :json
@@ -33,6 +34,6 @@ class Api::V1::Instances::ActivityController < Api::BaseController
   end
 
   def require_enabled_api!
-    head 404 unless Setting.activity_api_enabled
+    head 404 unless Setting.activity_api_enabled && !whitelist_mode?
   end
 end
diff --git a/app/controllers/api/v1/instances/peers_controller.rb b/app/controllers/api/v1/instances/peers_controller.rb
index 450e6502f..75c3cb4ba 100644
--- a/app/controllers/api/v1/instances/peers_controller.rb
+++ b/app/controllers/api/v1/instances/peers_controller.rb
@@ -2,6 +2,7 @@
 
 class Api::V1::Instances::PeersController < Api::BaseController
   before_action :require_enabled_api!
+
   skip_before_action :set_cache_headers
 
   respond_to :json
@@ -14,6 +15,6 @@ class Api::V1::Instances::PeersController < Api::BaseController
   private
 
   def require_enabled_api!
-    head 404 unless Setting.peers_api_enabled
+    head 404 unless Setting.peers_api_enabled && !whitelist_mode?
   end
 end
diff --git a/app/controllers/api/v1/instances_controller.rb b/app/controllers/api/v1/instances_controller.rb
index 93e4f0003..8d8231423 100644
--- a/app/controllers/api/v1/instances_controller.rb
+++ b/app/controllers/api/v1/instances_controller.rb
@@ -2,6 +2,7 @@
 
 class Api::V1::InstancesController < Api::BaseController
   respond_to :json
+
   skip_before_action :set_cache_headers
 
   def show
diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb
index 41ce1a0ca..0d3913ee0 100644
--- a/app/controllers/application_controller.rb
+++ b/app/controllers/application_controller.rb
@@ -11,12 +11,14 @@ class ApplicationController < ActionController::Base
   include UserTrackingConcern
   include SessionTrackingConcern
   include CacheConcern
+  include DomainControlHelper
 
   helper_method :current_account
   helper_method :current_session
   helper_method :current_theme
   helper_method :single_user_mode?
   helper_method :use_seamless_external_login?
+  helper_method :whitelist_mode?
 
   rescue_from ActionController::RoutingError, with: :not_found
   rescue_from ActiveRecord::RecordNotFound, with: :not_found
@@ -38,7 +40,7 @@ class ApplicationController < ActionController::Base
   end
 
   def authorized_fetch_mode?
-    ENV['AUTHORIZED_FETCH'] == 'true'
+    ENV['AUTHORIZED_FETCH'] == 'true' || Rails.configuration.x.whitelist_mode
   end
 
   def public_fetch_mode?
diff --git a/app/controllers/concerns/account_owned_concern.rb b/app/controllers/concerns/account_owned_concern.rb
index 99c240fe9..460f71f65 100644
--- a/app/controllers/concerns/account_owned_concern.rb
+++ b/app/controllers/concerns/account_owned_concern.rb
@@ -4,6 +4,7 @@ module AccountOwnedConcern
   extend ActiveSupport::Concern
 
   included do
+    before_action :authenticate_user!, if: -> { whitelist_mode? && request.format != :json }
     before_action :set_account, if: :account_required?
     before_action :check_account_approval, if: :account_required?
     before_action :check_account_suspension, if: :account_required?
diff --git a/app/controllers/directories_controller.rb b/app/controllers/directories_controller.rb
index 594907674..d2ef76f06 100644
--- a/app/controllers/directories_controller.rb
+++ b/app/controllers/directories_controller.rb
@@ -3,7 +3,8 @@
 class DirectoriesController < ApplicationController
   layout 'public'
 
-  before_action :check_enabled
+  before_action :authenticate_user!, if: :whitelist_mode?
+  before_action :require_enabled!
   before_action :set_instance_presenter
   before_action :set_tag, only: :show
   before_action :set_tags
@@ -19,7 +20,7 @@ class DirectoriesController < ApplicationController
 
   private
 
-  def check_enabled
+  def require_enabled!
     return not_found unless Setting.profile_directory
   end
 
diff --git a/app/controllers/home_controller.rb b/app/controllers/home_controller.rb
index 42493cd78..22d507e77 100644
--- a/app/controllers/home_controller.rb
+++ b/app/controllers/home_controller.rb
@@ -55,7 +55,7 @@ class HomeController < ApplicationController
   end
 
   def default_redirect_path
-    if request.path.start_with?('/web')
+    if request.path.start_with?('/web') || whitelist_mode?
       new_user_session_path
     elsif single_user_mode?
       short_account_path(Account.local.without_suspended.where('id > 0').first)
diff --git a/app/controllers/media_controller.rb b/app/controllers/media_controller.rb
index b3b7519a1..1f693de32 100644
--- a/app/controllers/media_controller.rb
+++ b/app/controllers/media_controller.rb
@@ -5,6 +5,7 @@ class MediaController < ApplicationController
 
   skip_before_action :store_current_location
 
+  before_action :authenticate_user!, if: :whitelist_mode?
   before_action :set_media_attachment
   before_action :verify_permitted_status!
   before_action :check_playable, only: :player
diff --git a/app/controllers/media_proxy_controller.rb b/app/controllers/media_proxy_controller.rb
index 8fc18dd06..8da6c6fe0 100644
--- a/app/controllers/media_proxy_controller.rb
+++ b/app/controllers/media_proxy_controller.rb
@@ -5,6 +5,8 @@ class MediaProxyController < ApplicationController
 
   skip_before_action :store_current_location
 
+  before_action :authenticate_user!, if: :whitelist_mode?
+
   def show
     RedisLock.acquire(lock_options) do |lock|
       if lock.acquired?
diff --git a/app/controllers/public_timelines_controller.rb b/app/controllers/public_timelines_controller.rb
index 23506b990..324bdc508 100644
--- a/app/controllers/public_timelines_controller.rb
+++ b/app/controllers/public_timelines_controller.rb
@@ -3,7 +3,8 @@
 class PublicTimelinesController < ApplicationController
   layout 'public'
 
-  before_action :check_enabled
+  before_action :authenticate_user!, if: :whitelist_mode?
+  before_action :require_enabled!
   before_action :set_body_classes
   before_action :set_instance_presenter
 
@@ -16,7 +17,7 @@ class PublicTimelinesController < ApplicationController
 
   private
 
-  def check_enabled
+  def require_enabled!
     not_found unless Setting.timeline_preview
   end
 
diff --git a/app/controllers/remote_interaction_controller.rb b/app/controllers/remote_interaction_controller.rb
index cc6993c52..fa742fb0a 100644
--- a/app/controllers/remote_interaction_controller.rb
+++ b/app/controllers/remote_interaction_controller.rb
@@ -5,6 +5,7 @@ class RemoteInteractionController < ApplicationController
 
   layout 'modal'
 
+  before_action :authenticate_user!, if: :whitelist_mode?
   before_action :set_interaction_type
   before_action :set_status
   before_action :set_body_classes
diff --git a/app/controllers/tags_controller.rb b/app/controllers/tags_controller.rb
index d08e5a61a..3cd2d9e20 100644
--- a/app/controllers/tags_controller.rb
+++ b/app/controllers/tags_controller.rb
@@ -8,6 +8,7 @@ class TagsController < ApplicationController
   layout 'public'
 
   before_action :require_signature!, if: -> { request.format == :json && authorized_fetch_mode? }
+  before_action :authenticate_user!, if: :whitelist_mode?
   before_action :set_tag
   before_action :set_body_classes
   before_action :set_instance_presenter
diff --git a/app/helpers/domain_control_helper.rb b/app/helpers/domain_control_helper.rb
index efd328f81..067b2c2cd 100644
--- a/app/helpers/domain_control_helper.rb
+++ b/app/helpers/domain_control_helper.rb
@@ -12,6 +12,14 @@ module DomainControlHelper
       end
     end
 
-    DomainBlock.blocked?(domain)
+    if whitelist_mode?
+      !DomainAllow.allowed?(domain)
+    else
+      DomainBlock.blocked?(domain)
+    end
+  end
+
+  def whitelist_mode?
+    Rails.configuration.x.whitelist_mode
   end
 end
diff --git a/app/models/domain_allow.rb b/app/models/domain_allow.rb
new file mode 100644
index 000000000..85018b636
--- /dev/null
+++ b/app/models/domain_allow.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+# == Schema Information
+#
+# Table name: domain_allows
+#
+#  id         :bigint(8)        not null, primary key
+#  domain     :string           default(""), not null
+#  created_at :datetime         not null
+#  updated_at :datetime         not null
+#
+
+class DomainAllow < ApplicationRecord
+  include DomainNormalizable
+
+  validates :domain, presence: true, uniqueness: true
+
+  scope :matches_domain, ->(value) { where(arel_table[:domain].matches("%#{value}%")) }
+
+  class << self
+    def allowed?(domain)
+      !rule_for(domain).nil?
+    end
+
+    def rule_for(domain)
+      return if domain.blank?
+
+      uri = Addressable::URI.new.tap { |u| u.host = domain.gsub(/[\/]/, '') }
+
+      find_by(domain: uri.normalized_host)
+    end
+  end
+end
diff --git a/app/models/instance.rb b/app/models/instance.rb
index 797a191e0..3c740f8a2 100644
--- a/app/models/instance.rb
+++ b/app/models/instance.rb
@@ -7,8 +7,9 @@ class Instance
 
   def initialize(resource)
     @domain         = resource.domain
-    @accounts_count = resource.is_a?(DomainBlock) ? nil : resource.accounts_count
+    @accounts_count = resource.respond_to?(:accounts_count) ? resource.accounts_count : nil
     @domain_block   = resource.is_a?(DomainBlock) ? resource : DomainBlock.rule_for(domain)
+    @domain_allow   = resource.is_a?(DomainAllow) ? resource : DomainAllow.rule_for(domain)
   end
 
   def countable?
diff --git a/app/models/instance_filter.rb b/app/models/instance_filter.rb
index 848fff53e..8bfab826d 100644
--- a/app/models/instance_filter.rb
+++ b/app/models/instance_filter.rb
@@ -12,6 +12,10 @@ class InstanceFilter
       scope = DomainBlock
       scope = scope.matches_domain(params[:by_domain]) if params[:by_domain].present?
       scope.order(id: :desc)
+    elsif params[:allowed].present?
+      scope = DomainAllow
+      scope = scope.matches_domain(params[:by_domain]) if params[:by_domain].present?
+      scope.order(id: :desc)
     else
       scope = Account.remote
       scope = scope.matches_domain(params[:by_domain]) if params[:by_domain].present?
diff --git a/app/policies/domain_allow_policy.rb b/app/policies/domain_allow_policy.rb
new file mode 100644
index 000000000..5030453bb
--- /dev/null
+++ b/app/policies/domain_allow_policy.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class DomainAllowPolicy < ApplicationPolicy
+  def create?
+    admin?
+  end
+
+  def destroy?
+    admin?
+  end
+end
diff --git a/app/services/concerns/payloadable.rb b/app/services/concerns/payloadable.rb
index 953740faa..7f9f21c4b 100644
--- a/app/services/concerns/payloadable.rb
+++ b/app/services/concerns/payloadable.rb
@@ -14,6 +14,6 @@ module Payloadable
   end
 
   def signing_enabled?
-    ENV['AUTHORIZED_FETCH'] != 'true'
+    ENV['AUTHORIZED_FETCH'] != 'true' && !Rails.configuration.x.whitelist_mode
   end
 end
diff --git a/app/services/unallow_domain_service.rb b/app/services/unallow_domain_service.rb
new file mode 100644
index 000000000..d4387c1a1
--- /dev/null
+++ b/app/services/unallow_domain_service.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class UnallowDomainService < BaseService
+  def call(domain_allow)
+    Account.where(domain: domain_allow.domain).find_each do |account|
+      SuspendAccountService.new.call(account, destroy: true)
+    end
+
+    domain_allow.destroy
+  end
+end
diff --git a/app/views/admin/domain_allows/new.html.haml b/app/views/admin/domain_allows/new.html.haml
new file mode 100644
index 000000000..52599857a
--- /dev/null
+++ b/app/views/admin/domain_allows/new.html.haml
@@ -0,0 +1,14 @@
+- content_for :header_tags do
+  = javascript_pack_tag 'admin', integrity: true, async: true, crossorigin: 'anonymous'
+
+- content_for :page_title do
+  = t('admin.domain_allows.add_new')
+
+= simple_form_for @domain_allow, url: admin_domain_allows_path do |f|
+  = render 'shared/error_messages', object: @domain_allow
+
+  .fields-group
+    = f.input :domain, wrapper: :with_label, label: t('admin.domain_blocks.domain'), required: true
+
+  .actions
+    = f.button :button, t('admin.domain_allows.add_new'), type: :submit
diff --git a/app/views/admin/instances/index.html.haml b/app/views/admin/instances/index.html.haml
index 61e578409..982dc5035 100644
--- a/app/views/admin/instances/index.html.haml
+++ b/app/views/admin/instances/index.html.haml
@@ -6,24 +6,30 @@
     %strong= t('admin.instances.moderation.title')
     %ul
       %li= filter_link_to t('admin.instances.moderation.all'), limited: nil
-      %li= filter_link_to t('admin.instances.moderation.limited'), limited: '1'
+
+      - unless whitelist_mode?
+        %li= filter_link_to t('admin.instances.moderation.limited'), limited: '1'
 
   %div{ style: 'flex: 1 1 auto; text-align: right' }
-    = link_to t('admin.domain_blocks.add_new'), new_admin_domain_block_path, class: 'button'
+    - if whitelist_mode?
+      = link_to t('admin.domain_allows.add_new'), new_admin_domain_allow_path, class: 'button'
+    - else
+      = link_to t('admin.domain_blocks.add_new'), new_admin_domain_block_path, class: 'button'
 
-= form_tag admin_instances_url, method: 'GET', class: 'simple_form' do
-  .fields-group
-    - Admin::FilterHelper::INSTANCES_FILTERS.each do |key|
-      - if params[key].present?
-        = hidden_field_tag key, params[key]
+- unless whitelist_mode?
+  = form_tag admin_instances_url, method: 'GET', class: 'simple_form' do
+    .fields-group
+      - Admin::FilterHelper::INSTANCES_FILTERS.each do |key|
+        - if params[key].present?
+          = hidden_field_tag key, params[key]
 
-    - %i(by_domain).each do |key|
-      .input.string.optional
-        = text_field_tag key, params[key], class: 'string optional', placeholder: I18n.t("admin.instances.#{key}")
+      - %i(by_domain).each do |key|
+        .input.string.optional
+          = text_field_tag key, params[key], class: 'string optional', placeholder: I18n.t("admin.instances.#{key}")
 
-    .actions
-      %button= t('admin.accounts.search')
-      = link_to t('admin.accounts.reset'), admin_instances_path, class: 'button negative'
+      .actions
+        %button= t('admin.accounts.search')
+        = link_to t('admin.accounts.reset'), admin_instances_path, class: 'button negative'
 
 %hr.spacer/
 
@@ -47,8 +53,11 @@
               - unless first_item
                 &bull;
               = t('admin.domain_blocks.rejecting_reports')
+          - elsif whitelist_mode?
+            = t('admin.accounts.whitelisted')
           - else
             = t('admin.accounts.no_limits_imposed')
       - if instance.countable?
         .trends__item__current{ title: t('admin.instances.known_accounts', count: instance.accounts_count) }= number_to_human instance.accounts_count, strip_insignificant_zeros: true
+
 = paginate paginated_instances
diff --git a/app/views/admin/instances/show.html.haml b/app/views/admin/instances/show.html.haml
index c7992a490..fbb49ba02 100644
--- a/app/views/admin/instances/show.html.haml
+++ b/app/views/admin/instances/show.html.haml
@@ -38,7 +38,9 @@
     = link_to t('admin.accounts.title'), admin_accounts_path(remote: '1', by_domain: @instance.domain), class: 'button'
 
   %div{ style: 'float: right' }
-    - if @domain_block
+    - if @domain_allow
+      = link_to t('admin.domain_allows.undo'), admin_domain_allow_path(@domain_allow), class: 'button button--destructive', data: { confirm: t('admin.accounts.are_you_sure'), method: :delete }
+    - elsif @domain_block
       = link_to t('admin.domain_blocks.undo'), admin_domain_block_path(@domain_block), class: 'button'
     - else
       = link_to t('admin.domain_blocks.add_new'), new_admin_domain_block_path(_domain: @instance.domain), class: 'button'
diff --git a/app/views/admin/settings/edit.html.haml b/app/views/admin/settings/edit.html.haml
index b3bf3849c..1e2ed3f77 100644
--- a/app/views/admin/settings/edit.html.haml
+++ b/app/views/admin/settings/edit.html.haml
@@ -42,11 +42,12 @@
 
   %hr.spacer/
 
-  .fields-group
-    = f.input :timeline_preview, as: :boolean, wrapper: :with_label, label: t('admin.settings.timeline_preview.title'), hint: t('admin.settings.timeline_preview.desc_html')
+  - unless whitelist_mode?
+    .fields-group
+      = f.input :timeline_preview, as: :boolean, wrapper: :with_label, label: t('admin.settings.timeline_preview.title'), hint: t('admin.settings.timeline_preview.desc_html')
 
-  .fields-group
-    = f.input :show_known_fediverse_at_about_page, as: :boolean, wrapper: :with_label, label: t('admin.settings.show_known_fediverse_at_about_page.title'), hint: t('admin.settings.show_known_fediverse_at_about_page.desc_html')
+    .fields-group
+      = f.input :show_known_fediverse_at_about_page, as: :boolean, wrapper: :with_label, label: t('admin.settings.show_known_fediverse_at_about_page.title'), hint: t('admin.settings.show_known_fediverse_at_about_page.desc_html')
 
   .fields-group
     = f.input :show_staff_badge, as: :boolean, wrapper: :with_label, label: t('admin.settings.show_staff_badge.title'), hint: t('admin.settings.show_staff_badge.desc_html')
@@ -54,17 +55,18 @@
   .fields-group
     = f.input :open_deletion, as: :boolean, wrapper: :with_label, label: t('admin.settings.registrations.deletion.title'), hint: t('admin.settings.registrations.deletion.desc_html')
 
-  .fields-group
-    = f.input :activity_api_enabled, as: :boolean, wrapper: :with_label, label: t('admin.settings.activity_api_enabled.title'), hint: t('admin.settings.activity_api_enabled.desc_html')
+  - unless whitelist_mode?
+    .fields-group
+      = f.input :activity_api_enabled, as: :boolean, wrapper: :with_label, label: t('admin.settings.activity_api_enabled.title'), hint: t('admin.settings.activity_api_enabled.desc_html')
 
-  .fields-group
-    = f.input :peers_api_enabled, as: :boolean, wrapper: :with_label, label: t('admin.settings.peers_api_enabled.title'), hint: t('admin.settings.peers_api_enabled.desc_html')
+    .fields-group
+      = f.input :peers_api_enabled, as: :boolean, wrapper: :with_label, label: t('admin.settings.peers_api_enabled.title'), hint: t('admin.settings.peers_api_enabled.desc_html')
 
-  .fields-group
-    = f.input :preview_sensitive_media, as: :boolean, wrapper: :with_label, label: t('admin.settings.preview_sensitive_media.title'), hint: t('admin.settings.preview_sensitive_media.desc_html')
+    .fields-group
+      = f.input :preview_sensitive_media, as: :boolean, wrapper: :with_label, label: t('admin.settings.preview_sensitive_media.title'), hint: t('admin.settings.preview_sensitive_media.desc_html')
 
-  .fields-group
-    = f.input :profile_directory, as: :boolean, wrapper: :with_label, label: t('admin.settings.profile_directory.title'), hint: t('admin.settings.profile_directory.desc_html')
+    .fields-group
+      = f.input :profile_directory, as: :boolean, wrapper: :with_label, label: t('admin.settings.profile_directory.title'), hint: t('admin.settings.profile_directory.desc_html')
 
   .fields-group
     = f.input :spam_check_enabled, as: :boolean, wrapper: :with_label, label: t('admin.settings.spam_check_enabled.title'), hint: t('admin.settings.spam_check_enabled.desc_html')
@@ -76,7 +78,7 @@
 
   .fields-group
     = f.input :closed_registrations_message, as: :text, wrapper: :with_block_label, label: t('admin.settings.registrations.closed_message.title'), hint: t('admin.settings.registrations.closed_message.desc_html'), input_html: { rows: 8 }
-    = f.input :site_extended_description, wrapper: :with_block_label, as: :text, label: t('admin.settings.site_description_extended.title'), hint: t('admin.settings.site_description_extended.desc_html'), input_html: { rows: 8 }
+    = f.input :site_extended_description, wrapper: :with_block_label, as: :text, label: t('admin.settings.site_description_extended.title'), hint: t('admin.settings.site_description_extended.desc_html'), input_html: { rows: 8 } unless whitelist_mode?
     = f.input :site_terms, wrapper: :with_block_label, as: :text, label: t('admin.settings.site_terms.title'), hint: t('admin.settings.site_terms.desc_html'), input_html: { rows: 8 }
     = f.input :custom_css, wrapper: :with_block_label, as: :text, input_html: { rows: 8 }, label: t('admin.settings.custom_css.title'), hint: t('admin.settings.custom_css.desc_html')
 
diff --git a/app/views/auth/registrations/new.html.haml b/app/views/auth/registrations/new.html.haml
index b4a7cced5..83384d737 100644
--- a/app/views/auth/registrations/new.html.haml
+++ b/app/views/auth/registrations/new.html.haml
@@ -33,7 +33,7 @@
   = f.input :invite_code, as: :hidden
 
   .fields-group
-    = f.input :agreement, as: :boolean, wrapper: :with_label, label: t('auth.checkbox_agreement_html', rules_path: about_more_path, terms_path: terms_path)
+    = f.input :agreement, as: :boolean, wrapper: :with_label, label: whitelist_mode? ? t('auth.checkbox_agreement_without_rules_html', terms_path: terms_path) : t('auth.checkbox_agreement_html', rules_path: about_more_path, terms_path: terms_path)
 
   .actions
     = f.button :button, @invite.present? ? t('auth.register') : sign_up_message, type: :submit
diff --git a/app/views/layouts/public.html.haml b/app/views/layouts/public.html.haml
index 2929ac599..69738a2f7 100644
--- a/app/views/layouts/public.html.haml
+++ b/app/views/layouts/public.html.haml
@@ -10,10 +10,13 @@
             = link_to root_url, class: 'brand' do
               = svg_logo_full
 
-            = link_to t('directories.directory'), explore_path, class: 'nav-link optional' if Setting.profile_directory
-            = link_to t('about.about_this'), about_more_path, class: 'nav-link optional'
-            = link_to t('about.apps'), 'https://joinmastodon.org/apps', class: 'nav-link optional'
+            - unless whitelist_mode?
+              = link_to t('directories.directory'), explore_path, class: 'nav-link optional' if Setting.profile_directory
+              = link_to t('about.about_this'), about_more_path, class: 'nav-link optional'
+              = link_to t('about.apps'), 'https://joinmastodon.org/apps', class: 'nav-link optional'
+
           .nav-center
+
           .nav-right
             - if user_signed_in?
               = link_to t('settings.back'), root_url, class: 'nav-link nav-button webapp-btn'
diff --git a/config/initializers/2_whitelist_mode.rb b/config/initializers/2_whitelist_mode.rb
new file mode 100644
index 000000000..a17ad07a2
--- /dev/null
+++ b/config/initializers/2_whitelist_mode.rb
@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+Rails.application.configure do
+  config.x.whitelist_mode = ENV['WHITELIST_MODE'] == 'true'
+end
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 9e1be87be..6c1a34300 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -186,6 +186,7 @@ en:
       username: Username
       warn: Warn
       web: Web
+      whitelisted: Whitelisted
     action_logs:
       actions:
         assigned_to_self_report: "%{name} assigned report %{target} to themselves"
@@ -269,6 +270,11 @@ en:
       week_interactions: interactions this week
       week_users_active: active this week
       week_users_new: users this week
+    domain_allows:
+      add_new: Whitelist domain
+      created_msg: Domain has been successfully whitelisted
+      destroyed_msg: Domain has been removed from the whitelist
+      undo: Remove from whitelist
     domain_blocks:
       add_new: Add new domain block
       created_msg: Domain block is now being processed
@@ -524,6 +530,7 @@ en:
     apply_for_account: Request an invite
     change_password: Password
     checkbox_agreement_html: I agree to the <a href="%{rules_path}" target="_blank">server rules</a> and <a href="%{terms_path}" target="_blank">terms of service</a>
+    checkbox_agreement_without_rules_html: I agree to the <a href="%{terms_path}" target="_blank">terms of service</a>
     delete_account: Delete account
     delete_account_html: If you wish to delete your account, you can <a href="%{path}">proceed here</a>. You will be asked for confirmation.
     didnt_get_confirmation: Didn't receive confirmation instructions?
diff --git a/config/locales/simple_form.en.yml b/config/locales/simple_form.en.yml
index 12a7ec2b3..10b30e627 100644
--- a/config/locales/simple_form.en.yml
+++ b/config/locales/simple_form.en.yml
@@ -38,6 +38,8 @@ en:
         setting_use_pending_items: Hide timeline updates behind a click instead of automatically scrolling the feed
         username: Your username will be unique on %{domain}
         whole_word: When the keyword or phrase is alphanumeric only, it will only be applied if it matches the whole word
+      domain_allow:
+        domain: This domain will be able to fetch data from this server and incoming data from it will be processed and stored
       featured_tag:
         name: 'You might want to use one of these:'
       imports:
diff --git a/config/navigation.rb b/config/navigation.rb
index 5ab2e4399..9b46da603 100644
--- a/config/navigation.rb
+++ b/config/navigation.rb
@@ -39,7 +39,7 @@ SimpleNavigation::Configuration.run do |navigation|
       s.item :accounts, safe_join([fa_icon('users fw'), t('admin.accounts.title')]), admin_accounts_url, highlights_on: %r{/admin/accounts|/admin/pending_accounts}
       s.item :invites, safe_join([fa_icon('user-plus fw'), t('admin.invites.title')]), admin_invites_path
       s.item :tags, safe_join([fa_icon('tag fw'), t('admin.tags.title')]), admin_tags_path
-      s.item :instances, safe_join([fa_icon('cloud fw'), t('admin.instances.title')]), admin_instances_url(limited: '1'), highlights_on: %r{/admin/instances|/admin/domain_blocks}, if: -> { current_user.admin? }
+      s.item :instances, safe_join([fa_icon('cloud fw'), t('admin.instances.title')]), admin_instances_url(limited: whitelist_mode? ? nil : '1'), highlights_on: %r{/admin/instances|/admin/domain_blocks|/admin/domain_allows}, if: -> { current_user.admin? }
       s.item :email_domain_blocks, safe_join([fa_icon('envelope fw'), t('admin.email_domain_blocks.title')]), admin_email_domain_blocks_url, highlights_on: %r{/admin/email_domain_blocks}, if: -> { current_user.admin? }
     end
 
diff --git a/config/routes.rb b/config/routes.rb
index b6c215888..04424bbbd 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -154,6 +154,7 @@ Rails.application.routes.draw do
   namespace :admin do
     get '/dashboard', to: 'dashboard#index'
 
+    resources :domain_allows, only: [:new, :create, :show, :destroy]
     resources :domain_blocks, only: [:new, :create, :show, :destroy]
     resources :email_domain_blocks, only: [:index, :new, :create, :destroy]
     resources :action_logs, only: [:index]
diff --git a/db/migrate/20190705002136_create_domain_allows.rb b/db/migrate/20190705002136_create_domain_allows.rb
new file mode 100644
index 000000000..83b0728d9
--- /dev/null
+++ b/db/migrate/20190705002136_create_domain_allows.rb
@@ -0,0 +1,9 @@
+class CreateDomainAllows < ActiveRecord::Migration[5.2]
+  def change
+    create_table :domain_allows do |t|
+      t.string :domain, default: '', null: false, index: { unique: true }
+
+      t.timestamps
+    end
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 1847305c7..2d83d8b76 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_07_26_175042) do
+ActiveRecord::Schema.define(version: 2019_07_28_084117) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -245,6 +245,13 @@ ActiveRecord::Schema.define(version: 2019_07_26_175042) do
     t.index ["account_id"], name: "index_custom_filters_on_account_id"
   end
 
+  create_table "domain_allows", force: :cascade do |t|
+    t.string "domain", default: "", null: false
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["domain"], name: "index_domain_allows_on_domain", unique: true
+  end
+
   create_table "domain_blocks", force: :cascade do |t|
     t.string "domain", default: "", null: false
     t.datetime "created_at", null: false
diff --git a/lib/mastodon/domains_cli.rb b/lib/mastodon/domains_cli.rb
index b081581fe..f30062363 100644
--- a/lib/mastodon/domains_cli.rb
+++ b/lib/mastodon/domains_cli.rb
@@ -12,17 +12,33 @@ module Mastodon
     end
 
     option :dry_run, type: :boolean
-    desc 'purge DOMAIN', 'Remove accounts from a DOMAIN without a trace'
+    option :whitelist_mode, type: :boolean
+    desc 'purge [DOMAIN]', 'Remove accounts from a DOMAIN without a trace'
     long_desc <<-LONG_DESC
       Remove all accounts from a given DOMAIN without leaving behind any
       records. Unlike a suspension, if the DOMAIN still exists in the wild,
       it means the accounts could return if they are resolved again.
+
+      When the --whitelist-mode option is given, instead of purging accounts
+      from a single domain, all accounts from domains that are not whitelisted
+      are removed from the database.
     LONG_DESC
-    def purge(domain)
+    def purge(domain = nil)
       removed = 0
       dry_run = options[:dry_run] ? ' (DRY RUN)' : ''
 
-      Account.where(domain: domain).find_each do |account|
+      scope = begin
+        if options[:whitelist_mode]
+          Account.remote.where.not(domain: DomainAllow.pluck(:domain))
+        elsif domain.present?
+          Account.remote.where(domain: domain)
+        else
+          say('No domain given', :red)
+          exit(1)
+        end
+      end
+
+      scope.find_each do |account|
         SuspendAccountService.new.call(account, destroy: true) unless options[:dry_run]
         removed += 1
         say('.', :green, false)
diff --git a/spec/fabricators/domain_allow_fabricator.rb b/spec/fabricators/domain_allow_fabricator.rb
new file mode 100644
index 000000000..6226b1e20
--- /dev/null
+++ b/spec/fabricators/domain_allow_fabricator.rb
@@ -0,0 +1,3 @@
+Fabricator(:domain_allow) do
+  domain "MyString"
+end
diff --git a/spec/models/domain_allow_spec.rb b/spec/models/domain_allow_spec.rb
new file mode 100644
index 000000000..e65435127
--- /dev/null
+++ b/spec/models/domain_allow_spec.rb
@@ -0,0 +1,5 @@
+require 'rails_helper'
+
+RSpec.describe DomainAllow, type: :model do
+  pending "add some examples to (or delete) #{__FILE__}"
+end
diff --git a/streaming/index.js b/streaming/index.js
index 0529804b1..304e7e046 100644
--- a/streaming/index.js
+++ b/streaming/index.js
@@ -12,6 +12,7 @@ const uuid = require('uuid');
 const fs = require('fs');
 
 const env = process.env.NODE_ENV || 'development';
+const alwaysRequireAuth = process.env.WHITELIST_MODE === 'true' || process.env.AUTHORIZED_FETCH === 'true';
 
 dotenv.config({
   path: env === 'production' ? '.env.production' : '.env',
@@ -271,7 +272,7 @@ const startWorker = (workerId) => {
 
   const wsVerifyClient = (info, cb) => {
     const location = url.parse(info.req.url, true);
-    const authRequired = !PUBLIC_STREAMS.some(stream => stream === location.query.stream);
+    const authRequired = alwaysRequireAuth || !PUBLIC_STREAMS.some(stream => stream === location.query.stream);
     const allowedScopes = [];
 
     if (authRequired) {
@@ -306,7 +307,7 @@ const startWorker = (workerId) => {
       return;
     }
 
-    const authRequired = !PUBLIC_ENDPOINTS.some(endpoint => endpoint === req.path);
+    const authRequired = alwaysRequireAuth || !PUBLIC_ENDPOINTS.some(endpoint => endpoint === req.path);
     const allowedScopes = [];
 
     if (authRequired) {
-- 
cgit 


From 648cdbc04a21580a89d337edb0f45308aff1b93f Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Tue, 30 Jul 2019 13:10:40 +0200
Subject: Add hashtag score for better sorting of autosuggestions (#11427)

* Add hashtag score for better sorting of autosuggestions

* Do not use `~<~` operator with no text_pattern_ops index
---
 app/javascript/mastodon/reducers/compose.js    | 4 ++--
 app/models/tag.rb                              | 3 ++-
 app/models/trending_tags.rb                    | 7 ++++++-
 db/migrate/20190729185330_add_score_to_tags.rb | 5 +++++
 db/schema.rb                                   | 3 ++-
 5 files changed, 17 insertions(+), 5 deletions(-)
 create mode 100644 db/migrate/20190729185330_add_score_to_tags.rb

(limited to 'db/migrate')

diff --git a/app/javascript/mastodon/reducers/compose.js b/app/javascript/mastodon/reducers/compose.js
index e683a9c1a..7b0cdd5a5 100644
--- a/app/javascript/mastodon/reducers/compose.js
+++ b/app/javascript/mastodon/reducers/compose.js
@@ -153,9 +153,9 @@ const sortHashtagsByUse = (state, tags) => {
     if (usedA === usedB) {
       return 0;
     } else if (usedA && !usedB) {
-      return 1;
-    } else {
       return -1;
+    } else {
+      return 1;
     }
   });
 };
diff --git a/app/models/tag.rb b/app/models/tag.rb
index 46e3a3ec0..a2d6078f4 100644
--- a/app/models/tag.rb
+++ b/app/models/tag.rb
@@ -7,6 +7,7 @@
 #  name       :string           default(""), not null
 #  created_at :datetime         not null
 #  updated_at :datetime         not null
+#  score      :integer
 #
 
 class Tag < ApplicationRecord
@@ -78,7 +79,7 @@ class Tag < ApplicationRecord
       pattern = sanitize_sql_like(normalize(term.strip)) + '%'
 
       Tag.where(arel_table[:name].lower.matches(pattern.mb_chars.downcase.to_s))
-         .order(:name)
+         .order(Arel.sql('length(name) ASC, score DESC, name ASC'))
          .limit(limit)
          .offset(offset)
     end
diff --git a/app/models/trending_tags.rb b/app/models/trending_tags.rb
index 148535c21..34a4abbc5 100644
--- a/app/models/trending_tags.rb
+++ b/app/models/trending_tags.rb
@@ -48,12 +48,17 @@ class TrendingTags
         redis.zrem(key, tag_id.to_s)
       else
         score = ((observed - expected)**2) / expected
-        redis.zadd(key, score, tag_id.to_s)
+        added = redis.zadd(key, score, tag_id.to_s)
+        bump_tag_score!(tag_id) if added == 1
       end
 
       redis.expire(key, EXPIRE_TRENDS_AFTER)
     end
 
+    def bump_tag_score!(tag_id)
+      Tag.where(id: tag_id).update_all('score = COALESCE(score, 0) + 1')
+    end
+
     def disallowed_hashtags
       return @disallowed_hashtags if defined?(@disallowed_hashtags)
 
diff --git a/db/migrate/20190729185330_add_score_to_tags.rb b/db/migrate/20190729185330_add_score_to_tags.rb
new file mode 100644
index 000000000..75fee4b57
--- /dev/null
+++ b/db/migrate/20190729185330_add_score_to_tags.rb
@@ -0,0 +1,5 @@
+class AddScoreToTags < ActiveRecord::Migration[5.2]
+  def change
+    add_column :tags, :score, :int
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 2d83d8b76..e3af9c31a 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_07_28_084117) do
+ActiveRecord::Schema.define(version: 2019_07_29_185330) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -659,6 +659,7 @@ ActiveRecord::Schema.define(version: 2019_07_28_084117) do
     t.string "name", default: "", null: false
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
+    t.integer "score"
     t.index "lower((name)::text)", name: "index_tags_on_name_lower", unique: true
   end
 
-- 
cgit 


From 115dab78f1cc5357281dcb593f04ac8b2629cec6 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Mon, 5 Aug 2019 19:54:29 +0200
Subject: Change admin UI for hashtags and add back whitelisted trends (#11490)

Fix #271

Add back the `GET /api/v1/trends` API with the caveat that it does
not return tags that have not been allowed to trend by the staff.

When a hashtag begins to trend (internally) and that hashtag has
not been previously reviewed by the staff, the staff is notified.

The new admin UI for hashtags allows filtering hashtags by where
they are used (e.g. in the profile directory), whether they have
been reviewed or are pending reviewal, they show by how many people
the hashtag is used in the directory, how many people used it
today, how many statuses with it have been created today, and it
allows fixing the name of the hashtag to make it more readable.

The disallowed hashtags feature has been reworked. It is now
controlled from the admin UI for hashtags instead of from
the file `config/settings.yml`
---
 app/controllers/admin/dashboard_controller.rb      |  2 +-
 app/controllers/admin/tags_controller.rb           | 36 ++++++++-----
 app/controllers/api/v1/trends_controller.rb        | 17 ++++++
 app/controllers/settings/preferences_controller.rb |  2 +-
 app/helpers/admin/filter_helper.rb                 |  5 +-
 app/mailers/admin_mailer.rb                        | 10 ++++
 app/models/application_record.rb                   | 11 ++++
 app/models/tag.rb                                  | 60 +++++++++++++++++++---
 app/models/trending_tags.rb                        | 48 ++++++++---------
 app/models/user.rb                                 |  4 ++
 app/policies/tag_policy.rb                         |  4 +-
 app/validators/disallowed_hashtags_validator.rb    | 21 +-------
 app/views/admin/dashboard/index.html.haml          |  2 +-
 app/views/admin/tags/_tag.html.haml                | 24 +++++----
 app/views/admin/tags/index.html.haml               | 26 +++++-----
 app/views/admin/tags/show.html.haml                | 16 ++++++
 app/views/admin_mailer/new_trending_tag.text.erb   |  5 ++
 .../preferences/notifications/show.html.haml       |  1 +
 config/locales/en.yml                              | 18 ++++---
 config/locales/simple_form.en.yml                  |  7 +++
 config/navigation.rb                               |  2 +-
 config/routes.rb                                   |  9 +---
 config/settings.yml                                |  1 +
 .../20190805123746_add_capabilities_to_tags.rb     |  9 ++++
 db/schema.rb                                       |  7 ++-
 spec/controllers/admin/tags_controller_spec.rb     | 56 ++------------------
 spec/policies/tag_policy_spec.rb                   |  2 +-
 .../disallowed_hashtags_validator_spec.rb          | 26 +++++-----
 28 files changed, 258 insertions(+), 173 deletions(-)
 create mode 100644 app/controllers/api/v1/trends_controller.rb
 create mode 100644 app/views/admin/tags/show.html.haml
 create mode 100644 app/views/admin_mailer/new_trending_tag.text.erb
 create mode 100644 db/migrate/20190805123746_add_capabilities_to_tags.rb

(limited to 'db/migrate')

diff --git a/app/controllers/admin/dashboard_controller.rb b/app/controllers/admin/dashboard_controller.rb
index e74e4755f..70afdedd7 100644
--- a/app/controllers/admin/dashboard_controller.rb
+++ b/app/controllers/admin/dashboard_controller.rb
@@ -27,7 +27,7 @@ module Admin
       @saml_enabled          = ENV['SAML_ENABLED'] == 'true'
       @pam_enabled           = ENV['PAM_ENABLED'] == 'true'
       @hidden_service        = ENV['ALLOW_ACCESS_TO_HIDDEN_SERVICE'] == 'true'
-      @trending_hashtags     = TrendingTags.get(7)
+      @trending_hashtags     = TrendingTags.get(10, filtered: false)
       @profile_directory     = Setting.profile_directory
       @timeline_preview      = Setting.timeline_preview
       @spam_check_enabled    = Setting.spam_check_enabled
diff --git a/app/controllers/admin/tags_controller.rb b/app/controllers/admin/tags_controller.rb
index e9f4f2cfa..0e9dda302 100644
--- a/app/controllers/admin/tags_controller.rb
+++ b/app/controllers/admin/tags_controller.rb
@@ -4,41 +4,49 @@ module Admin
   class TagsController < BaseController
     before_action :set_tags, only: :index
     before_action :set_tag, except: :index
-    before_action :set_filter_params
 
     def index
       authorize :tag, :index?
     end
 
-    def hide
-      authorize @tag, :hide?
-      @tag.account_tag_stat.update!(hidden: true)
-      redirect_to admin_tags_path(@filter_params)
+    def show
+      authorize @tag, :show?
     end
 
-    def unhide
-      authorize @tag, :unhide?
-      @tag.account_tag_stat.update!(hidden: false)
-      redirect_to admin_tags_path(@filter_params)
+    def update
+      authorize @tag, :update?
+
+      if @tag.update(tag_params.merge(reviewed_at: Time.now.utc))
+        redirect_to admin_tag_path(@tag.id)
+      else
+        render :show
+      end
     end
 
     private
 
     def set_tags
-      @tags = Tag.discoverable
-      @tags.merge!(Tag.hidden) if filter_params[:hidden]
+      @tags = filtered_tags.page(params[:page])
     end
 
     def set_tag
       @tag = Tag.find(params[:id])
     end
 
-    def set_filter_params
-      @filter_params = filter_params.to_hash.symbolize_keys
+    def filtered_tags
+      scope = Tag
+      scope = scope.discoverable if filter_params[:context] == 'directory'
+      scope = scope.reviewed if filter_params[:review] == 'reviewed'
+      scope = scope.pending_review if filter_params[:review] == 'pending_review'
+      scope.reorder(score: :desc)
     end
 
     def filter_params
-      params.permit(:hidden)
+      params.slice(:context, :review).permit(:context, :review)
+    end
+
+    def tag_params
+      params.require(:tag).permit(:name, :trendable, :usable, :listable)
     end
   end
 end
diff --git a/app/controllers/api/v1/trends_controller.rb b/app/controllers/api/v1/trends_controller.rb
new file mode 100644
index 000000000..bcea9857e
--- /dev/null
+++ b/app/controllers/api/v1/trends_controller.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+class Api::V1::TrendsController < Api::BaseController
+  before_action :set_tags
+
+  respond_to :json
+
+  def index
+    render json: @tags, each_serializer: REST::TagSerializer
+  end
+
+  private
+
+  def set_tags
+    @tags = TrendingTags.get(limit_param(10))
+  end
+end
diff --git a/app/controllers/settings/preferences_controller.rb b/app/controllers/settings/preferences_controller.rb
index 742c97cdb..d548072a8 100644
--- a/app/controllers/settings/preferences_controller.rb
+++ b/app/controllers/settings/preferences_controller.rb
@@ -56,7 +56,7 @@ class Settings::PreferencesController < Settings::BaseController
       :setting_advanced_layout,
       :setting_use_blurhash,
       :setting_use_pending_items,
-      notification_emails: %i(follow follow_request reblog favourite mention digest report pending_account),
+      notification_emails: %i(follow follow_request reblog favourite mention digest report pending_account trending_tag),
       interactions: %i(must_be_follower must_be_following must_be_following_dm)
     )
   end
diff --git a/app/helpers/admin/filter_helper.rb b/app/helpers/admin/filter_helper.rb
index 0bda25974..506429e10 100644
--- a/app/helpers/admin/filter_helper.rb
+++ b/app/helpers/admin/filter_helper.rb
@@ -5,15 +5,16 @@ module Admin::FilterHelper
   REPORT_FILTERS       = %i(resolved account_id target_account_id).freeze
   INVITE_FILTER        = %i(available expired).freeze
   CUSTOM_EMOJI_FILTERS = %i(local remote by_domain shortcode).freeze
-  TAGS_FILTERS         = %i(hidden).freeze
+  TAGS_FILTERS         = %i(context review).freeze
   INSTANCES_FILTERS    = %i(limited by_domain).freeze
   FOLLOWERS_FILTERS    = %i(relationship status by_domain activity order).freeze
 
   FILTERS = ACCOUNT_FILTERS + REPORT_FILTERS + INVITE_FILTER + CUSTOM_EMOJI_FILTERS + TAGS_FILTERS + INSTANCES_FILTERS + FOLLOWERS_FILTERS
 
   def filter_link_to(text, link_to_params, link_class_params = link_to_params)
-    new_url = filtered_url_for(link_to_params)
+    new_url   = filtered_url_for(link_to_params)
     new_class = filtered_url_for(link_class_params)
+
     link_to text, new_url, class: filter_link_class(new_class)
   end
 
diff --git a/app/mailers/admin_mailer.rb b/app/mailers/admin_mailer.rb
index 9ab3e2bbd..8abce5f05 100644
--- a/app/mailers/admin_mailer.rb
+++ b/app/mailers/admin_mailer.rb
@@ -24,4 +24,14 @@ class AdminMailer < ApplicationMailer
       mail to: @me.user_email, subject: I18n.t('admin_mailer.new_pending_account.subject', instance: @instance, username: @account.username)
     end
   end
+
+  def new_trending_tag(recipient, tag)
+    @tag      = tag
+    @me       = recipient
+    @instance = Rails.configuration.x.local_domain
+
+    locale_for_account(@me) do
+      mail to: @me.user_email, subject: I18n.t('admin_mailer.new_trending_tag.subject', instance: @instance, name: @tag.name)
+    end
+  end
 end
diff --git a/app/models/application_record.rb b/app/models/application_record.rb
index 83134d41a..c1b873da6 100644
--- a/app/models/application_record.rb
+++ b/app/models/application_record.rb
@@ -2,5 +2,16 @@
 
 class ApplicationRecord < ActiveRecord::Base
   self.abstract_class = true
+
   include Remotable
+
+  def boolean_with_default(key, default_value)
+    value = attributes[key]
+
+    if value.nil?
+      default_value
+    else
+      value
+    end
+  end
 end
diff --git a/app/models/tag.rb b/app/models/tag.rb
index c7f0af86d..6a02581fa 100644
--- a/app/models/tag.rb
+++ b/app/models/tag.rb
@@ -3,11 +3,16 @@
 #
 # Table name: tags
 #
-#  id         :bigint(8)        not null, primary key
-#  name       :string           default(""), not null
-#  created_at :datetime         not null
-#  updated_at :datetime         not null
-#  score      :integer
+#  id                  :bigint(8)        not null, primary key
+#  name                :string           default(""), not null
+#  created_at          :datetime         not null
+#  updated_at          :datetime         not null
+#  score               :integer
+#  usable              :boolean
+#  trendable           :boolean
+#  listable            :boolean
+#  reviewed_at         :datetime
+#  requested_review_at :datetime
 #
 
 class Tag < ApplicationRecord
@@ -22,16 +27,17 @@ class Tag < ApplicationRecord
   HASHTAG_RE = /(?:^|[^\/\)\w])#(#{HASHTAG_NAME_RE})/i
 
   validates :name, presence: true, format: { with: /\A(#{HASHTAG_NAME_RE})\z/i }
+  validate :validate_name_change, if: -> { !new_record? && name_changed? }
 
-  scope :discoverable, -> { joins(:account_tag_stat).where(AccountTagStat.arel_table[:accounts_count].gt(0)).where(account_tag_stats: { hidden: false }).order(Arel.sql('account_tag_stats.accounts_count desc')) }
-  scope :hidden, -> { where(account_tag_stats: { hidden: true }) }
+  scope :reviewed, -> { where.not(reviewed_at: nil) }
+  scope :pending_review, -> { where(reviewed_at: nil).where.not(requested_review_at: nil) }
+  scope :discoverable, -> { where.not(listable: false).joins(:account_tag_stat).where(AccountTagStat.arel_table[:accounts_count].gt(0)).order(Arel.sql('account_tag_stats.accounts_count desc')) }
   scope :most_used, ->(account) { joins(:statuses).where(statuses: { account: account }).group(:id).order(Arel.sql('count(*) desc')) }
 
   delegate :accounts_count,
            :accounts_count=,
            :increment_count!,
            :decrement_count!,
-           :hidden?,
            to: :account_tag_stat
 
   after_save :save_account_tag_stat
@@ -48,6 +54,40 @@ class Tag < ApplicationRecord
     name
   end
 
+  def usable
+    boolean_with_default('usable', true)
+  end
+
+  alias usable? usable
+
+  def listable
+    boolean_with_default('listable', true)
+  end
+
+  alias listable? listable
+
+  def trendable
+    boolean_with_default('trendable', false)
+  end
+
+  alias trendable? trendable
+
+  def requires_review?
+    reviewed_at.nil?
+  end
+
+  def reviewed?
+    reviewed_at.present?
+  end
+
+  def requested_review?
+    requested_review_at.present?
+  end
+
+  def trending?
+    TrendingTags.trending?(self)
+  end
+
   def history
     days = []
 
@@ -117,4 +157,8 @@ class Tag < ApplicationRecord
     return unless account_tag_stat&.changed?
     account_tag_stat.save
   end
+
+  def validate_name_change
+    errors.add(:name, I18n.t('tags.does_not_match_previous_name')) unless name_was.mb_chars.casecmp(name.mb_chars).zero?
+  end
 end
diff --git a/app/models/trending_tags.rb b/app/models/trending_tags.rb
index 211c8f1dc..e9b9b25e3 100644
--- a/app/models/trending_tags.rb
+++ b/app/models/trending_tags.rb
@@ -10,20 +10,28 @@ class TrendingTags
     include Redisable
 
     def record_use!(tag, account, at_time = Time.now.utc)
-      return if disallowed_hashtags.include?(tag.name) || account.silenced? || account.bot?
+      return if account.silenced? || account.bot? || !tag.usable? || !(tag.trendable? || tag.requires_review?)
 
       increment_historical_use!(tag.id, at_time)
       increment_unique_use!(tag.id, account.id, at_time)
-      increment_vote!(tag.id, at_time)
+      increment_vote!(tag, at_time)
     end
 
-    def get(limit)
-      key     = "#{KEY}:#{Time.now.utc.beginning_of_day.to_i}"
-      tag_ids = redis.zrevrange(key, 0, limit - 1).map(&:to_i)
-      tags    = Tag.where(id: tag_ids).to_a.each_with_object({}) { |tag, h| h[tag.id] = tag }
+    def get(limit, filtered: true)
+      tag_ids = redis.zrevrange("#{KEY}:#{Time.now.utc.beginning_of_day.to_i}", 0, limit - 1).map(&:to_i)
+
+      tags = Tag.where(id: tag_ids)
+      tags = tags.where(trendable: true) if filtered
+      tags = tags.each_with_object({}) { |tag, h| h[tag.id] = tag }
+
       tag_ids.map { |tag_id| tags[tag_id] }.compact
     end
 
+    def trending?(tag)
+      rank = redis.zrevrank("#{KEY}:#{Time.now.utc.beginning_of_day.to_i}", tag.id)
+      rank.present? && rank <= 10
+    end
+
     private
 
     def increment_historical_use!(tag_id, at_time)
@@ -38,33 +46,27 @@ class TrendingTags
       redis.expire(key, EXPIRE_HISTORY_AFTER)
     end
 
-    def increment_vote!(tag_id, at_time)
+    def increment_vote!(tag, at_time)
       key      = "#{KEY}:#{at_time.beginning_of_day.to_i}"
-      expected = redis.pfcount("activity:tags:#{tag_id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts").to_f
+      expected = redis.pfcount("activity:tags:#{tag.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts").to_f
       expected = 1.0 if expected.zero?
-      observed = redis.pfcount("activity:tags:#{tag_id}:#{at_time.beginning_of_day.to_i}:accounts").to_f
+      observed = redis.pfcount("activity:tags:#{tag.id}:#{at_time.beginning_of_day.to_i}:accounts").to_f
 
       if expected > observed || observed < THRESHOLD
-        redis.zrem(key, tag_id.to_s)
+        redis.zrem(key, tag.id)
       else
-        score = ((observed - expected)**2) / expected
-        added = redis.zadd(key, score, tag_id.to_s)
-        bump_tag_score!(tag_id) if added
+        score    = ((observed - expected)**2) / expected
+        old_rank = redis.zrevrank(key, tag.id)
+
+        redis.zadd(key, score, tag.id)
+        request_review!(tag) if (old_rank.nil? || old_rank > 10) && redis.zrevrank(key, tag.id) <= 10 && !tag.trendable? && tag.requires_review? && !tag.requested_review?
       end
 
       redis.expire(key, EXPIRE_TRENDS_AFTER)
     end
 
-    def bump_tag_score!(tag_id)
-      Tag.where(id: tag_id).update_all('score = COALESCE(score, 0) + 1')
-    end
-
-    def disallowed_hashtags
-      return @disallowed_hashtags if defined?(@disallowed_hashtags)
-
-      @disallowed_hashtags = Setting.disallowed_hashtags.nil? ? [] : Setting.disallowed_hashtags
-      @disallowed_hashtags = @disallowed_hashtags.split(' ') if @disallowed_hashtags.is_a? String
-      @disallowed_hashtags = @disallowed_hashtags.map(&:downcase)
+    def request_review!(tag)
+      User.staff.includes(:account).find_each { |u| AdminMailer.new_trending_tag(u.account, tag).deliver_later! if u.allows_trending_tag_emails? }
     end
   end
 end
diff --git a/app/models/user.rb b/app/models/user.rb
index 6806c0362..b83e26af3 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -207,6 +207,10 @@ class User < ApplicationRecord
     settings.notification_emails['pending_account']
   end
 
+  def allows_trending_tag_emails?
+    settings.notification_emails['trending_tag']
+  end
+
   def hides_network?
     @hides_network ||= settings.hide_network
   end
diff --git a/app/policies/tag_policy.rb b/app/policies/tag_policy.rb
index c63de01db..aaf70fcab 100644
--- a/app/policies/tag_policy.rb
+++ b/app/policies/tag_policy.rb
@@ -5,11 +5,11 @@ class TagPolicy < ApplicationPolicy
     staff?
   end
 
-  def hide?
+  def show?
     staff?
   end
 
-  def unhide?
+  def update?
     staff?
   end
 end
diff --git a/app/validators/disallowed_hashtags_validator.rb b/app/validators/disallowed_hashtags_validator.rb
index ee06b20f6..d745b767f 100644
--- a/app/validators/disallowed_hashtags_validator.rb
+++ b/app/validators/disallowed_hashtags_validator.rb
@@ -4,24 +4,7 @@ class DisallowedHashtagsValidator < ActiveModel::Validator
   def validate(status)
     return unless status.local? && !status.reblog?
 
-    @status = status
-    tags    = select_tags
-
-    status.errors.add(:text, I18n.t('statuses.disallowed_hashtags', tags: tags.join(', '), count: tags.size)) unless tags.empty?
-  end
-
-  private
-
-  def select_tags
-    tags = Extractor.extract_hashtags(@status.text)
-    tags.keep_if { |tag| disallowed_hashtags.include? tag.downcase }
-  end
-
-  def disallowed_hashtags
-    return @disallowed_hashtags if @disallowed_hashtags
-
-    @disallowed_hashtags = Setting.disallowed_hashtags.nil? ? [] : Setting.disallowed_hashtags
-    @disallowed_hashtags = @disallowed_hashtags.split(' ') if @disallowed_hashtags.is_a? String
-    @disallowed_hashtags = @disallowed_hashtags.map(&:downcase)
+    disallowed_hashtags = Tag.matching_name(Extractor.extract_hashtags(status.text)).reject(&:usable?)
+    status.errors.add(:text, I18n.t('statuses.disallowed_hashtags', tags: disallowed_hashtags.map(&:name).join(', '), count: disallowed_hashtags.size)) unless disallowed_hashtags.empty?
   end
 end
diff --git a/app/views/admin/dashboard/index.html.haml b/app/views/admin/dashboard/index.html.haml
index 77cc1a2a0..910896075 100644
--- a/app/views/admin/dashboard/index.html.haml
+++ b/app/views/admin/dashboard/index.html.haml
@@ -107,5 +107,5 @@
       %ul
         - @trending_hashtags.each do |tag|
           %li
-            = link_to "##{tag.name}", web_url("timelines/tag/#{tag.name}")
+            = link_to content_tag(:span, "##{tag.name}", class: !tag.trendable? && !tag.reviewed? ? 'warning-hint' : (!tag.trendable? ? 'negative-hint' : nil)), admin_tag_path(tag.id)
             %span.pull-right= number_with_delimiter(tag.history[0][:accounts].to_i)
diff --git a/app/views/admin/tags/_tag.html.haml b/app/views/admin/tags/_tag.html.haml
index 961b83f93..91af8e492 100644
--- a/app/views/admin/tags/_tag.html.haml
+++ b/app/views/admin/tags/_tag.html.haml
@@ -1,12 +1,16 @@
-%tr
-  %td
-    = link_to explore_hashtag_path(tag) do
+.directory__tag
+  = link_to admin_tag_path(tag.id) do
+    %h4
       = fa_icon 'hashtag'
       = tag.name
-  %td
-    = t('directories.people', count: tag.accounts_count)
-  %td
-    - if tag.hidden?
-      = table_link_to 'eye', t('admin.tags.unhide'), unhide_admin_tag_path(tag.id, **@filter_params), method: :post
-    - else
-      = table_link_to 'eye-slash', t('admin.tags.hide'), hide_admin_tag_path(tag.id, **@filter_params), method: :post
+
+      %small
+        = t('admin.tags.in_directory', count: tag.accounts_count)
+        &bull;
+        = t('admin.tags.unique_uses_today', count: tag.history.first[:accounts])
+
+        - if tag.trending?
+          = fa_icon 'fire fw'
+          = t('admin.tags.trending_right_now')
+
+    .trends__item__current= number_to_human tag.history.first[:uses], strip_insignificant_zeros: true
diff --git a/app/views/admin/tags/index.html.haml b/app/views/admin/tags/index.html.haml
index 4ba395860..5e4ee21f5 100644
--- a/app/views/admin/tags/index.html.haml
+++ b/app/views/admin/tags/index.html.haml
@@ -3,17 +3,19 @@
 
 .filters
   .filter-subset
-    %strong= t('admin.reports.status')
+    %strong= t('admin.tags.context')
     %ul
-      %li= filter_link_to t('admin.tags.visible'), hidden: nil
-      %li= filter_link_to t('admin.tags.hidden'), hidden: '1'
+      %li= filter_link_to t('generic.all'), context: nil
+      %li= filter_link_to t('admin.tags.directory'), context: 'directory'
 
-.table-wrapper
-  %table.table
-    %thead
-      %tr
-        %th= t('admin.tags.name')
-        %th= t('admin.tags.accounts')
-        %th
-    %tbody
-      = render @tags
+  .filter-subset
+    %strong= t('admin.tags.review')
+    %ul
+      %li= filter_link_to t('generic.all'), review: nil
+      %li= filter_link_to t('admin.tags.reviewed'), review: 'reviewed'
+      %li= filter_link_to safe_join([t('admin.accounts.moderation.pending'), "(#{Tag.pending_review.count})"], ' '), review: 'pending_review'
+
+%hr.spacer/
+
+= render @tags
+= paginate @tags
diff --git a/app/views/admin/tags/show.html.haml b/app/views/admin/tags/show.html.haml
new file mode 100644
index 000000000..27c8dc92b
--- /dev/null
+++ b/app/views/admin/tags/show.html.haml
@@ -0,0 +1,16 @@
+- content_for :page_title do
+  = "##{@tag.name}"
+
+= simple_form_for @tag, url: admin_tag_path(@tag.id) do |f|
+  = render 'shared/error_messages', object: @tag
+
+  .fields-group
+    = f.input :name, wrapper: :with_block_label
+
+  .fields-group
+    = f.input :usable, as: :boolean, wrapper: :with_label
+    = f.input :trendable, as: :boolean, wrapper: :with_label
+    = f.input :listable, as: :boolean, wrapper: :with_label
+
+  .actions
+    = f.button :button, t('generic.save_changes'), type: :submit
diff --git a/app/views/admin_mailer/new_trending_tag.text.erb b/app/views/admin_mailer/new_trending_tag.text.erb
new file mode 100644
index 000000000..f3087df37
--- /dev/null
+++ b/app/views/admin_mailer/new_trending_tag.text.erb
@@ -0,0 +1,5 @@
+<%= raw t('application_mailer.salutation', name: display_name(@me)) %>
+
+<%= raw t('admin_mailer.new_trending_tag.body', name: @tag.name) %>
+
+<%= raw t('application_mailer.view')%> <%= admin_tags_url(review: 'pending_review') %>
diff --git a/app/views/settings/preferences/notifications/show.html.haml b/app/views/settings/preferences/notifications/show.html.haml
index acc646fc3..f666ae4ff 100644
--- a/app/views/settings/preferences/notifications/show.html.haml
+++ b/app/views/settings/preferences/notifications/show.html.haml
@@ -15,6 +15,7 @@
       - if current_user.staff?
         = ff.input :report, as: :boolean, wrapper: :with_label
         = ff.input :pending_account, as: :boolean, wrapper: :with_label
+        = ff.input :trending_tag, as: :boolean, wrapper: :with_label
 
   .fields-group
     = f.simple_fields_for :notification_emails, hash_to_object(current_user.settings.notification_emails) do |ff|
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 6c1a34300..9b62aac3a 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -483,13 +483,14 @@ en:
       title: Account statuses
       with_media: With media
     tags:
-      accounts: Accounts
-      hidden: Hidden
-      hide: Hide from directory
-      name: Hashtag
+      context: Context
+      directory: In directory
+      in_directory: "%{count} in directory"
+      review: Review status
+      reviewed: Reviewed
       title: Hashtags
-      unhide: Show in directory
-      visible: Visible
+      trending_right_now: Trending right now
+      unique_uses_today: "%{count} posting today"
     title: Administration
     warning_presets:
       add_new: Add new
@@ -505,6 +506,9 @@ en:
       body: "%{reporter} has reported %{target}"
       body_remote: Someone from %{domain} has reported %{target}
       subject: New report for %{instance} (#%{id})
+    new_trending_tag:
+      body: 'The hashtag #%{name} is trending today, but has not been previously reviewed. It will not be displayed publicly unless you allow it to, or just save the form as it is to never hear about it again.'
+      subject: New hashtag up for review on %{instance} (#%{name})
   appearance:
     advanced_web_interface: Advanced web interface
     advanced_web_interface_hint: 'If you want to make use of your entire screen width, the advanced web interface allows you to configure many different columns to see as much information at the same time as you want: Home, notifications, federated timeline, any number of lists and hashtags.'
@@ -939,6 +943,8 @@ en:
     pinned: Pinned toot
     reblogged: boosted
     sensitive_content: Sensitive content
+  tags:
+    does_not_match_previous_name: does not match the previous name
   terms:
     body_html: |
       <h2>Privacy Policy</h2>
diff --git a/config/locales/simple_form.en.yml b/config/locales/simple_form.en.yml
index 10b30e627..6fdfc9d7b 100644
--- a/config/locales/simple_form.en.yml
+++ b/config/locales/simple_form.en.yml
@@ -48,6 +48,8 @@ en:
         text: This will help us review your application
       sessions:
         otp: 'Enter the two-factor code generated by your phone app or use one of your recovery codes:'
+      tag:
+        name: You can only change the casing of the letters, for example, to make it more readable
       user:
         chosen_languages: When checked, only toots in selected languages will be displayed in public timelines
     labels:
@@ -137,6 +139,11 @@ en:
         pending_account: Send e-mail when a new account needs review
         reblog: Send e-mail when someone boosts your status
         report: Send e-mail when a new report is submitted
+        trending_tag: Send e-mail when an unreviewed hashtag is trending
+      tag:
+        listable: Allow this hashtag to appear on the profile directory
+        trendable: Allow this hashtag to appear under trends
+        usable: Allow toots to use this hashtag
     'no': 'No'
     recommended: Recommended
     required:
diff --git a/config/navigation.rb b/config/navigation.rb
index 9b46da603..38668bbf7 100644
--- a/config/navigation.rb
+++ b/config/navigation.rb
@@ -38,7 +38,7 @@ SimpleNavigation::Configuration.run do |navigation|
       s.item :reports, safe_join([fa_icon('flag fw'), t('admin.reports.title')]), admin_reports_url, highlights_on: %r{/admin/reports}
       s.item :accounts, safe_join([fa_icon('users fw'), t('admin.accounts.title')]), admin_accounts_url, highlights_on: %r{/admin/accounts|/admin/pending_accounts}
       s.item :invites, safe_join([fa_icon('user-plus fw'), t('admin.invites.title')]), admin_invites_path
-      s.item :tags, safe_join([fa_icon('tag fw'), t('admin.tags.title')]), admin_tags_path
+      s.item :tags, safe_join([fa_icon('hashtag fw'), t('admin.tags.title')]), admin_tags_path, highlights_on: %r{/admin/tags}
       s.item :instances, safe_join([fa_icon('cloud fw'), t('admin.instances.title')]), admin_instances_url(limited: whitelist_mode? ? nil : '1'), highlights_on: %r{/admin/instances|/admin/domain_blocks|/admin/domain_allows}, if: -> { current_user.admin? }
       s.item :email_domain_blocks, safe_join([fa_icon('envelope fw'), t('admin.email_domain_blocks.title')]), admin_email_domain_blocks_url, highlights_on: %r{/admin/email_domain_blocks}, if: -> { current_user.admin? }
     end
diff --git a/config/routes.rb b/config/routes.rb
index 04424bbbd..60f7d2e05 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -243,13 +243,7 @@ Rails.application.routes.draw do
     end
 
     resources :account_moderation_notes, only: [:create, :destroy]
-
-    resources :tags, only: [:index] do
-      member do
-        post :hide
-        post :unhide
-      end
-    end
+    resources :tags, only: [:index, :show, :update]
   end
 
   get '/admin', to: redirect('/admin/dashboard', status: 302)
@@ -311,6 +305,7 @@ Rails.application.routes.draw do
       resources :mutes,        only: [:index]
       resources :favourites,   only: [:index]
       resources :reports,      only: [:create]
+      resources :trends,       only: [:index]
       resources :filters,      only: [:index, :create, :show, :update, :destroy]
       resources :endorsements, only: [:index]
 
diff --git a/config/settings.yml b/config/settings.yml
index ad2970bb7..10180201f 100644
--- a/config/settings.yml
+++ b/config/settings.yml
@@ -43,6 +43,7 @@ defaults: &defaults
     digest: true
     report: true
     pending_account: true
+    trending_tag: true
   interactions:
     must_be_follower: false
     must_be_following: false
diff --git a/db/migrate/20190805123746_add_capabilities_to_tags.rb b/db/migrate/20190805123746_add_capabilities_to_tags.rb
new file mode 100644
index 000000000..43c7763b1
--- /dev/null
+++ b/db/migrate/20190805123746_add_capabilities_to_tags.rb
@@ -0,0 +1,9 @@
+class AddCapabilitiesToTags < ActiveRecord::Migration[5.2]
+  def change
+    add_column :tags, :usable, :boolean
+    add_column :tags, :trendable, :boolean
+    add_column :tags, :listable, :boolean
+    add_column :tags, :reviewed_at, :datetime
+    add_column :tags, :requested_review_at, :datetime
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index e3af9c31a..d1b6825b4 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_07_29_185330) do
+ActiveRecord::Schema.define(version: 2019_08_05_123746) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -660,6 +660,11 @@ ActiveRecord::Schema.define(version: 2019_07_29_185330) do
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
     t.integer "score"
+    t.boolean "usable"
+    t.boolean "trendable"
+    t.boolean "listable"
+    t.datetime "reviewed_at"
+    t.datetime "requested_review_at"
     t.index "lower((name)::text)", name: "index_tags_on_name_lower", unique: true
   end
 
diff --git a/spec/controllers/admin/tags_controller_spec.rb b/spec/controllers/admin/tags_controller_spec.rb
index 3af994071..5c1944fc7 100644
--- a/spec/controllers/admin/tags_controller_spec.rb
+++ b/spec/controllers/admin/tags_controller_spec.rb
@@ -10,62 +10,14 @@ RSpec.describe Admin::TagsController, type: :controller do
   end
 
   describe 'GET #index' do
-    before do
-      account_tag_stat = Fabricate(:tag).account_tag_stat
-      account_tag_stat.update(hidden: hidden, accounts_count: 1)
-      get :index, params: { hidden: hidden }
-    end
-
-    context 'with hidden tags' do
-      let(:hidden) { true }
-
-      it 'returns status 200' do
-        expect(response).to have_http_status(200)
-      end
-    end
-
-    context 'without hidden tags' do
-      let(:hidden) { false }
-
-      it 'returns status 200' do
-        expect(response).to have_http_status(200)
-      end
-    end
-  end
-
-  describe 'POST #hide' do
-    let(:tag) { Fabricate(:tag) }
+    let!(:tag) { Fabricate(:tag) }
 
     before do
-      tag.account_tag_stat.update(hidden: false)
-      post :hide, params: { id: tag.id }
-    end
-
-    it 'hides tag' do
-      tag.reload
-      expect(tag).to be_hidden
-    end
-
-    it 'redirects to admin_tags_path' do
-      expect(response).to redirect_to(admin_tags_path(controller.instance_variable_get(:@filter_params)))
-    end
-  end
-
-  describe 'POST #unhide' do
-    let(:tag) { Fabricate(:tag) }
-
-    before do
-      tag.account_tag_stat.update(hidden: true)
-      post :unhide, params: { id: tag.id }
-    end
-
-    it 'unhides tag' do
-      tag.reload
-      expect(tag).not_to be_hidden
+      get :index
     end
 
-    it 'redirects to admin_tags_path' do
-      expect(response).to redirect_to(admin_tags_path(controller.instance_variable_get(:@filter_params)))
+    it 'returns status 200' do
+      expect(response).to have_http_status(200)
     end
   end
 end
diff --git a/spec/policies/tag_policy_spec.rb b/spec/policies/tag_policy_spec.rb
index c7afaa7c9..c63875dc0 100644
--- a/spec/policies/tag_policy_spec.rb
+++ b/spec/policies/tag_policy_spec.rb
@@ -8,7 +8,7 @@ RSpec.describe TagPolicy do
   let(:admin)   { Fabricate(:user, admin: true).account }
   let(:john)    { Fabricate(:user).account }
 
-  permissions :index?, :hide?, :unhide? do
+  permissions :index?, :show?, :update? do
     context 'staff?' do
       it 'permits' do
         expect(subject).to permit(admin, Tag)
diff --git a/spec/validators/disallowed_hashtags_validator_spec.rb b/spec/validators/disallowed_hashtags_validator_spec.rb
index 8ec1302ab..9deec0bb9 100644
--- a/spec/validators/disallowed_hashtags_validator_spec.rb
+++ b/spec/validators/disallowed_hashtags_validator_spec.rb
@@ -3,42 +3,44 @@
 require 'rails_helper'
 
 RSpec.describe DisallowedHashtagsValidator, type: :validator do
+  let(:disallowed_tags) { [] }
+
   describe '#validate' do
     before do
-      allow_any_instance_of(described_class).to receive(:select_tags) { tags }
+      disallowed_tags.each { |name| Fabricate(:tag, name: name, usable: false) }
       described_class.new.validate(status)
     end
 
-    let(:status) { double(errors: errors, local?: local, reblog?: reblog, text: '') }
+    let(:status) { double(errors: errors, local?: local, reblog?: reblog, text: disallowed_tags.map { |x| '#' + x }.join(' ')) }
     let(:errors) { double(add: nil) }
 
-    context 'unless status.local? && !status.reblog?' do
+    context 'for a remote reblog' do
       let(:local)  { false }
       let(:reblog) { true }
 
-      it 'not calls errors.add' do
+      it 'does not add errors' do
         expect(errors).not_to have_received(:add).with(:text, any_args)
       end
     end
 
-    context 'status.local? && !status.reblog?' do
+    context 'for a local original status' do
       let(:local)  { true }
       let(:reblog) { false }
 
-      context 'tags.empty?' do
-        let(:tags) { [] }
+      context 'when does not contain any disallowed hashtags' do
+        let(:disallowed_tags) { [] }
 
-        it 'not calls errors.add' do
+        it 'does not add errors' do
           expect(errors).not_to have_received(:add).with(:text, any_args)
         end
       end
 
-      context '!tags.empty?' do
-        let(:tags) { %w(a b c) }
+      context 'when contains disallowed hashtags' do
+        let(:disallowed_tags) { %w(a b c) }
 
-        it 'calls errors.add' do
+        it 'adds an error' do
           expect(errors).to have_received(:add)
-            .with(:text, I18n.t('statuses.disallowed_hashtags', tags: tags.join(', '), count: tags.size))
+            .with(:text, I18n.t('statuses.disallowed_hashtags', tags: disallowed_tags.join(', '), count: disallowed_tags.size))
         end
       end
     end
-- 
cgit 


From bced70469a6c4aecdb3c71055f329a0f579eb14c Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Wed, 7 Aug 2019 20:20:23 +0200
Subject: Add domain block notes (#11515)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Add database columns for adding notes to domain blocks/restrctions

* Add admin UI to set private and public comments when blocking a domain

* Add text for private and public comments on domain blocks

* Show domain block comments in admin UI

* Add comments to the domain block undo page

* Make UnblockDomainService more robust regarding upgraded domain blocks

* Allow editing domain blocks

* Rename button from “undo domain block” to “view domain block” in account admin UI

* Change test to unsilence silenced users from upgraded blocks
---
 app/controllers/admin/domain_blocks_controller.rb  | 28 ++++++++++++++++++--
 app/controllers/admin/instances_controller.rb      |  2 ++
 app/models/domain_block.rb                         | 16 +++++++-----
 app/services/block_domain_service.rb               | 11 +++++++-
 app/services/unblock_domain_service.rb             | 19 ++------------
 app/views/admin/accounts/show.html.haml            |  2 +-
 app/views/admin/domain_blocks/edit.html.haml       | 30 ++++++++++++++++++++++
 app/views/admin/domain_blocks/new.html.haml        |  6 +++++
 app/views/admin/domain_blocks/show.html.haml       | 12 +++++++++
 app/views/admin/instances/show.html.haml           | 13 ++++++++++
 app/workers/domain_block_worker.rb                 |  4 +--
 config/locales/en.yml                              |  8 ++++++
 config/routes.rb                                   |  6 ++++-
 ...20190807135426_add_comments_to_domain_blocks.rb |  7 +++++
 db/schema.rb                                       |  4 ++-
 spec/services/unblock_domain_service_spec.rb       |  2 +-
 spec/workers/domain_block_worker_spec.rb           |  2 +-
 17 files changed, 138 insertions(+), 34 deletions(-)
 create mode 100644 app/views/admin/domain_blocks/edit.html.haml
 create mode 100644 db/migrate/20190807135426_add_comments_to_domain_blocks.rb

(limited to 'db/migrate')

diff --git a/app/controllers/admin/domain_blocks_controller.rb b/app/controllers/admin/domain_blocks_controller.rb
index 7129656da..74a36b79c 100644
--- a/app/controllers/admin/domain_blocks_controller.rb
+++ b/app/controllers/admin/domain_blocks_controller.rb
@@ -2,13 +2,17 @@
 
 module Admin
   class DomainBlocksController < BaseController
-    before_action :set_domain_block, only: [:show, :destroy]
+    before_action :set_domain_block, only: [:show, :destroy, :edit, :update]
 
     def new
       authorize :domain_block, :create?
       @domain_block = DomainBlock.new(domain: params[:_domain])
     end
 
+    def edit
+      authorize :domain_block, :create?
+    end
+
     def create
       authorize :domain_block, :create?
 
@@ -35,6 +39,22 @@ module Admin
       end
     end
 
+    def update
+      authorize :domain_block, :create?
+
+      @domain_block.update(update_params)
+
+      severity_changed = @domain_block.severity_changed?
+
+      if @domain_block.save
+        DomainBlockWorker.perform_async(@domain_block.id, severity_changed)
+        log_action :create, @domain_block
+        redirect_to admin_instances_path(limited: '1'), notice: I18n.t('admin.domain_blocks.created_msg')
+      else
+        render :edit
+      end
+    end
+
     def show
       authorize @domain_block, :show?
     end
@@ -52,8 +72,12 @@ module Admin
       @domain_block = DomainBlock.find(params[:id])
     end
 
+    def update_params
+      params.require(:domain_block).permit(:severity, :reject_media, :reject_reports, :private_comment, :public_comment)
+    end
+
     def resource_params
-      params.require(:domain_block).permit(:domain, :severity, :reject_media, :reject_reports)
+      params.require(:domain_block).permit(:domain, :severity, :reject_media, :reject_reports, :private_comment, :public_comment)
     end
   end
 end
diff --git a/app/controllers/admin/instances_controller.rb b/app/controllers/admin/instances_controller.rb
index d4f201807..b47b18f8e 100644
--- a/app/controllers/admin/instances_controller.rb
+++ b/app/controllers/admin/instances_controller.rb
@@ -21,6 +21,8 @@ module Admin
       @blocks_count    = Block.where(target_account: Account.where(domain: params[:id])).count
       @available       = DeliveryFailureTracker.available?(Account.select(:shared_inbox_url).where(domain: params[:id]).first&.shared_inbox_url)
       @media_storage   = MediaAttachment.where(account: Account.where(domain: params[:id])).sum(:file_file_size)
+      @private_comment = @domain_block&.private_comment
+      @public_comment  = @domain_block&.public_comment
     end
 
     private
diff --git a/app/models/domain_block.rb b/app/models/domain_block.rb
index 25d3b87ef..3f5b9f23e 100644
--- a/app/models/domain_block.rb
+++ b/app/models/domain_block.rb
@@ -3,13 +3,15 @@
 #
 # Table name: domain_blocks
 #
-#  id             :bigint(8)        not null, primary key
-#  domain         :string           default(""), not null
-#  created_at     :datetime         not null
-#  updated_at     :datetime         not null
-#  severity       :integer          default("silence")
-#  reject_media   :boolean          default(FALSE), not null
-#  reject_reports :boolean          default(FALSE), not null
+#  id              :bigint(8)        not null, primary key
+#  domain          :string           default(""), not null
+#  created_at      :datetime         not null
+#  updated_at      :datetime         not null
+#  severity        :integer          default("silence")
+#  reject_media    :boolean          default(FALSE), not null
+#  reject_reports  :boolean          default(FALSE), not null
+#  private_comment :text
+#  public_comment  :text
 #
 
 class DomainBlock < ApplicationRecord
diff --git a/app/services/block_domain_service.rb b/app/services/block_domain_service.rb
index c5e5e5761..0ec6be503 100644
--- a/app/services/block_domain_service.rb
+++ b/app/services/block_domain_service.rb
@@ -3,13 +3,22 @@
 class BlockDomainService < BaseService
   attr_reader :domain_block
 
-  def call(domain_block)
+  def call(domain_block, update = false)
     @domain_block = domain_block
     process_domain_block!
+    process_retroactive_updates! if update
   end
 
   private
 
+  def process_retroactive_updates!
+    # If the domain block severity has been changed, undo the appropriate limitations
+    scope = Account.by_domain_and_subdomains(domain_block.domain)
+
+    scope.where(silenced_at: domain_block.created_at).in_batches.update_all(silenced_at: nil) unless domain_block.silence?
+    scope.where(suspended_at: domain_block.created_at).in_batches.update_all(suspended_at: nil) unless domain_block.suspend?
+  end
+
   def process_domain_block!
     clear_media! if domain_block.reject_media?
 
diff --git a/app/services/unblock_domain_service.rb b/app/services/unblock_domain_service.rb
index fc262a50a..d502d9e49 100644
--- a/app/services/unblock_domain_service.rb
+++ b/app/services/unblock_domain_service.rb
@@ -10,24 +10,9 @@ class UnblockDomainService < BaseService
   end
 
   def process_retroactive_updates
-    blocked_accounts.in_batches.update_all(update_options) unless domain_block.noop?
-  end
-
-  def blocked_accounts
     scope = Account.by_domain_and_subdomains(domain_block.domain)
 
-    if domain_block.silence?
-      scope.where(silenced_at: @domain_block.created_at)
-    else
-      scope.where(suspended_at: @domain_block.created_at)
-    end
-  end
-
-  def update_options
-    { domain_block_impact => nil }
-  end
-
-  def domain_block_impact
-    domain_block.silence? ? :silenced_at : :suspended_at
+    scope.where(silenced_at: domain_block.created_at).in_batches.update_all(silenced_at: nil) unless domain_block.noop?
+    scope.where(suspended_at: domain_block.created_at).in_batches.update_all(suspended_at: nil) if domain_block.suspend?
   end
 end
diff --git a/app/views/admin/accounts/show.html.haml b/app/views/admin/accounts/show.html.haml
index 7494c9fa2..59babd3b0 100644
--- a/app/views/admin/accounts/show.html.haml
+++ b/app/views/admin/accounts/show.html.haml
@@ -174,7 +174,7 @@
 
       - unless @account.local?
         - if DomainBlock.where(domain: @account.domain).exists?
-          = link_to t('admin.domain_blocks.undo'), admin_instance_path(@account.domain), class: 'button'
+          = link_to t('admin.domain_blocks.view'), admin_instance_path(@account.domain), class: 'button'
         - else
           = link_to t('admin.domain_blocks.add_new'), new_admin_domain_block_path(_domain: @account.domain), class: 'button button--destructive'
 
diff --git a/app/views/admin/domain_blocks/edit.html.haml b/app/views/admin/domain_blocks/edit.html.haml
new file mode 100644
index 000000000..29e47ef3b
--- /dev/null
+++ b/app/views/admin/domain_blocks/edit.html.haml
@@ -0,0 +1,30 @@
+- content_for :header_tags do
+  = javascript_pack_tag 'admin', integrity: true, async: true, crossorigin: 'anonymous'
+
+- content_for :page_title do
+  = t('admin.domain_blocks.edit')
+
+= simple_form_for @domain_block, url: admin_domain_block_path(@domain_block), method: :put do |f|
+  = render 'shared/error_messages', object: @domain_block
+
+  .fields-row
+    .fields-row__column.fields-row__column-6.fields-group
+      = f.input :domain, wrapper: :with_label, label: t('admin.domain_blocks.domain'), hint: t('admin.domain_blocks.new.hint'), required: true, readonly: true, disabled: true
+
+    .fields-row__column.fields-row__column-6.fields-group
+      = f.input :severity, collection: DomainBlock.severities.keys, wrapper: :with_label, include_blank: false, label_method: lambda { |type| t("admin.domain_blocks.new.severity.#{type}") }, hint: t('admin.domain_blocks.new.severity.desc_html')
+
+  .fields-group
+    = f.input :reject_media, as: :boolean, wrapper: :with_label, label: I18n.t('admin.domain_blocks.reject_media'), hint: I18n.t('admin.domain_blocks.reject_media_hint')
+
+  .fields-group
+    = f.input :reject_reports, as: :boolean, wrapper: :with_label, label: I18n.t('admin.domain_blocks.reject_reports'), hint: I18n.t('admin.domain_blocks.reject_reports_hint')
+
+  .field-group
+    = f.input :private_comment, wrapper: :with_label, label: I18n.t('admin.domain_blocks.private_comment'), hint: t('admin.domain_blocks.private_comment_hint'), rows: 6
+
+  .field-group
+    = f.input :public_comment, wrapper: :with_label, label: I18n.t('admin.domain_blocks.public_comment'), hint: t('admin.domain_blocks.public_comment_hint'), rows: 6
+
+  .actions
+    = f.button :button, t('generic.save_changes'), type: :submit
diff --git a/app/views/admin/domain_blocks/new.html.haml b/app/views/admin/domain_blocks/new.html.haml
index 055d2fbd7..ed1581936 100644
--- a/app/views/admin/domain_blocks/new.html.haml
+++ b/app/views/admin/domain_blocks/new.html.haml
@@ -20,5 +20,11 @@
   .fields-group
     = f.input :reject_reports, as: :boolean, wrapper: :with_label, label: I18n.t('admin.domain_blocks.reject_reports'), hint: I18n.t('admin.domain_blocks.reject_reports_hint')
 
+  .field-group
+    = f.input :private_comment, wrapper: :with_label, label: I18n.t('admin.domain_blocks.private_comment'), hint: t('admin.domain_blocks.private_comment_hint'), rows: 6
+
+  .field-group
+    = f.input :public_comment, wrapper: :with_label, label: I18n.t('admin.domain_blocks.public_comment'), hint: t('admin.domain_blocks.public_comment_hint'), rows: 6
+
   .actions
     = f.button :button, t('.create'), type: :submit
diff --git a/app/views/admin/domain_blocks/show.html.haml b/app/views/admin/domain_blocks/show.html.haml
index dca4dbac7..e64aaa629 100644
--- a/app/views/admin/domain_blocks/show.html.haml
+++ b/app/views/admin/domain_blocks/show.html.haml
@@ -1,6 +1,18 @@
 - content_for :page_title do
   = t('admin.domain_blocks.show.title', domain: @domain_block.domain)
 
+- if @domain_block.private_comment.present?
+  .speech-bubble
+    .speech-bubble__bubble
+      = simple_format(h(@domain_block.private_comment))
+    .speech-bubble__owner= t 'admin.instances.private_comment'
+
+- if @domain_block.public_comment.present?
+  .speech-bubble
+    .speech-bubble__bubble
+      = simple_format(h(@domain_block.public_comment))
+    .speech-bubble__owner= t 'admin.instances.public_comment'
+
 = simple_form_for @domain_block, url: admin_domain_block_path(@domain_block), method: :delete do |f|
 
   - unless (@domain_block.noop?)
diff --git a/app/views/admin/instances/show.html.haml b/app/views/admin/instances/show.html.haml
index fbb49ba02..294c9495d 100644
--- a/app/views/admin/instances/show.html.haml
+++ b/app/views/admin/instances/show.html.haml
@@ -31,6 +31,18 @@
           = fa_icon 'times'
       .dashboard__counters__label= t 'admin.instances.delivery_available'
 
+- if @private_comment.present?
+  .speech-bubble
+    .speech-bubble__bubble
+      = simple_format(h(@private_comment))
+    .speech-bubble__owner= t 'admin.instances.private_comment'
+
+- if @public_comment.present?
+  .speech-bubble
+    .speech-bubble__bubble
+      = simple_format(h(@public_comment))
+    .speech-bubble__owner= t 'admin.instances.public_comment'
+
 %hr.spacer/
 
 %div{ style: 'overflow: hidden' }
@@ -41,6 +53,7 @@
     - if @domain_allow
       = link_to t('admin.domain_allows.undo'), admin_domain_allow_path(@domain_allow), class: 'button button--destructive', data: { confirm: t('admin.accounts.are_you_sure'), method: :delete }
     - elsif @domain_block
+      = link_to t('admin.domain_blocks.edit'), edit_admin_domain_block_path(@domain_block), class: 'button'
       = link_to t('admin.domain_blocks.undo'), admin_domain_block_path(@domain_block), class: 'button'
     - else
       = link_to t('admin.domain_blocks.add_new'), new_admin_domain_block_path(_domain: @instance.domain), class: 'button'
diff --git a/app/workers/domain_block_worker.rb b/app/workers/domain_block_worker.rb
index 884477829..35518d6b5 100644
--- a/app/workers/domain_block_worker.rb
+++ b/app/workers/domain_block_worker.rb
@@ -3,8 +3,8 @@
 class DomainBlockWorker
   include Sidekiq::Worker
 
-  def perform(domain_block_id)
-    BlockDomainService.new.call(DomainBlock.find(domain_block_id))
+  def perform(domain_block_id, update = false)
+    BlockDomainService.new.call(DomainBlock.find(domain_block_id), update)
   rescue ActiveRecord::RecordNotFound
     true
   end
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 17ff24726..b677a6651 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -284,6 +284,7 @@ en:
       created_msg: Domain block is now being processed
       destroyed_msg: Domain block has been undone
       domain: Domain
+      edit: Edit domain block
       existing_domain_block_html: You have already imposed stricter limits on %{name}, you need to <a href="%{unblock_url}">unblock it</a> first.
       new:
         create: Create block
@@ -294,6 +295,10 @@ en:
           silence: Silence
           suspend: Suspend
         title: New domain block
+      private_comment: Private comment
+      private_comment_hint: Comment about this domain limitation for internal use by the moderators.
+      public_comment: Public comment
+      public_comment_hint: Comment about this domain limitation for the general public, if advertising the list of domain limitations is enabled.
       reject_media: Reject media files
       reject_media_hint: Removes locally stored media files and refuses to download any in the future. Irrelevant for suspensions
       reject_reports: Reject reports
@@ -313,6 +318,7 @@ en:
         title: Undo domain block for %{domain}
         undo: Undo
       undo: Undo domain block
+      view: View domain block
     email_domain_blocks:
       add_new: Add new
       created_msg: Successfully added e-mail domain to blacklist
@@ -336,6 +342,8 @@ en:
         all: All
         limited: Limited
         title: Moderation
+      private_comment: Private comment
+      public_comment: Public comment
       title: Federation
       total_blocked_by_us: Blocked by us
       total_followed_by_them: Followed by them
diff --git a/config/routes.rb b/config/routes.rb
index 60f7d2e05..9c33b8190 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -155,7 +155,11 @@ Rails.application.routes.draw do
     get '/dashboard', to: 'dashboard#index'
 
     resources :domain_allows, only: [:new, :create, :show, :destroy]
-    resources :domain_blocks, only: [:new, :create, :show, :destroy]
+    resources :domain_blocks, only: [:new, :create, :show, :destroy, :update] do
+      member do
+        get :edit
+      end
+    end
     resources :email_domain_blocks, only: [:index, :new, :create, :destroy]
     resources :action_logs, only: [:index]
     resources :warning_presets, except: [:new]
diff --git a/db/migrate/20190807135426_add_comments_to_domain_blocks.rb b/db/migrate/20190807135426_add_comments_to_domain_blocks.rb
new file mode 100644
index 000000000..b660a71ad
--- /dev/null
+++ b/db/migrate/20190807135426_add_comments_to_domain_blocks.rb
@@ -0,0 +1,7 @@
+class AddCommentsToDomainBlocks < ActiveRecord::Migration[5.2]
+  def change
+    add_column :domain_blocks, :private_comment, :text
+    add_column :domain_blocks, :public_comment, :text
+  end
+end
+
diff --git a/db/schema.rb b/db/schema.rb
index d1b6825b4..f8fc6a821 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_08_05_123746) do
+ActiveRecord::Schema.define(version: 2019_08_07_135426) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -259,6 +259,8 @@ ActiveRecord::Schema.define(version: 2019_08_05_123746) do
     t.integer "severity", default: 0
     t.boolean "reject_media", default: false, null: false
     t.boolean "reject_reports", default: false, null: false
+    t.text "private_comment"
+    t.text "public_comment"
     t.index ["domain"], name: "index_domain_blocks_on_domain", unique: true
   end
 
diff --git a/spec/services/unblock_domain_service_spec.rb b/spec/services/unblock_domain_service_spec.rb
index 619aefb5c..27dbc92ad 100644
--- a/spec/services/unblock_domain_service_spec.rb
+++ b/spec/services/unblock_domain_service_spec.rb
@@ -31,7 +31,7 @@ describe UnblockDomainService, type: :service do
       subject.call(@domain_block)
       expect_deleted_domain_block
       expect(@suspended.reload.suspended?).to be false
-      expect(@silenced.reload.silenced?).to be true
+      expect(@silenced.reload.silenced?).to be false
       expect(@independently_suspended.reload.suspended?).to be true
       expect(@independently_silenced.reload.silenced?).to be true
     end
diff --git a/spec/workers/domain_block_worker_spec.rb b/spec/workers/domain_block_worker_spec.rb
index c4138501f..48b3e38c4 100644
--- a/spec/workers/domain_block_worker_spec.rb
+++ b/spec/workers/domain_block_worker_spec.rb
@@ -14,7 +14,7 @@ describe DomainBlockWorker do
       result = subject.perform(domain_block.id)
 
       expect(result).to be_nil
-      expect(service).to have_received(:call).with(domain_block)
+      expect(service).to have_received(:call).with(domain_block, false)
     end
 
     it 'calls domain block service for relevant domain block' do
-- 
cgit 


From cc0a55cf9aead00e1cb044649f84c2187e0e4a35 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sun, 18 Aug 2019 03:45:51 +0200
Subject: Add more accurate hashtag search (#11579)

* Add more accurate hashtag search

Using ElasticSearch to index hashtags with edge n-grams and score
them by usage within the last 7 days since last activity. Only
hashtags that have been reviewed and are listable can appear in
searches, unless they match the query exactly

* Fix search analyzer dropping non-ascii characters
---
 app/chewy/tags_index.rb                            | 37 ++++++++++
 app/models/tag.rb                                  | 14 ++--
 app/models/trending_tags.rb                        |  3 +
 app/services/account_search_service.rb             |  2 +-
 app/services/search_service.rb                     |  8 +--
 app/services/tag_search_service.rb                 | 82 ++++++++++++++++++++++
 config/locales/simple_form.en.yml                  |  2 +-
 .../20190815225426_add_last_status_at_to_tags.rb   |  6 ++
 db/schema.rb                                       |  4 +-
 spec/models/tag_spec.rb                            |  4 +-
 10 files changed, 149 insertions(+), 13 deletions(-)
 create mode 100644 app/chewy/tags_index.rb
 create mode 100644 app/services/tag_search_service.rb
 create mode 100644 db/migrate/20190815225426_add_last_status_at_to_tags.rb

(limited to 'db/migrate')

diff --git a/app/chewy/tags_index.rb b/app/chewy/tags_index.rb
new file mode 100644
index 000000000..300fc128f
--- /dev/null
+++ b/app/chewy/tags_index.rb
@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+class TagsIndex < Chewy::Index
+  settings index: { refresh_interval: '15m' }, analysis: {
+    analyzer: {
+      content: {
+        tokenizer: 'keyword',
+        filter: %w(lowercase asciifolding cjk_width),
+      },
+
+      edge_ngram: {
+        tokenizer: 'edge_ngram',
+        filter: %w(lowercase asciifolding cjk_width),
+      },
+    },
+
+    tokenizer: {
+      edge_ngram: {
+        type: 'edge_ngram',
+        min_gram: 2,
+        max_gram: 15,
+      },
+    },
+  }
+
+  define_type ::Tag.listable, delete_if: ->(tag) { tag.destroyed? || !tag.listable? } do
+    root date_detection: false do
+      field :name, type: 'text', analyzer: 'content' do
+        field :edge_ngram, type: 'text', analyzer: 'edge_ngram', search_analyzer: 'content'
+      end
+
+      field :reviewed, type: 'boolean', value: ->(tag) { tag.reviewed? }
+      field :usage, type: 'long', value: ->(tag) { tag.history.reduce(0) { |total, day| total + day[:accounts].to_i } }
+      field :last_status_at, type: 'date', value: ->(tag) { tag.last_status_at || tag.created_at }
+    end
+  end
+end
diff --git a/app/models/tag.rb b/app/models/tag.rb
index 1364d1dba..5094d973d 100644
--- a/app/models/tag.rb
+++ b/app/models/tag.rb
@@ -13,6 +13,8 @@
 #  listable            :boolean
 #  reviewed_at         :datetime
 #  requested_review_at :datetime
+#  last_status_at      :datetime
+#  last_trend_at       :datetime
 #
 
 class Tag < ApplicationRecord
@@ -33,7 +35,8 @@ class Tag < ApplicationRecord
   scope :unreviewed, -> { where(reviewed_at: nil) }
   scope :pending_review, -> { unreviewed.where.not(requested_review_at: nil) }
   scope :usable, -> { where(usable: [true, nil]) }
-  scope :discoverable, -> { where(listable: [true, nil]).joins(:account_tag_stat).where(AccountTagStat.arel_table[:accounts_count].gt(0)).order(Arel.sql('account_tag_stats.accounts_count desc')) }
+  scope :listable, -> { where(listable: [true, nil]) }
+  scope :discoverable, -> { listable.joins(:account_tag_stat).where(AccountTagStat.arel_table[:accounts_count].gt(0)).order(Arel.sql('account_tag_stats.accounts_count desc')) }
   scope :most_used, ->(account) { joins(:statuses).where(statuses: { account: account }).group(:id).order(Arel.sql('count(*) desc')) }
 
   delegate :accounts_count,
@@ -44,6 +47,8 @@ class Tag < ApplicationRecord
 
   after_save :save_account_tag_stat
 
+  update_index('tags#tag', :self) if Chewy.enabled?
+
   def account_tag_stat
     super || build_account_tag_stat
   end
@@ -121,9 +126,10 @@ class Tag < ApplicationRecord
       normalized_term = normalize(term.strip).mb_chars.downcase.to_s
       pattern         = sanitize_sql_like(normalized_term) + '%'
 
-      Tag.where(arel_table[:name].lower.matches(pattern))
-         .where(arel_table[:score].gt(0).or(arel_table[:name].lower.eq(normalized_term)))
-         .order(Arel.sql('length(name) ASC, score DESC, name ASC'))
+      Tag.listable
+         .where(arel_table[:name].lower.matches(pattern))
+         .where(arel_table[:name].lower.eq(normalized_term).or(arel_table[:reviewed_at].not_eq(nil)))
+         .order(Arel.sql('length(name) ASC, name ASC'))
          .limit(limit)
          .offset(offset)
     end
diff --git a/app/models/trending_tags.rb b/app/models/trending_tags.rb
index 3d60a7fea..e4ce988c1 100644
--- a/app/models/trending_tags.rb
+++ b/app/models/trending_tags.rb
@@ -17,6 +17,9 @@ class TrendingTags
       increment_historical_use!(tag.id, at_time)
       increment_unique_use!(tag.id, account.id, at_time)
       increment_vote!(tag, at_time)
+
+      tag.update(last_status_at: Time.now.utc) if tag.last_status_at.nil? || tag.last_status_at < 12.hours.ago
+      tag.update(last_trend_at: Time.now.utc)  if trending?(tag) && (tag.last_trend_at.nil? || tag.last_trend_at < 12.hours.ago)
     end
 
     def get(limit, filtered: true)
diff --git a/app/services/account_search_service.rb b/app/services/account_search_service.rb
index d7bccdfe0..01caaefa9 100644
--- a/app/services/account_search_service.rb
+++ b/app/services/account_search_service.rb
@@ -109,7 +109,7 @@ class AccountSearchService < BaseService
       field_value_factor: {
         field: 'followers_count',
         modifier: 'log2p',
-        missing: 1,
+        missing: 0,
       },
     }
   end
diff --git a/app/services/search_service.rb b/app/services/search_service.rb
index 786d34b15..fe601bbf4 100644
--- a/app/services/search_service.rb
+++ b/app/services/search_service.rb
@@ -57,10 +57,10 @@ class SearchService < BaseService
   end
 
   def perform_hashtags_search!
-    Tag.search_for(
-      @query.gsub(/\A#/, ''),
-      @limit,
-      @offset
+    TagSearchService.new.call(
+      @query,
+      limit: @limit,
+      offset: @offset
     )
   end
 
diff --git a/app/services/tag_search_service.rb b/app/services/tag_search_service.rb
new file mode 100644
index 000000000..64dd76bb7
--- /dev/null
+++ b/app/services/tag_search_service.rb
@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+class TagSearchService < BaseService
+  def call(query, options = {})
+    @query  = query.strip.gsub(/\A#/, '')
+    @offset = options[:offset].to_i
+    @limit  = options[:limit].to_i
+
+    if Chewy.enabled?
+      from_elasticsearch
+    else
+      from_database
+    end
+  end
+
+  private
+
+  def from_elasticsearch
+    query = {
+      function_score: {
+        query: {
+          multi_match: {
+            query: @query,
+            fields: %w(name.edge_ngram name),
+            type: 'most_fields',
+            operator: 'and',
+          },
+        },
+
+        functions: [
+          {
+            field_value_factor: {
+              field: 'usage',
+              modifier: 'log2p',
+              missing: 0,
+            },
+          },
+
+          {
+            gauss: {
+              last_status_at: {
+                scale: '7d',
+                offset: '14d',
+                decay: 0.5,
+              },
+            },
+          },
+        ],
+
+        boost_mode: 'multiply',
+      },
+    }
+
+    filter = {
+      bool: {
+        should: [
+          {
+            term: {
+              reviewed: {
+                value: true,
+              },
+            },
+          },
+
+          {
+            term: {
+              name: {
+                value: @query,
+              },
+            },
+          },
+        ],
+      },
+    }
+
+    TagsIndex.query(query).filter(filter).limit(@limit).offset(@offset).objects.compact
+  end
+
+  def from_database
+    Tag.search_for(@query, @limit, @offset)
+  end
+end
diff --git a/config/locales/simple_form.en.yml b/config/locales/simple_form.en.yml
index e15d5904f..98f0843d0 100644
--- a/config/locales/simple_form.en.yml
+++ b/config/locales/simple_form.en.yml
@@ -142,7 +142,7 @@ en:
         report: Send e-mail when a new report is submitted
         trending_tag: Send e-mail when an unreviewed hashtag is trending
       tag:
-        listable: Allow this hashtag to appear on the profile directory
+        listable: Allow this hashtag to appear in searches and on the profile directory
         trendable: Allow this hashtag to appear under trends
         usable: Allow toots to use this hashtag
     'no': 'No'
diff --git a/db/migrate/20190815225426_add_last_status_at_to_tags.rb b/db/migrate/20190815225426_add_last_status_at_to_tags.rb
new file mode 100644
index 000000000..d83537c47
--- /dev/null
+++ b/db/migrate/20190815225426_add_last_status_at_to_tags.rb
@@ -0,0 +1,6 @@
+class AddLastStatusAtToTags < ActiveRecord::Migration[5.2]
+  def change
+    add_column :tags, :last_status_at, :datetime
+    add_column :tags, :last_trend_at, :datetime
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index f8fc6a821..0da20d62f 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_08_07_135426) do
+ActiveRecord::Schema.define(version: 2019_08_15_225426) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -667,6 +667,8 @@ ActiveRecord::Schema.define(version: 2019_08_07_135426) do
     t.boolean "listable"
     t.datetime "reviewed_at"
     t.datetime "requested_review_at"
+    t.datetime "last_status_at"
+    t.datetime "last_trend_at"
     t.index "lower((name)::text)", name: "index_tags_on_name_lower", unique: true
   end
 
diff --git a/spec/models/tag_spec.rb b/spec/models/tag_spec.rb
index 9d700849b..2bb30fb57 100644
--- a/spec/models/tag_spec.rb
+++ b/spec/models/tag_spec.rb
@@ -136,8 +136,8 @@ RSpec.describe Tag, type: :model do
     end
 
     it 'finds the exact matching tag as the first item' do
-      similar_tag = Fabricate(:tag, name: "matchlater", score: 1)
-      tag = Fabricate(:tag, name: "match", score: 1)
+      similar_tag = Fabricate(:tag, name: "matchlater", reviewed_at: Time.now.utc)
+      tag = Fabricate(:tag, name: "match", reviewed_at: Time.now.utc)
 
       results = Tag.search_for("match")
 
-- 
cgit 


From cb62a83a71a9679061f8daa468119124b5da5e76 Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Mon, 19 Aug 2019 11:40:42 +0200
Subject: Add invite comments (#10465)

---
 app/controllers/invites_controller.rb               | 2 +-
 app/models/invite.rb                                | 3 +++
 app/views/invites/_form.html.haml                   | 3 +++
 app/views/invites/_invite.html.haml                 | 3 +++
 app/views/invites/index.html.haml                   | 1 +
 db/migrate/20190403141604_add_comment_to_invites.rb | 5 +++++
 db/schema.rb                                        | 1 +
 7 files changed, 17 insertions(+), 1 deletion(-)
 create mode 100644 db/migrate/20190403141604_add_comment_to_invites.rb

(limited to 'db/migrate')

diff --git a/app/controllers/invites_controller.rb b/app/controllers/invites_controller.rb
index de5280305..8d92147e2 100644
--- a/app/controllers/invites_controller.rb
+++ b/app/controllers/invites_controller.rb
@@ -43,7 +43,7 @@ class InvitesController < ApplicationController
   end
 
   def resource_params
-    params.require(:invite).permit(:max_uses, :expires_in, :autofollow)
+    params.require(:invite).permit(:max_uses, :expires_in, :autofollow, :comment)
   end
 
   def set_body_classes
diff --git a/app/models/invite.rb b/app/models/invite.rb
index 02ab8e0b2..29d25eae8 100644
--- a/app/models/invite.rb
+++ b/app/models/invite.rb
@@ -12,6 +12,7 @@
 #  created_at :datetime         not null
 #  updated_at :datetime         not null
 #  autofollow :boolean          default(FALSE), not null
+#  comment    :text
 #
 
 class Invite < ApplicationRecord
@@ -22,6 +23,8 @@ class Invite < ApplicationRecord
 
   scope :available, -> { where(expires_at: nil).or(where('expires_at >= ?', Time.now.utc)) }
 
+  validates :comment, length: { maximum: 420 }
+
   before_validation :set_code
 
   def valid_for_use?
diff --git a/app/views/invites/_form.html.haml b/app/views/invites/_form.html.haml
index 3a2a5ef0e..b19f70539 100644
--- a/app/views/invites/_form.html.haml
+++ b/app/views/invites/_form.html.haml
@@ -10,5 +10,8 @@
   .fields-group
     = f.input :autofollow, wrapper: :with_label
 
+  .fields-group
+    = f.input :comment, wrapper: :with_label, input_html: { maxlength: 420 }
+
   .actions
     = f.button :button, t('invites.generate'), type: :submit
diff --git a/app/views/invites/_invite.html.haml b/app/views/invites/_invite.html.haml
index 62799ca5b..03050c868 100644
--- a/app/views/invites/_invite.html.haml
+++ b/app/views/invites/_invite.html.haml
@@ -20,6 +20,9 @@
     %td{ colspan: 2 }
       = t('invites.expired')
 
+  %td
+    = invite.comment
+
   %td
     - if invite.valid_for_use? && policy(invite).destroy?
       = table_link_to 'times', t('invites.delete'), invite_path(invite), method: :delete
diff --git a/app/views/invites/index.html.haml b/app/views/invites/index.html.haml
index 61420ab1e..62065d6ae 100644
--- a/app/views/invites/index.html.haml
+++ b/app/views/invites/index.html.haml
@@ -15,6 +15,7 @@
         %th
         %th= t('invites.table.uses')
         %th= t('invites.table.expires_at')
+        %th= t('invites.table.comment')
         %th
     %tbody
       = render @invites
diff --git a/db/migrate/20190403141604_add_comment_to_invites.rb b/db/migrate/20190403141604_add_comment_to_invites.rb
new file mode 100644
index 000000000..f0d7b1dcd
--- /dev/null
+++ b/db/migrate/20190403141604_add_comment_to_invites.rb
@@ -0,0 +1,5 @@
+class AddCommentToInvites < ActiveRecord::Migration[5.2]
+  def change
+    add_column :invites, :comment, :text
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 0da20d62f..18f615d61 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -344,6 +344,7 @@ ActiveRecord::Schema.define(version: 2019_08_15_225426) do
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
     t.boolean "autofollow", default: false, null: false
+    t.text "comment"
     t.index ["code"], name: "index_invites_on_code", unique: true
     t.index ["user_id"], name: "index_invites_on_user_id"
   end
-- 
cgit 


From 282ea170782e4ce1ed5251a1b94857a512412397 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Thu, 22 Aug 2019 21:55:56 +0200
Subject: Add soft delete for statuses for instant deletes through API (#11623)

* Add soft delete for statuses to allow them to appear instant

* Allow reporting soft-deleted statuses and show them in the admin UI

* Change index for getting an account's statuses
---
 Gemfile                                                 |  1 +
 Gemfile.lock                                            |  3 +++
 app/controllers/api/v1/reports_controller.rb            |  2 +-
 app/controllers/api/v1/statuses/reblogs_controller.rb   |  3 ++-
 app/controllers/api/v1/statuses_controller.rb           |  1 +
 app/models/form/status_batch.rb                         |  1 +
 app/models/report.rb                                    |  2 +-
 app/models/status.rb                                    |  6 +++++-
 app/views/admin/reports/_status.html.haml               |  5 ++++-
 app/workers/removal_worker.rb                           |  2 +-
 config/locales/en.yml                                   |  1 +
 db/migrate/20190819134503_add_deleted_at_to_statuses.rb |  5 +++++
 db/migrate/20190820003045_update_statuses_index.rb      | 13 +++++++++++++
 db/schema.rb                                            |  5 +++--
 14 files changed, 42 insertions(+), 8 deletions(-)
 create mode 100644 db/migrate/20190819134503_add_deleted_at_to_statuses.rb
 create mode 100644 db/migrate/20190820003045_update_statuses_index.rb

(limited to 'db/migrate')

diff --git a/Gemfile b/Gemfile
index 250a28a3a..86dab965a 100644
--- a/Gemfile
+++ b/Gemfile
@@ -43,6 +43,7 @@ gem 'omniauth-cas', '~> 1.1'
 gem 'omniauth-saml', '~> 1.10'
 gem 'omniauth', '~> 1.9'
 
+gem 'discard', '~> 1.1'
 gem 'doorkeeper', '~> 5.1'
 gem 'fast_blank', '~> 1.0'
 gem 'fastimage'
diff --git a/Gemfile.lock b/Gemfile.lock
index 1da6d73a6..b896909a4 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -204,6 +204,8 @@ GEM
       devise (>= 4.0.0)
       rpam2 (~> 4.0)
     diff-lcs (1.3)
+    discard (1.1.0)
+      activerecord (>= 4.2, < 7)
     docile (1.3.2)
     domain_name (0.5.20180417)
       unf (>= 0.0.5, < 1.0.0)
@@ -692,6 +694,7 @@ DEPENDENCIES
   devise (~> 4.6)
   devise-two-factor (~> 3.1)
   devise_pam_authenticatable2 (~> 9.2)
+  discard (~> 1.1)
   doorkeeper (~> 5.1)
   dotenv-rails (~> 2.7)
   fabrication (~> 2.20)
diff --git a/app/controllers/api/v1/reports_controller.rb b/app/controllers/api/v1/reports_controller.rb
index e182a9c6c..1b0b4b05b 100644
--- a/app/controllers/api/v1/reports_controller.rb
+++ b/app/controllers/api/v1/reports_controller.rb
@@ -21,7 +21,7 @@ class Api::V1::ReportsController < Api::BaseController
   private
 
   def reported_status_ids
-    reported_account.statuses.find(status_ids).pluck(:id)
+    reported_account.statuses.with_discarded.find(status_ids).pluck(:id)
   end
 
   def status_ids
diff --git a/app/controllers/api/v1/statuses/reblogs_controller.rb b/app/controllers/api/v1/statuses/reblogs_controller.rb
index ed4f55100..42381a37f 100644
--- a/app/controllers/api/v1/statuses/reblogs_controller.rb
+++ b/app/controllers/api/v1/statuses/reblogs_controller.rb
@@ -18,6 +18,7 @@ class Api::V1::Statuses::ReblogsController < Api::BaseController
     @reblogs_map = { @status.id => false }
 
     authorize status_for_destroy, :unreblog?
+    status_for_destroy.discard
     RemovalWorker.perform_async(status_for_destroy.id)
 
     render json: @status, serializer: REST::StatusSerializer, relationships: StatusRelationshipsPresenter.new([@status], current_user&.account_id, reblogs_map: @reblogs_map)
@@ -30,7 +31,7 @@ class Api::V1::Statuses::ReblogsController < Api::BaseController
   end
 
   def status_for_destroy
-    current_user.account.statuses.where(reblog_of_id: params[:status_id]).first!
+    @status_for_destroy ||= current_user.account.statuses.where(reblog_of_id: params[:status_id]).first!
   end
 
   def reblog_params
diff --git a/app/controllers/api/v1/statuses_controller.rb b/app/controllers/api/v1/statuses_controller.rb
index 39ca56482..bba3c0651 100644
--- a/app/controllers/api/v1/statuses_controller.rb
+++ b/app/controllers/api/v1/statuses_controller.rb
@@ -53,6 +53,7 @@ class Api::V1::StatusesController < Api::BaseController
     @status = Status.where(account_id: current_user.account).find(params[:id])
     authorize @status, :destroy?
 
+    @status.discard
     RemovalWorker.perform_async(@status.id, redraft: true)
 
     render json: @status, serializer: REST::StatusSerializer, source_requested: true
diff --git a/app/models/form/status_batch.rb b/app/models/form/status_batch.rb
index 831d8b7c5..e09cc2594 100644
--- a/app/models/form/status_batch.rb
+++ b/app/models/form/status_batch.rb
@@ -34,6 +34,7 @@ class Form::StatusBatch
 
   def delete_statuses
     Status.where(id: status_ids).reorder(nil).find_each do |status|
+      status.discard
       RemovalWorker.perform_async(status.id, redraft: false)
       Tombstone.find_or_create_by(uri: status.uri, account: status.account, by_moderator: true)
       log_action :destroy, status
diff --git a/app/models/report.rb b/app/models/report.rb
index 5192ceef7..1e707ff1c 100644
--- a/app/models/report.rb
+++ b/app/models/report.rb
@@ -43,7 +43,7 @@ class Report < ApplicationRecord
   end
 
   def statuses
-    Status.where(id: status_ids).includes(:account, :media_attachments, :mentions)
+    Status.with_discarded.where(id: status_ids).includes(:account, :media_attachments, :mentions)
   end
 
   def media_attachments
diff --git a/app/models/status.rb b/app/models/status.rb
index 0538c4e9e..9cfaddcec 100644
--- a/app/models/status.rb
+++ b/app/models/status.rb
@@ -22,15 +22,19 @@
 #  application_id         :bigint(8)
 #  in_reply_to_account_id :bigint(8)
 #  poll_id                :bigint(8)
+#  deleted_at             :datetime
 #
 
 class Status < ApplicationRecord
   before_destroy :unlink_from_conversations
 
+  include Discard::Model
   include Paginable
   include Cacheable
   include StatusThreadingConcern
 
+  self.discard_column = :deleted_at
+
   # If `override_timestamps` is set at creation time, Snowflake ID creation
   # will be based on current time instead of `created_at`
   attr_accessor :override_timestamps
@@ -72,7 +76,7 @@ class Status < ApplicationRecord
 
   accepts_nested_attributes_for :poll
 
-  default_scope { recent }
+  default_scope { recent.kept }
 
   scope :recent, -> { reorder(id: :desc) }
   scope :remote, -> { where(local: false).where.not(uri: nil) }
diff --git a/app/views/admin/reports/_status.html.haml b/app/views/admin/reports/_status.html.haml
index 9376db7ff..6facc0a56 100644
--- a/app/views/admin/reports/_status.html.haml
+++ b/app/views/admin/reports/_status.html.haml
@@ -16,11 +16,14 @@
         - video = status.proper.media_attachments.first
         = react_component :video, src: video.file.url(:original), preview: video.file.url(:small), sensitive: !current_account&.user&.show_all_media? && status.proper.sensitive? || current_account&.user&.hide_all_media?, width: 610, height: 343, inline: true, alt: video.description
       - else
-        = react_component :media_gallery, height: 343, sensitive: !current_account&.user&.show_all_media? && status.sensitive? || current_account&.user&.hide_all_media?, 'autoPlayGif': current_account&.user&.setting_auto_play_gif, media: status.proper.media_attachments.map { |a| ActiveModelSerializers::SerializableResource.new(a, serializer: REST::MediaAttachmentSerializer).as_json }
+        = react_component :media_gallery, height: 343, sensitive: !current_account&.user&.show_all_media? && status.proper.sensitive? || current_account&.user&.hide_all_media?, 'autoPlayGif': current_account&.user&.setting_auto_play_gif, media: status.proper.media_attachments.map { |a| ActiveModelSerializers::SerializableResource.new(a, serializer: REST::MediaAttachmentSerializer).as_json }
 
     .detailed-status__meta
       = link_to ActivityPub::TagManager.instance.url_for(status), class: 'detailed-status__datetime', target: stream_link_target, rel: 'noopener' do
         %time.formatted{ datetime: status.created_at.iso8601, title: l(status.created_at) }= l(status.created_at)
+      - if status.discarded?
+        ·
+        %span.negative-hint= t('admin.statuses.deleted')
       ·
       - if status.reblog?
         = fa_icon('retweet fw')
diff --git a/app/workers/removal_worker.rb b/app/workers/removal_worker.rb
index 14423a4fb..2a1eaa89b 100644
--- a/app/workers/removal_worker.rb
+++ b/app/workers/removal_worker.rb
@@ -4,7 +4,7 @@ class RemovalWorker
   include Sidekiq::Worker
 
   def perform(status_id, options = {})
-    RemoveStatusService.new.call(Status.find(status_id), **options.symbolize_keys)
+    RemoveStatusService.new.call(Status.with_discarded.find(status_id), **options.symbolize_keys)
   rescue ActiveRecord::RecordNotFound
     true
   end
diff --git a/config/locales/en.yml b/config/locales/en.yml
index 8d267065c..a50dcb8a5 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -499,6 +499,7 @@ en:
         delete: Delete
         nsfw_off: Mark as not sensitive
         nsfw_on: Mark as sensitive
+      deleted: Deleted
       failed_to_execute: Failed to execute
       media:
         title: Media
diff --git a/db/migrate/20190819134503_add_deleted_at_to_statuses.rb b/db/migrate/20190819134503_add_deleted_at_to_statuses.rb
new file mode 100644
index 000000000..5af109097
--- /dev/null
+++ b/db/migrate/20190819134503_add_deleted_at_to_statuses.rb
@@ -0,0 +1,5 @@
+class AddDeletedAtToStatuses < ActiveRecord::Migration[5.2]
+  def change
+    add_column :statuses, :deleted_at, :datetime
+  end
+end
diff --git a/db/migrate/20190820003045_update_statuses_index.rb b/db/migrate/20190820003045_update_statuses_index.rb
new file mode 100644
index 000000000..5c2ea1f6a
--- /dev/null
+++ b/db/migrate/20190820003045_update_statuses_index.rb
@@ -0,0 +1,13 @@
+class UpdateStatusesIndex < ActiveRecord::Migration[5.2]
+  disable_ddl_transaction!
+
+  def up
+    safety_assured { add_index :statuses, [:account_id, :id, :visibility, :updated_at], where: 'deleted_at IS NULL', order: { id: :desc }, algorithm: :concurrently, name: :index_statuses_20190820 }
+    remove_index :statuses, name: :index_statuses_20180106
+  end
+
+  def down
+    safety_assured { add_index :statuses, [:account_id, :id, :visibility, :updated_at], order: { id: :desc }, algorithm: :concurrently, name: :index_statuses_20180106 }
+    remove_index :statuses, name: :index_statuses_20190820
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 18f615d61..afa6d724c 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_08_15_225426) do
+ActiveRecord::Schema.define(version: 2019_08_20_003045) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -644,7 +644,8 @@ ActiveRecord::Schema.define(version: 2019_08_15_225426) do
     t.bigint "application_id"
     t.bigint "in_reply_to_account_id"
     t.bigint "poll_id"
-    t.index ["account_id", "id", "visibility", "updated_at"], name: "index_statuses_20180106", order: { id: :desc }
+    t.datetime "deleted_at"
+    t.index ["account_id", "id", "visibility", "updated_at"], name: "index_statuses_20190820", order: { id: :desc }, where: "(deleted_at IS NULL)"
     t.index ["in_reply_to_account_id"], name: "index_statuses_on_in_reply_to_account_id"
     t.index ["in_reply_to_id"], name: "index_statuses_on_in_reply_to_id"
     t.index ["reblog_of_id", "account_id"], name: "index_statuses_on_reblog_of_id_and_account_id"
-- 
cgit 


From aa6b5b42df7bcde08da6bc4d686a83b4533207df Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sat, 24 Aug 2019 04:12:27 +0200
Subject: Fix slow local timeline query (#11648)

Fix #11643
---
 db/migrate/20190823221802_add_local_index_to_statuses.rb | 11 +++++++++++
 db/schema.rb                                             |  3 ++-
 2 files changed, 13 insertions(+), 1 deletion(-)
 create mode 100644 db/migrate/20190823221802_add_local_index_to_statuses.rb

(limited to 'db/migrate')

diff --git a/db/migrate/20190823221802_add_local_index_to_statuses.rb b/db/migrate/20190823221802_add_local_index_to_statuses.rb
new file mode 100644
index 000000000..deca25c35
--- /dev/null
+++ b/db/migrate/20190823221802_add_local_index_to_statuses.rb
@@ -0,0 +1,11 @@
+class AddLocalIndexToStatuses < ActiveRecord::Migration[5.2]
+  disable_ddl_transaction!
+
+  def up
+    add_index :statuses, [:id, :account_id], name: :index_statuses_local_20190824, algorithm: :concurrently, order: { id: :desc }, where: '(local OR (uri IS NULL)) AND deleted_at IS NULL AND visibility = 0 AND reblog_of_id IS NULL AND ((NOT reply) OR (in_reply_to_account_id = account_id))'
+  end
+
+  def down
+    remove_index :statuses, name: :index_statuses_local_20190824
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index afa6d724c..482bca367 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_08_20_003045) do
+ActiveRecord::Schema.define(version: 2019_08_23_221802) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -646,6 +646,7 @@ ActiveRecord::Schema.define(version: 2019_08_20_003045) do
     t.bigint "poll_id"
     t.datetime "deleted_at"
     t.index ["account_id", "id", "visibility", "updated_at"], name: "index_statuses_20190820", order: { id: :desc }, where: "(deleted_at IS NULL)"
+    t.index ["id", "account_id"], name: "index_statuses_local_20190824", order: { id: :desc }, where: "((local OR (uri IS NULL)) AND (deleted_at IS NULL) AND (visibility = 0) AND (reblog_of_id IS NULL) AND ((NOT reply) OR (in_reply_to_account_id = account_id)))"
     t.index ["in_reply_to_account_id"], name: "index_statuses_on_in_reply_to_account_id"
     t.index ["in_reply_to_id"], name: "index_statuses_on_in_reply_to_id"
     t.index ["reblog_of_id", "account_id"], name: "index_statuses_on_reblog_of_id_and_account_id"
-- 
cgit 


From 70ddef2654a931827ce5e4323e3042365f6078f2 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Mon, 2 Sep 2019 18:11:13 +0200
Subject: Change trending hashtags to not disappear instantly after midnight
 (#11712)

---
 app/controllers/admin/tags_controller.rb           |   2 +-
 app/lib/feed_manager.rb                            |   2 +-
 app/models/tag.rb                                  |   4 +-
 app/models/trending_tags.rb                        | 102 +++++++++++++++------
 app/workers/scheduler/trending_tags_scheduler.rb   |  11 +++
 config/sidekiq.yml                                 |   3 +
 db/migrate/20190901035623_add_max_score_to_tags.rb |   6 ++
 .../20190901040524_remove_score_from_tags.rb       |  12 +++
 db/schema.rb                                       |   6 +-
 spec/models/trending_tags_spec.rb                  |  68 ++++++++++++++
 10 files changed, 179 insertions(+), 37 deletions(-)
 create mode 100644 app/workers/scheduler/trending_tags_scheduler.rb
 create mode 100644 db/migrate/20190901035623_add_max_score_to_tags.rb
 create mode 100644 db/post_migrate/20190901040524_remove_score_from_tags.rb
 create mode 100644 spec/models/trending_tags_spec.rb

(limited to 'db/migrate')

diff --git a/app/controllers/admin/tags_controller.rb b/app/controllers/admin/tags_controller.rb
index 25d9b7d3d..8bd4e5f8b 100644
--- a/app/controllers/admin/tags_controller.rb
+++ b/app/controllers/admin/tags_controller.rb
@@ -57,7 +57,7 @@ module Admin
       scope = scope.unreviewed if filter_params[:review] == 'unreviewed'
       scope = scope.reviewed.order(reviewed_at: :desc) if filter_params[:review] == 'reviewed'
       scope = scope.pending_review.order(requested_review_at: :desc) if filter_params[:review] == 'pending_review'
-      scope.order(score: :desc)
+      scope.order(max_score: :desc)
     end
 
     def filter_params
diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb
index ca3d890a8..871ec5c19 100644
--- a/app/lib/feed_manager.rb
+++ b/app/lib/feed_manager.rb
@@ -63,7 +63,7 @@ class FeedManager
     reblog_key   = key(type, account_id, 'reblogs')
 
     # Remove any items past the MAX_ITEMS'th entry in our feed
-    redis.zremrangebyrank(timeline_key, '0', (-(FeedManager::MAX_ITEMS + 1)).to_s)
+    redis.zremrangebyrank(timeline_key, 0, -(FeedManager::MAX_ITEMS + 1))
 
     # Get the score of the REBLOG_FALLOFF'th item in our feed, and stop
     # tracking anything after it for deduplication purposes.
diff --git a/app/models/tag.rb b/app/models/tag.rb
index 945e3a3c6..135e0a030 100644
--- a/app/models/tag.rb
+++ b/app/models/tag.rb
@@ -7,14 +7,14 @@
 #  name                :string           default(""), not null
 #  created_at          :datetime         not null
 #  updated_at          :datetime         not null
-#  score               :integer
 #  usable              :boolean
 #  trendable           :boolean
 #  listable            :boolean
 #  reviewed_at         :datetime
 #  requested_review_at :datetime
 #  last_status_at      :datetime
-#  last_trend_at       :datetime
+#  max_score           :float
+#  max_score_at        :datetime
 #
 
 class Tag < ApplicationRecord
diff --git a/app/models/trending_tags.rb b/app/models/trending_tags.rb
index e4ce988c1..e1b92b175 100644
--- a/app/models/trending_tags.rb
+++ b/app/models/trending_tags.rb
@@ -7,6 +7,8 @@ class TrendingTags
   THRESHOLD            = 5
   LIMIT                = 10
   REVIEW_THRESHOLD     = 3
+  MAX_SCORE_COOLDOWN   = 3.days.freeze
+  MAX_SCORE_HALFLIFE   = 6.hours.freeze
 
   class << self
     include Redisable
@@ -16,14 +18,75 @@ class TrendingTags
 
       increment_historical_use!(tag.id, at_time)
       increment_unique_use!(tag.id, account.id, at_time)
-      increment_vote!(tag, at_time)
+      increment_use!(tag.id, at_time)
 
       tag.update(last_status_at: Time.now.utc) if tag.last_status_at.nil? || tag.last_status_at < 12.hours.ago
-      tag.update(last_trend_at: Time.now.utc)  if trending?(tag) && (tag.last_trend_at.nil? || tag.last_trend_at < 12.hours.ago)
+    end
+
+    def update!(at_time = Time.now.utc)
+      tag_ids = redis.smembers("#{KEY}:used:#{at_time.beginning_of_day.to_i}") + redis.zrange(KEY, 0, -1)
+      tags    = Tag.where(id: tag_ids.uniq)
+
+      # First pass to calculate scores and update the set
+
+      tags.each do |tag|
+        expected  = redis.pfcount("activity:tags:#{tag.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts").to_f
+        expected  = 1.0 if expected.zero?
+        observed  = redis.pfcount("activity:tags:#{tag.id}:#{at_time.beginning_of_day.to_i}:accounts").to_f
+        max_time  = tag.max_score_at
+        max_score = tag.max_score
+        max_score = 0 if max_time.nil? || max_time < (at_time - MAX_SCORE_COOLDOWN)
+
+        score = begin
+          if expected > observed || observed < THRESHOLD
+            0
+          else
+            ((observed - expected)**2) / expected
+          end
+        end
+
+        if score > max_score
+          max_score = score
+          max_time  = at_time
+
+          # Not interested in triggering any callbacks for this
+          tag.update_columns(max_score: max_score, max_score_at: max_time)
+        end
+
+        decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / MAX_SCORE_HALFLIFE.to_f))
+
+        if decaying_score.zero?
+          redis.zrem(KEY, tag.id)
+        else
+          redis.zadd(KEY, decaying_score, tag.id)
+        end
+      end
+
+      users_for_review = User.staff.includes(:account).to_a.select(&:allows_trending_tag_emails?)
+
+      # Second pass to notify about previously unreviewed trends
+
+      tags.each do |tag|
+        current_rank              = redis.zrevrank(KEY, tag.id)
+        needs_review_notification = tag.requires_review? && !tag.requested_review?
+        rank_passes_threshold     = current_rank.present? && current_rank <= REVIEW_THRESHOLD
+
+        next unless !tag.trendable? && rank_passes_threshold && needs_review_notification
+
+        tag.touch(:requested_review_at)
+
+        users_for_review.each do |user|
+          AdminMailer.new_trending_tag(user.account, tag).deliver_later!
+        end
+      end
+
+      # Trim older items
+
+      redis.zremrangebyrank(KEY, 0, -(LIMIT + 1))
     end
 
     def get(limit, filtered: true)
-      tag_ids = redis.zrevrange("#{KEY}:#{Time.now.utc.beginning_of_day.to_i}", 0, LIMIT - 1).map(&:to_i)
+      tag_ids = redis.zrevrange(KEY, 0, LIMIT - 1).map(&:to_i)
 
       tags = Tag.where(id: tag_ids)
       tags = tags.where(trendable: true) if filtered
@@ -33,8 +96,8 @@ class TrendingTags
     end
 
     def trending?(tag)
-      rank = redis.zrevrank("#{KEY}:#{Time.now.utc.beginning_of_day.to_i}", tag.id)
-      rank.present? && rank <= LIMIT
+      rank = redis.zrevrank(KEY, tag.id)
+      rank.present? && rank < LIMIT
     end
 
     private
@@ -51,31 +114,10 @@ class TrendingTags
       redis.expire(key, EXPIRE_HISTORY_AFTER)
     end
 
-    def increment_vote!(tag, at_time)
-      key      = "#{KEY}:#{at_time.beginning_of_day.to_i}"
-      expected = redis.pfcount("activity:tags:#{tag.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts").to_f
-      expected = 1.0 if expected.zero?
-      observed = redis.pfcount("activity:tags:#{tag.id}:#{at_time.beginning_of_day.to_i}:accounts").to_f
-
-      if expected > observed || observed < THRESHOLD
-        redis.zrem(key, tag.id)
-      else
-        score    = ((observed - expected)**2) / expected
-        old_rank = redis.zrevrank(key, tag.id)
-
-        redis.zadd(key, score, tag.id)
-        request_review!(tag) if (old_rank.nil? || old_rank > REVIEW_THRESHOLD) && redis.zrevrank(key, tag.id) <= REVIEW_THRESHOLD && !tag.trendable? && tag.requires_review? && !tag.requested_review?
-      end
-
-      redis.expire(key, EXPIRE_TRENDS_AFTER)
-    end
-
-    def request_review!(tag)
-      return unless Setting.trends
-
-      tag.touch(:requested_review_at)
-
-      User.staff.includes(:account).find_each { |u| AdminMailer.new_trending_tag(u.account, tag).deliver_later! if u.allows_trending_tag_emails? }
+    def increment_use!(tag_id, at_time)
+      key = "#{KEY}:used:#{at_time.beginning_of_day.to_i}"
+      redis.sadd(key, tag_id)
+      redis.expire(key, EXPIRE_HISTORY_AFTER)
     end
   end
 end
diff --git a/app/workers/scheduler/trending_tags_scheduler.rb b/app/workers/scheduler/trending_tags_scheduler.rb
new file mode 100644
index 000000000..77f0d5747
--- /dev/null
+++ b/app/workers/scheduler/trending_tags_scheduler.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+class Scheduler::TrendingTagsScheduler
+  include Sidekiq::Worker
+
+  sidekiq_options unique: :until_executed, retry: 0
+
+  def perform
+    TrendingTags.update! if Setting.trends
+  end
+end
diff --git a/config/sidekiq.yml b/config/sidekiq.yml
index 6ebe450b0..5de25de23 100644
--- a/config/sidekiq.yml
+++ b/config/sidekiq.yml
@@ -9,6 +9,9 @@
   scheduled_statuses_scheduler:
     every: '5m'
     class: Scheduler::ScheduledStatusesScheduler
+  trending_tags_scheduler:
+    every: '5m'
+    class: Scheduler::TrendingTagsScheduler
   media_cleanup_scheduler:
     cron: '<%= Random.rand(0..59) %> <%= Random.rand(3..5) %> * * *'
     class: Scheduler::MediaCleanupScheduler
diff --git a/db/migrate/20190901035623_add_max_score_to_tags.rb b/db/migrate/20190901035623_add_max_score_to_tags.rb
new file mode 100644
index 000000000..f936e9871
--- /dev/null
+++ b/db/migrate/20190901035623_add_max_score_to_tags.rb
@@ -0,0 +1,6 @@
+class AddMaxScoreToTags < ActiveRecord::Migration[5.2]
+  def change
+    add_column :tags, :max_score, :float
+    add_column :tags, :max_score_at, :datetime
+  end
+end
diff --git a/db/post_migrate/20190901040524_remove_score_from_tags.rb b/db/post_migrate/20190901040524_remove_score_from_tags.rb
new file mode 100644
index 000000000..a1112700b
--- /dev/null
+++ b/db/post_migrate/20190901040524_remove_score_from_tags.rb
@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+class RemoveScoreFromTags < ActiveRecord::Migration[5.2]
+  disable_ddl_transaction!
+
+  def change
+    safety_assured do
+      remove_column :tags, :score, :int
+      remove_column :tags, :last_trend_at, :datetime
+    end
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 482bca367..5576f70bf 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_08_23_221802) do
+ActiveRecord::Schema.define(version: 2019_09_01_040524) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -664,14 +664,14 @@ ActiveRecord::Schema.define(version: 2019_08_23_221802) do
     t.string "name", default: "", null: false
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
-    t.integer "score"
     t.boolean "usable"
     t.boolean "trendable"
     t.boolean "listable"
     t.datetime "reviewed_at"
     t.datetime "requested_review_at"
     t.datetime "last_status_at"
-    t.datetime "last_trend_at"
+    t.float "max_score"
+    t.datetime "max_score_at"
     t.index "lower((name)::text)", name: "index_tags_on_name_lower", unique: true
   end
 
diff --git a/spec/models/trending_tags_spec.rb b/spec/models/trending_tags_spec.rb
new file mode 100644
index 000000000..b6122c994
--- /dev/null
+++ b/spec/models/trending_tags_spec.rb
@@ -0,0 +1,68 @@
+require 'rails_helper'
+
+RSpec.describe TrendingTags do
+  describe '.record_use!' do
+    pending
+  end
+
+  describe '.update!' do
+    let!(:at_time) { Time.now.utc }
+    let!(:tag1) { Fabricate(:tag, name: 'Catstodon') }
+    let!(:tag2) { Fabricate(:tag, name: 'DogsOfMastodon') }
+    let!(:tag3) { Fabricate(:tag, name: 'OCs') }
+
+    before do
+      allow(Redis.current).to receive(:pfcount) do |key|
+        case key
+        when "activity:tags:#{tag1.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts"
+          2
+        when "activity:tags:#{tag1.id}:#{at_time.beginning_of_day.to_i}:accounts"
+          16
+        when "activity:tags:#{tag2.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts"
+          0
+        when "activity:tags:#{tag2.id}:#{at_time.beginning_of_day.to_i}:accounts"
+          4
+        when "activity:tags:#{tag3.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts"
+          13
+        end
+      end
+
+      Redis.current.zadd('trending_tags', 0.9, tag3.id)
+      Redis.current.sadd("trending_tags:used:#{at_time.beginning_of_day.to_i}", [tag1.id, tag2.id])
+
+      tag3.update(max_score: 0.9, max_score_at: (at_time - 1.day).beginning_of_day + 12.hours)
+
+      described_class.update!(at_time)
+    end
+
+    it 'calculates and re-calculates scores' do
+      expect(described_class.get(10, filtered: false)).to eq [tag1, tag3]
+    end
+
+    it 'omits hashtags below threshold' do
+      expect(described_class.get(10, filtered: false)).to_not include(tag2)
+    end
+
+    it 'decays scores' do
+      expect(Redis.current.zscore('trending_tags', tag3.id)).to be < 0.9
+    end
+  end
+
+  describe '.trending?' do
+    let(:tag) { Fabricate(:tag) }
+
+    before do
+      10.times { |i| Redis.current.zadd('trending_tags', i + 1, Fabricate(:tag).id) }
+    end
+
+    it 'returns true if the hashtag is within limit' do
+      Redis.current.zadd('trending_tags', 11, tag.id)
+      expect(described_class.trending?(tag)).to be true
+    end
+
+    it 'returns false if the hashtag is outside the limit' do
+      Redis.current.zadd('trending_tags', 0, tag.id)
+      expect(described_class.trending?(tag)).to be false
+    end
+  end
+end
-- 
cgit 


From e445a8af64908b2bdb721bec74c113e8258a129b Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Fri, 6 Sep 2019 13:55:51 +0200
Subject: Add timeline read markers API (#11762)

Fix #4093
---
 app/controllers/api/v1/markers_controller.rb       | 44 +++++++++++++++
 app/javascript/mastodon/actions/markers.js         | 30 ++++++++++
 app/javascript/mastodon/features/ui/index.js       |  5 +-
 app/models/marker.rb                               | 23 ++++++++
 app/models/user.rb                                 |  1 +
 app/serializers/rest/marker_serializer.rb          | 13 +++++
 config/routes.rb                                   |  1 +
 db/migrate/20190904222339_create_markers.rb        | 14 +++++
 db/schema.rb                                       | 14 ++++-
 spec/controllers/api/v1/markers_controller_spec.rb | 65 ++++++++++++++++++++++
 spec/fabricators/marker_fabricator.rb              |  6 ++
 spec/models/marker_spec.rb                         |  5 ++
 12 files changed, 219 insertions(+), 2 deletions(-)
 create mode 100644 app/controllers/api/v1/markers_controller.rb
 create mode 100644 app/javascript/mastodon/actions/markers.js
 create mode 100644 app/models/marker.rb
 create mode 100644 app/serializers/rest/marker_serializer.rb
 create mode 100644 db/migrate/20190904222339_create_markers.rb
 create mode 100644 spec/controllers/api/v1/markers_controller_spec.rb
 create mode 100644 spec/fabricators/marker_fabricator.rb
 create mode 100644 spec/models/marker_spec.rb

(limited to 'db/migrate')

diff --git a/app/controllers/api/v1/markers_controller.rb b/app/controllers/api/v1/markers_controller.rb
new file mode 100644
index 000000000..28c2ec791
--- /dev/null
+++ b/app/controllers/api/v1/markers_controller.rb
@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+class Api::V1::MarkersController < Api::BaseController
+  before_action -> { doorkeeper_authorize! :read, :'read:statuses' }, only: [:index]
+  before_action -> { doorkeeper_authorize! :write, :'write:statuses' }, except: [:index]
+
+  before_action :require_user!
+
+  def index
+    @markers = current_user.markers.where(timeline: Array(params[:timeline])).each_with_object({}) { |marker, h| h[marker.timeline] = marker }
+    render json: serialize_map(@markers)
+  end
+
+  def create
+    Marker.transaction do
+      @markers = {}
+
+      resource_params.each_pair do |timeline, timeline_params|
+        @markers[timeline] = current_user.markers.find_or_initialize_by(timeline: timeline)
+        @markers[timeline].update!(timeline_params)
+      end
+    end
+
+    render json: serialize_map(@markers)
+  rescue ActiveRecord::StaleObjectError
+    render json: { error: 'Conflict during update, please try again' }, status: 409
+  end
+
+  private
+
+  def serialize_map(map)
+    serialized = {}
+
+    map.each_pair do |key, value|
+      serialized[key] = ActiveModelSerializers::SerializableResource.new(value, serializer: REST::MarkerSerializer).as_json
+    end
+
+    Oj.dump(serialized)
+  end
+
+  def resource_params
+    params.slice(*Marker::TIMELINES).permit(*Marker::TIMELINES.map { |timeline| { timeline.to_sym => [:last_read_id] } })
+  end
+end
diff --git a/app/javascript/mastodon/actions/markers.js b/app/javascript/mastodon/actions/markers.js
new file mode 100644
index 000000000..c3a5fe86f
--- /dev/null
+++ b/app/javascript/mastodon/actions/markers.js
@@ -0,0 +1,30 @@
+export const submitMarkers = () => (dispatch, getState) => {
+  const accessToken = getState().getIn(['meta', 'access_token'], '');
+  const params      = {};
+
+  const lastHomeId         = getState().getIn(['timelines', 'home', 'items', 0]);
+  const lastNotificationId = getState().getIn(['notifications', 'items', 0, 'id']);
+
+  if (lastHomeId) {
+    params.home = {
+      last_read_id: lastHomeId,
+    };
+  }
+
+  if (lastNotificationId) {
+    params.notifications = {
+      last_read_id: lastNotificationId,
+    };
+  }
+
+  if (Object.keys(params).length === 0) {
+    return;
+  }
+
+  const client = new XMLHttpRequest();
+
+  client.open('POST', '/api/v1/markers', false);
+  client.setRequestHeader('Content-Type', 'application/json');
+  client.setRequestHeader('Authorization', `Bearer ${accessToken}`);
+  client.send(JSON.stringify(params));
+};
diff --git a/app/javascript/mastodon/features/ui/index.js b/app/javascript/mastodon/features/ui/index.js
index 49c5c8d0e..63c5622b6 100644
--- a/app/javascript/mastodon/features/ui/index.js
+++ b/app/javascript/mastodon/features/ui/index.js
@@ -16,6 +16,7 @@ import { expandNotifications } from '../../actions/notifications';
 import { fetchFilters } from '../../actions/filters';
 import { clearHeight } from '../../actions/height_cache';
 import { focusApp, unfocusApp } from 'mastodon/actions/app';
+import { submitMarkers } from 'mastodon/actions/markers';
 import { WrappedSwitch, WrappedRoute } from './util/react_router_helpers';
 import UploadArea from './components/upload_area';
 import ColumnsAreaContainer from './containers/columns_area_container';
@@ -241,7 +242,9 @@ class UI extends React.PureComponent {
   };
 
   handleBeforeUnload = e => {
-    const { intl, isComposing, hasComposingText, hasMediaAttachments } = this.props;
+    const { intl, dispatch, isComposing, hasComposingText, hasMediaAttachments } = this.props;
+
+    dispatch(submitMarkers());
 
     if (isComposing && (hasComposingText || hasMediaAttachments)) {
       // Setting returnValue to any string causes confirmation dialog.
diff --git a/app/models/marker.rb b/app/models/marker.rb
new file mode 100644
index 000000000..a5bd2176a
--- /dev/null
+++ b/app/models/marker.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+# == Schema Information
+#
+# Table name: markers
+#
+#  id           :bigint(8)        not null, primary key
+#  user_id      :bigint(8)
+#  timeline     :string           default(""), not null
+#  last_read_id :bigint(8)        default(0), not null
+#  lock_version :integer          default(0), not null
+#  created_at   :datetime         not null
+#  updated_at   :datetime         not null
+#
+
+class Marker < ApplicationRecord
+  TIMELINES = %w(home notifications).freeze
+
+  belongs_to :user
+
+  validates :timeline, :last_read_id, presence: true
+  validates :timeline, inclusion: { in: TIMELINES }
+end
diff --git a/app/models/user.rb b/app/models/user.rb
index a4a20d975..95f1d8fc5 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -74,6 +74,7 @@ class User < ApplicationRecord
   has_many :applications, class_name: 'Doorkeeper::Application', as: :owner
   has_many :backups, inverse_of: :user
   has_many :invites, inverse_of: :user
+  has_many :markers, inverse_of: :user, dependent: :destroy
 
   has_one :invite_request, class_name: 'UserInviteRequest', inverse_of: :user, dependent: :destroy
   accepts_nested_attributes_for :invite_request, reject_if: ->(attributes) { attributes['text'].blank? }
diff --git a/app/serializers/rest/marker_serializer.rb b/app/serializers/rest/marker_serializer.rb
new file mode 100644
index 000000000..2eaf3d507
--- /dev/null
+++ b/app/serializers/rest/marker_serializer.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+class REST::MarkerSerializer < ActiveModel::Serializer
+  attributes :last_read_id, :version, :updated_at
+
+  def last_read_id
+    object.last_read_id.to_s
+  end
+
+  def version
+    object.lock_version
+  end
+end
diff --git a/config/routes.rb b/config/routes.rb
index c5326052e..74a162f32 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -314,6 +314,7 @@ Rails.application.routes.draw do
       resources :trends,       only: [:index]
       resources :filters,      only: [:index, :create, :show, :update, :destroy]
       resources :endorsements, only: [:index]
+      resources :markers,      only: [:index, :create]
 
       namespace :apps do
         get :verify_credentials, to: 'credentials#show'
diff --git a/db/migrate/20190904222339_create_markers.rb b/db/migrate/20190904222339_create_markers.rb
new file mode 100644
index 000000000..71ca70ac3
--- /dev/null
+++ b/db/migrate/20190904222339_create_markers.rb
@@ -0,0 +1,14 @@
+class CreateMarkers < ActiveRecord::Migration[5.2]
+  def change
+    create_table :markers do |t|
+      t.references :user, foreign_key: { on_delete: :cascade, index: false }
+      t.string :timeline, default: '', null: false
+      t.bigint :last_read_id, default: 0, null: false
+      t.integer :lock_version, default: 0, null: false
+
+      t.timestamps
+    end
+
+    add_index :markers, [:user_id, :timeline], unique: true
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 5576f70bf..834dddd7b 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_09_01_040524) do
+ActiveRecord::Schema.define(version: 2019_09_04_222339) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -366,6 +366,17 @@ ActiveRecord::Schema.define(version: 2019_09_01_040524) do
     t.index ["account_id"], name: "index_lists_on_account_id"
   end
 
+  create_table "markers", force: :cascade do |t|
+    t.bigint "user_id"
+    t.string "timeline", default: "", null: false
+    t.bigint "last_read_id", default: 0, null: false
+    t.integer "lock_version", default: 0, null: false
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["user_id", "timeline"], name: "index_markers_on_user_id_and_timeline", unique: true
+    t.index ["user_id"], name: "index_markers_on_user_id"
+  end
+
   create_table "media_attachments", force: :cascade do |t|
     t.bigint "status_id"
     t.string "file_file_name"
@@ -791,6 +802,7 @@ ActiveRecord::Schema.define(version: 2019_09_01_040524) do
   add_foreign_key "list_accounts", "follows", on_delete: :cascade
   add_foreign_key "list_accounts", "lists", on_delete: :cascade
   add_foreign_key "lists", "accounts", on_delete: :cascade
+  add_foreign_key "markers", "users", on_delete: :cascade
   add_foreign_key "media_attachments", "accounts", name: "fk_96dd81e81b", on_delete: :nullify
   add_foreign_key "media_attachments", "scheduled_statuses", on_delete: :nullify
   add_foreign_key "media_attachments", "statuses", on_delete: :nullify
diff --git a/spec/controllers/api/v1/markers_controller_spec.rb b/spec/controllers/api/v1/markers_controller_spec.rb
new file mode 100644
index 000000000..556a75b9b
--- /dev/null
+++ b/spec/controllers/api/v1/markers_controller_spec.rb
@@ -0,0 +1,65 @@
+require 'rails_helper'
+
+RSpec.describe Api::V1::MarkersController, type: :controller do
+  render_views
+
+  let!(:user)  { Fabricate(:user, account: Fabricate(:account, username: 'alice')) }
+  let!(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'read:statuses write:statuses') }
+
+  before { allow(controller).to receive(:doorkeeper_token) { token } }
+
+  describe 'GET #index' do
+    before do
+      Fabricate(:marker, timeline: 'home', last_read_id: 123, user: user)
+      Fabricate(:marker, timeline: 'notifications', last_read_id: 456, user: user)
+
+      get :index, params: { timeline: %w(home notifications) }
+    end
+
+    it 'returns http success' do
+      expect(response).to have_http_status(200)
+    end
+
+    it 'returns markers' do
+      json = body_as_json
+
+      expect(json.key?(:home)).to be true
+      expect(json[:home][:last_read_id]).to eq '123'
+      expect(json.key?(:notifications)).to be true
+      expect(json[:notifications][:last_read_id]).to eq '456'
+    end
+  end
+
+  describe 'POST #create' do
+    context 'when no marker exists' do
+      before do
+        post :create, params: { home: { last_read_id: '69420' } }
+      end
+
+      it 'returns http success' do
+        expect(response).to have_http_status(200)
+      end
+
+      it 'creates a marker' do
+        expect(user.markers.first.timeline).to eq 'home'
+        expect(user.markers.first.last_read_id).to eq 69420
+      end
+    end
+
+    context 'when a marker exists' do
+      before do
+        post :create, params: { home: { last_read_id: '69420' } }
+        post :create, params: { home: { last_read_id: '70120' } }
+      end
+
+      it 'returns http success' do
+        expect(response).to have_http_status(200)
+      end
+
+      it 'updates a marker' do
+        expect(user.markers.first.timeline).to eq 'home'
+        expect(user.markers.first.last_read_id).to eq 70120
+      end
+    end
+  end
+end
diff --git a/spec/fabricators/marker_fabricator.rb b/spec/fabricators/marker_fabricator.rb
new file mode 100644
index 000000000..0c94150e0
--- /dev/null
+++ b/spec/fabricators/marker_fabricator.rb
@@ -0,0 +1,6 @@
+Fabricator(:marker) do
+  user
+  timeline     'home'
+  last_read_id 0
+  lock_version 0
+end
diff --git a/spec/models/marker_spec.rb b/spec/models/marker_spec.rb
new file mode 100644
index 000000000..d716aa75c
--- /dev/null
+++ b/spec/models/marker_spec.rb
@@ -0,0 +1,5 @@
+require 'rails_helper'
+
+RSpec.describe Marker, type: :model do
+  pending "add some examples to (or delete) #{__FILE__}"
+end
-- 
cgit 


From 0e6390753d4a93af7e705f7cca5946e232298fc5 Mon Sep 17 00:00:00 2001
From: abcang <abcang1015@gmail.com>
Date: Wed, 18 Sep 2019 17:58:08 +0900
Subject: Add users remember_token index (#11881)

---
 db/migrate/20190917213523_add_remember_token_index.rb | 9 +++++++++
 db/schema.rb                                          | 3 ++-
 2 files changed, 11 insertions(+), 1 deletion(-)
 create mode 100644 db/migrate/20190917213523_add_remember_token_index.rb

(limited to 'db/migrate')

diff --git a/db/migrate/20190917213523_add_remember_token_index.rb b/db/migrate/20190917213523_add_remember_token_index.rb
new file mode 100644
index 000000000..c5b41ce64
--- /dev/null
+++ b/db/migrate/20190917213523_add_remember_token_index.rb
@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class AddRememberTokenIndex < ActiveRecord::Migration[5.2]
+  disable_ddl_transaction!
+
+  def change
+    add_index :users, :remember_token, algorithm: :concurrently, unique: true
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 834dddd7b..749f79dee 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_09_04_222339) do
+ActiveRecord::Schema.define(version: 2019_09_17_213523) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -743,6 +743,7 @@ ActiveRecord::Schema.define(version: 2019_09_04_222339) do
     t.index ["confirmation_token"], name: "index_users_on_confirmation_token", unique: true
     t.index ["created_by_application_id"], name: "index_users_on_created_by_application_id"
     t.index ["email"], name: "index_users_on_email", unique: true
+    t.index ["remember_token"], name: "index_users_on_remember_token", unique: true
     t.index ["reset_password_token"], name: "index_users_on_reset_password_token", unique: true
   end
 
-- 
cgit 


From 3ed94dcc1acf73f1d0d1ab43567b88ee953f57c9 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Thu, 19 Sep 2019 20:58:19 +0200
Subject: Add account migration UI (#11846)

Fix #10736

- Change data export to be available for non-functional accounts
- Change non-functional accounts to include redirecting accounts
---
 .../concerns/export_controller_concern.rb          |  7 ++
 app/controllers/settings/aliases_controller.rb     | 42 +++++++++++
 app/controllers/settings/exports_controller.rb     |  7 ++
 app/controllers/settings/migrations_controller.rb  | 48 ++++++++++---
 app/helpers/settings_helper.rb                     |  8 +++
 app/models/account_alias.rb                        | 41 +++++++++++
 app/models/account_migration.rb                    | 74 +++++++++++++++++++
 app/models/concerns/account_associations.rb        |  2 +
 app/models/form/migration.rb                       | 25 -------
 app/models/remote_follow.rb                        |  2 +-
 app/models/user.rb                                 |  2 +-
 app/serializers/activitypub/move_serializer.rb     | 26 +++++++
 app/views/auth/registrations/_status.html.haml     | 30 ++++----
 app/views/auth/registrations/edit.html.haml        |  2 +-
 app/views/settings/aliases/index.html.haml         | 29 ++++++++
 app/views/settings/exports/show.html.haml          |  4 ++
 app/views/settings/migrations/show.html.haml       | 84 +++++++++++++++++++---
 app/views/settings/profiles/show.html.haml         |  5 ++
 .../activitypub/move_distribution_worker.rb        | 32 +++++++++
 config/locales/en.yml                              | 38 ++++++++--
 config/locales/simple_form.en.yml                  | 10 +++
 config/navigation.rb                               |  8 +--
 config/routes.rb                                   |  8 ++-
 .../20190914202517_create_account_migrations.rb    | 12 ++++
 .../20190915194355_create_account_aliases.rb       | 11 +++
 db/schema.rb                                       | 23 ++++++
 .../settings/migrations_controller_spec.rb         | 14 ++--
 spec/fabricators/account_alias_fabricator.rb       |  5 ++
 spec/fabricators/account_migration_fabricator.rb   |  6 ++
 spec/models/account_alias_spec.rb                  |  5 ++
 spec/models/account_migration_spec.rb              |  5 ++
 31 files changed, 542 insertions(+), 73 deletions(-)
 create mode 100644 app/controllers/settings/aliases_controller.rb
 create mode 100644 app/models/account_alias.rb
 create mode 100644 app/models/account_migration.rb
 delete mode 100644 app/models/form/migration.rb
 create mode 100644 app/serializers/activitypub/move_serializer.rb
 create mode 100644 app/views/settings/aliases/index.html.haml
 create mode 100644 app/workers/activitypub/move_distribution_worker.rb
 create mode 100644 db/migrate/20190914202517_create_account_migrations.rb
 create mode 100644 db/migrate/20190915194355_create_account_aliases.rb
 create mode 100644 spec/fabricators/account_alias_fabricator.rb
 create mode 100644 spec/fabricators/account_migration_fabricator.rb
 create mode 100644 spec/models/account_alias_spec.rb
 create mode 100644 spec/models/account_migration_spec.rb

(limited to 'db/migrate')

diff --git a/app/controllers/concerns/export_controller_concern.rb b/app/controllers/concerns/export_controller_concern.rb
index e20b71a30..bfe990c82 100644
--- a/app/controllers/concerns/export_controller_concern.rb
+++ b/app/controllers/concerns/export_controller_concern.rb
@@ -5,7 +5,10 @@ module ExportControllerConcern
 
   included do
     before_action :authenticate_user!
+    before_action :require_not_suspended!
     before_action :load_export
+
+    skip_before_action :require_functional!
   end
 
   private
@@ -27,4 +30,8 @@ module ExportControllerConcern
   def export_filename
     "#{controller_name}.csv"
   end
+
+  def require_not_suspended!
+    forbidden if current_account.suspended?
+  end
 end
diff --git a/app/controllers/settings/aliases_controller.rb b/app/controllers/settings/aliases_controller.rb
new file mode 100644
index 000000000..2b675f065
--- /dev/null
+++ b/app/controllers/settings/aliases_controller.rb
@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+class Settings::AliasesController < Settings::BaseController
+  layout 'admin'
+
+  before_action :authenticate_user!
+  before_action :set_aliases, except: :destroy
+  before_action :set_alias, only: :destroy
+
+  def index
+    @alias = current_account.aliases.build
+  end
+
+  def create
+    @alias = current_account.aliases.build(resource_params)
+
+    if @alias.save
+      redirect_to settings_aliases_path, notice: I18n.t('aliases.created_msg')
+    else
+      render :show
+    end
+  end
+
+  def destroy
+    @alias.destroy!
+    redirect_to settings_aliases_path, notice: I18n.t('aliases.deleted_msg')
+  end
+
+  private
+
+  def resource_params
+    params.require(:account_alias).permit(:acct)
+  end
+
+  def set_alias
+    @alias = current_account.aliases.find(params[:id])
+  end
+
+  def set_aliases
+    @aliases = current_account.aliases.order(id: :desc).reject(&:new_record?)
+  end
+end
diff --git a/app/controllers/settings/exports_controller.rb b/app/controllers/settings/exports_controller.rb
index 3012fbf77..0e93d07a9 100644
--- a/app/controllers/settings/exports_controller.rb
+++ b/app/controllers/settings/exports_controller.rb
@@ -6,6 +6,9 @@ class Settings::ExportsController < Settings::BaseController
   layout 'admin'
 
   before_action :authenticate_user!
+  before_action :require_not_suspended!
+
+  skip_before_action :require_functional!
 
   def show
     @export  = Export.new(current_account)
@@ -34,4 +37,8 @@ class Settings::ExportsController < Settings::BaseController
   def lock_options
     { redis: Redis.current, key: "backup:#{current_user.id}" }
   end
+
+  def require_not_suspended!
+    forbidden if current_account.suspended?
+  end
 end
diff --git a/app/controllers/settings/migrations_controller.rb b/app/controllers/settings/migrations_controller.rb
index 59eb48779..90092c692 100644
--- a/app/controllers/settings/migrations_controller.rb
+++ b/app/controllers/settings/migrations_controller.rb
@@ -4,31 +4,59 @@ class Settings::MigrationsController < Settings::BaseController
   layout 'admin'
 
   before_action :authenticate_user!
+  before_action :require_not_suspended!
+  before_action :set_migrations
+  before_action :set_cooldown
+
+  skip_before_action :require_functional!
 
   def show
-    @migration = Form::Migration.new(account: current_account.moved_to_account)
+    @migration = current_account.migrations.build
   end
 
-  def update
-    @migration = Form::Migration.new(resource_params)
+  def create
+    @migration = current_account.migrations.build(resource_params)
 
-    if @migration.valid? && migration_account_changed?
-      current_account.update!(moved_to_account: @migration.account)
+    if @migration.save_with_challenge(current_user)
+      current_account.update!(moved_to_account: @migration.target_account)
       ActivityPub::UpdateDistributionWorker.perform_async(current_account.id)
-      redirect_to settings_migration_path, notice: I18n.t('migrations.updated_msg')
+      ActivityPub::MoveDistributionWorker.perform_async(@migration.id)
+      redirect_to settings_migration_path, notice: I18n.t('migrations.moved_msg', acct: current_account.moved_to_account.acct)
     else
       render :show
     end
   end
 
+  def cancel
+    if current_account.moved_to_account_id.present?
+      current_account.update!(moved_to_account: nil)
+      ActivityPub::UpdateDistributionWorker.perform_async(current_account.id)
+    end
+
+    redirect_to settings_migration_path, notice: I18n.t('migrations.cancelled_msg')
+  end
+
+  helper_method :on_cooldown?
+
   private
 
   def resource_params
-    params.require(:migration).permit(:acct)
+    params.require(:account_migration).permit(:acct, :current_password, :current_username)
+  end
+
+  def set_migrations
+    @migrations = current_account.migrations.includes(:target_account).order(id: :desc).reject(&:new_record?)
+  end
+
+  def set_cooldown
+    @cooldown = current_account.migrations.within_cooldown.first
+  end
+
+  def on_cooldown?
+    @cooldown.present?
   end
 
-  def migration_account_changed?
-    current_account.moved_to_account_id != @migration.account&.id &&
-      current_account.id != @migration.account&.id
+  def require_not_suspended!
+    forbidden if current_account.suspended?
   end
 end
diff --git a/app/helpers/settings_helper.rb b/app/helpers/settings_helper.rb
index 2b3fd1263..ecc73baf5 100644
--- a/app/helpers/settings_helper.rb
+++ b/app/helpers/settings_helper.rb
@@ -87,4 +87,12 @@ module SettingsHelper
       'desktop'
     end
   end
+
+  def compact_account_link_to(account)
+    return if account.nil?
+
+    link_to ActivityPub::TagManager.instance.url_for(account), class: 'name-tag', title: account.acct do
+      safe_join([image_tag(account.avatar.url, width: 15, height: 15, alt: display_name(account), class: 'avatar'), content_tag(:span, account.acct, class: 'username')], ' ')
+    end
+  end
 end
diff --git a/app/models/account_alias.rb b/app/models/account_alias.rb
new file mode 100644
index 000000000..e9a0dd79e
--- /dev/null
+++ b/app/models/account_alias.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# == Schema Information
+#
+# Table name: account_aliases
+#
+#  id         :bigint(8)        not null, primary key
+#  account_id :bigint(8)
+#  acct       :string           default(""), not null
+#  uri        :string           default(""), not null
+#  created_at :datetime         not null
+#  updated_at :datetime         not null
+#
+
+class AccountAlias < ApplicationRecord
+  belongs_to :account
+
+  validates :acct, presence: true, domain: { acct: true }
+  validates :uri, presence: true
+
+  before_validation :set_uri
+  after_create :add_to_account
+  after_destroy :remove_from_account
+
+  private
+
+  def set_uri
+    target_account = ResolveAccountService.new.call(acct)
+    self.uri       = ActivityPub::TagManager.instance.uri_for(target_account) unless target_account.nil?
+  rescue Goldfinger::Error, HTTP::Error, OpenSSL::SSL::SSLError, Mastodon::Error
+    # Validation will take care of it
+  end
+
+  def add_to_account
+    account.update(also_known_as: account.also_known_as + [uri])
+  end
+
+  def remove_from_account
+    account.update(also_known_as: account.also_known_as.reject { |x| x == uri })
+  end
+end
diff --git a/app/models/account_migration.rb b/app/models/account_migration.rb
new file mode 100644
index 000000000..15830bffb
--- /dev/null
+++ b/app/models/account_migration.rb
@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+# == Schema Information
+#
+# Table name: account_migrations
+#
+#  id                :bigint(8)        not null, primary key
+#  account_id        :bigint(8)
+#  acct              :string           default(""), not null
+#  followers_count   :bigint(8)        default(0), not null
+#  target_account_id :bigint(8)
+#  created_at        :datetime         not null
+#  updated_at        :datetime         not null
+#
+
+class AccountMigration < ApplicationRecord
+  COOLDOWN_PERIOD = 30.days.freeze
+
+  belongs_to :account
+  belongs_to :target_account, class_name: 'Account'
+
+  before_validation :set_target_account
+  before_validation :set_followers_count
+
+  validates :acct, presence: true, domain: { acct: true }
+  validate :validate_migration_cooldown
+  validate :validate_target_account
+
+  scope :within_cooldown, ->(now = Time.now.utc) { where(arel_table[:created_at].gteq(now - COOLDOWN_PERIOD)) }
+
+  attr_accessor :current_password, :current_username
+
+  def save_with_challenge(current_user)
+    if current_user.encrypted_password.present?
+      errors.add(:current_password, :invalid) unless current_user.valid_password?(current_password)
+    else
+      errors.add(:current_username, :invalid) unless account.username == current_username
+    end
+
+    return false unless errors.empty?
+
+    save
+  end
+
+  def cooldown_at
+    created_at + COOLDOWN_PERIOD
+  end
+
+  private
+
+  def set_target_account
+    self.target_account = ResolveAccountService.new.call(acct)
+  rescue Goldfinger::Error, HTTP::Error, OpenSSL::SSL::SSLError, Mastodon::Error
+    # Validation will take care of it
+  end
+
+  def set_followers_count
+    self.followers_count = account.followers_count
+  end
+
+  def validate_target_account
+    if target_account.nil?
+      errors.add(:acct, I18n.t('migrations.errors.not_found'))
+    else
+      errors.add(:acct, I18n.t('migrations.errors.missing_also_known_as')) unless target_account.also_known_as.include?(ActivityPub::TagManager.instance.uri_for(account))
+      errors.add(:acct, I18n.t('migrations.errors.already_moved')) if account.moved_to_account_id.present? && account.moved_to_account_id == target_account.id
+      errors.add(:acct, I18n.t('migrations.errors.move_to_self')) if account.id == target_account.id
+    end
+  end
+
+  def validate_migration_cooldown
+    errors.add(:base, I18n.t('migrations.errors.on_cooldown')) if account.migrations.within_cooldown.exists?
+  end
+end
diff --git a/app/models/concerns/account_associations.rb b/app/models/concerns/account_associations.rb
index 1db7771c7..c9cc5c610 100644
--- a/app/models/concerns/account_associations.rb
+++ b/app/models/concerns/account_associations.rb
@@ -52,6 +52,8 @@ module AccountAssociations
 
     # Account migrations
     belongs_to :moved_to_account, class_name: 'Account', optional: true
+    has_many :migrations, class_name: 'AccountMigration', dependent: :destroy, inverse_of: :account
+    has_many :aliases, class_name: 'AccountAlias', dependent: :destroy, inverse_of: :account
 
     # Hashtags
     has_and_belongs_to_many :tags
diff --git a/app/models/form/migration.rb b/app/models/form/migration.rb
deleted file mode 100644
index c2a8655e1..000000000
--- a/app/models/form/migration.rb
+++ /dev/null
@@ -1,25 +0,0 @@
-# frozen_string_literal: true
-
-class Form::Migration
-  include ActiveModel::Validations
-
-  attr_accessor :acct, :account
-
-  def initialize(attrs = {})
-    @account = attrs[:account]
-    @acct    = attrs[:account].acct unless @account.nil?
-    @acct    = attrs[:acct].gsub(/\A@/, '').strip unless attrs[:acct].nil?
-  end
-
-  def valid?
-    return false unless super
-    set_account
-    errors.empty?
-  end
-
-  private
-
-  def set_account
-    self.account = (ResolveAccountService.new.call(acct) if account.nil? && acct.present?)
-  end
-end
diff --git a/app/models/remote_follow.rb b/app/models/remote_follow.rb
index 52dd3f67b..5ea535287 100644
--- a/app/models/remote_follow.rb
+++ b/app/models/remote_follow.rb
@@ -49,7 +49,7 @@ class RemoteFollow
   end
 
   def fetch_template!
-    return missing_resource if acct.blank?
+    return missing_resource_error if acct.blank?
 
     _, domain = acct.split('@')
 
diff --git a/app/models/user.rb b/app/models/user.rb
index b48455802..9a19a53b3 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -168,7 +168,7 @@ class User < ApplicationRecord
   end
 
   def functional?
-    confirmed? && approved? && !disabled? && !account.suspended?
+    confirmed? && approved? && !disabled? && !account.suspended? && account.moved_to_account_id.nil?
   end
 
   def unconfirmed_or_pending?
diff --git a/app/serializers/activitypub/move_serializer.rb b/app/serializers/activitypub/move_serializer.rb
new file mode 100644
index 000000000..5675875fa
--- /dev/null
+++ b/app/serializers/activitypub/move_serializer.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+class ActivityPub::MoveSerializer < ActivityPub::Serializer
+  attributes :id, :type, :target, :actor
+  attribute :virtual_object, key: :object
+
+  def id
+    [ActivityPub::TagManager.instance.uri_for(object.account), '#moves/', object.id].join
+  end
+
+  def type
+    'Move'
+  end
+
+  def target
+    ActivityPub::TagManager.instance.uri_for(object.target_account)
+  end
+
+  def virtual_object
+    ActivityPub::TagManager.instance.uri_for(object.account)
+  end
+
+  def actor
+    ActivityPub::TagManager.instance.uri_for(object.account)
+  end
+end
diff --git a/app/views/auth/registrations/_status.html.haml b/app/views/auth/registrations/_status.html.haml
index b38a83d67..47112dae0 100644
--- a/app/views/auth/registrations/_status.html.haml
+++ b/app/views/auth/registrations/_status.html.haml
@@ -1,16 +1,22 @@
 %h3= t('auth.status.account_status')
 
-- if @user.account.suspended?
-  %span.negative-hint= t('user_mailer.warning.explanation.suspend')
-- elsif @user.disabled?
-  %span.negative-hint= t('user_mailer.warning.explanation.disable')
-- elsif @user.account.silenced?
-  %span.warning-hint= t('user_mailer.warning.explanation.silence')
-- elsif !@user.confirmed?
-  %span.warning-hint= t('auth.status.confirming')
-- elsif !@user.approved?
-  %span.warning-hint= t('auth.status.pending')
-- else
-  %span.positive-hint= t('auth.status.functional')
+.simple_form
+  %p.hint
+    - if @user.account.suspended?
+      %span.negative-hint= t('user_mailer.warning.explanation.suspend')
+    - elsif @user.disabled?
+      %span.negative-hint= t('user_mailer.warning.explanation.disable')
+    - elsif @user.account.silenced?
+      %span.warning-hint= t('user_mailer.warning.explanation.silence')
+    - elsif !@user.confirmed?
+      %span.warning-hint= t('auth.status.confirming')
+      = link_to t('auth.didnt_get_confirmation'), new_user_confirmation_path
+    - elsif !@user.approved?
+      %span.warning-hint= t('auth.status.pending')
+    - elsif @user.account.moved_to_account_id.present?
+      %span.positive-hint= t('auth.status.redirecting_to', acct: @user.account.moved_to_account.acct)
+      = link_to t('migrations.cancel'), settings_migration_path
+    - else
+      %span.positive-hint= t('auth.status.functional')
 
 %hr.spacer/
diff --git a/app/views/auth/registrations/edit.html.haml b/app/views/auth/registrations/edit.html.haml
index 710ee5c68..885171c58 100644
--- a/app/views/auth/registrations/edit.html.haml
+++ b/app/views/auth/registrations/edit.html.haml
@@ -13,7 +13,7 @@
       .fields-row__column.fields-group.fields-row__column-6
         = f.input :email, wrapper: :with_label, input_html: { 'aria-label' => t('simple_form.labels.defaults.email') }, required: true, disabled: current_account.suspended?
       .fields-row__column.fields-group.fields-row__column-6
-        = f.input :current_password, wrapper: :with_label, input_html: { 'aria-label' => t('simple_form.labels.defaults.current_password'), :autocomplete => 'off' }, required: true, disabled: current_account.suspended?
+        = f.input :current_password, wrapper: :with_label, input_html: { 'aria-label' => t('simple_form.labels.defaults.current_password'), :autocomplete => 'off' }, required: true, disabled: current_account.suspended?, hint: false
 
     .fields-row
       .fields-row__column.fields-group.fields-row__column-6
diff --git a/app/views/settings/aliases/index.html.haml b/app/views/settings/aliases/index.html.haml
new file mode 100644
index 000000000..5b6986368
--- /dev/null
+++ b/app/views/settings/aliases/index.html.haml
@@ -0,0 +1,29 @@
+- content_for :page_title do
+  = t('settings.aliases')
+
+= simple_form_for @alias, url: settings_aliases_path do |f|
+  = render 'shared/error_messages', object: @alias
+
+  %p.hint= t('aliases.hint_html')
+
+  %hr.spacer/
+
+  .fields-group
+    = f.input :acct, wrapper: :with_block_label, input_html: { autocapitalize: 'none', autocorrect: 'off' }
+
+  .actions
+    = f.button :button, t('aliases.add_new'), type: :submit, class: 'button'
+
+%hr.spacer/
+
+.table-wrapper
+  %table.table.inline-table
+    %thead
+      %tr
+        %th= t('simple_form.labels.account_alias.acct')
+        %th
+    %tbody
+      - @aliases.each do |account_alias|
+        %tr
+          %td= account_alias.acct
+          %td= table_link_to 'trash', t('aliases.remove'), settings_alias_path(account_alias), data: { method: :delete }
diff --git a/app/views/settings/exports/show.html.haml b/app/views/settings/exports/show.html.haml
index b13cea976..76ff76bd9 100644
--- a/app/views/settings/exports/show.html.haml
+++ b/app/views/settings/exports/show.html.haml
@@ -37,12 +37,16 @@
         %td= number_with_delimiter @export.total_domain_blocks
         %td= table_link_to 'download', t('exports.csv'), settings_exports_domain_blocks_path(format: :csv)
 
+%hr.spacer/
+
 %p.muted-hint= t('exports.archive_takeout.hint_html')
 
 - if policy(:backup).create?
   %p= link_to t('exports.archive_takeout.request'), settings_export_path, class: 'button', method: :post
 
 - unless @backups.empty?
+  %hr.spacer/
+
   .table-wrapper
     %table.table
       %thead
diff --git a/app/views/settings/migrations/show.html.haml b/app/views/settings/migrations/show.html.haml
index c69061d50..1e5c47726 100644
--- a/app/views/settings/migrations/show.html.haml
+++ b/app/views/settings/migrations/show.html.haml
@@ -1,17 +1,85 @@
 - content_for :page_title do
   = t('settings.migrate')
 
-= simple_form_for @migration, as: :migration, url: settings_migration_path, html: { method: :put } do |f|
-  - if @migration.account
-    %p.hint= t('migrations.currently_redirecting')
+.simple_form
+  - if current_account.moved_to_account.present?
+    .fields-row
+      .fields-row__column.fields-group.fields-row__column-6
+        = render 'application/card', account: current_account.moved_to_account
+      .fields-row__column.fields-group.fields-row__column-6
+        %p.hint
+          %span.positive-hint= t('migrations.redirecting_to', acct: current_account.moved_to_account.acct)
 
-    .fields-group
-      = render partial: 'application/card', locals: { account: @migration.account }
+        %p.hint= t('migrations.cancel_explanation')
+
+        %p.hint= link_to t('migrations.cancel'), cancel_settings_migration_path, data: { method: :post }
+  - else
+    %p.hint
+      %span.positive-hint= t('migrations.not_redirecting')
+
+%hr.spacer/
+
+%h3= t 'migrations.proceed_with_move'
+
+= simple_form_for @migration, url: settings_migration_path do |f|
+  - if on_cooldown?
+    %span.warning-hint= t('migrations.on_cooldown', count: ((@cooldown.cooldown_at - Time.now.utc) / 1.day.seconds).ceil)
+  - else
+    %p.hint= t('migrations.warning.before')
+
+    %ul.hint
+      %li.warning-hint= t('migrations.warning.followers')
+      %li.warning-hint= t('migrations.warning.other_data')
+      %li.warning-hint= t('migrations.warning.backreference_required')
+      %li.warning-hint= t('migrations.warning.cooldown')
+      %li.warning-hint= t('migrations.warning.disabled_account')
+
+  %hr.spacer/
 
   = render 'shared/error_messages', object: @migration
 
-  .fields-group
-    = f.input :acct, placeholder: t('migrations.acct')
+  .fields-row
+    .fields-row__column.fields-group.fields-row__column-6
+      = f.input :acct, wrapper: :with_block_label, input_html: { autocapitalize: 'none', autocorrect: 'off' }, disabled: on_cooldown?
+
+    .fields-row__column.fields-group.fields-row__column-6
+      - if current_user.encrypted_password.present?
+        = f.input :current_password, wrapper: :with_block_label, input_html: { :autocomplete => 'off' }, required: true, disabled: on_cooldown?
+      - else
+        = f.input :current_username, wrapper: :with_block_label, input_html: { :autocomplete => 'off' }, required: true, disabled: on_cooldown?
 
   .actions
-    = f.button :button, t('migrations.proceed'), type: :submit, class: 'negative'
+    = f.button :button, t('migrations.proceed_with_move'), type: :submit, class: 'button button--destructive', disabled: on_cooldown?
+
+- unless @migrations.empty?
+  %hr.spacer/
+
+  %h3= t 'migrations.past_migrations'
+
+  %hr.spacer/
+
+  .table-wrapper
+    %table.table.inline-table
+      %thead
+        %tr
+          %th= t('migrations.acct')
+          %th= t('migrations.followers_count')
+          %th
+      %tbody
+        - @migrations.each do |migration|
+          %tr
+            %td
+              - if migration.target_account.present?
+                = compact_account_link_to migration.target_account
+              - else
+                = migration.acct
+
+            %td= number_with_delimiter migration.followers_count
+
+            %td
+              %time.time-ago{ datetime: migration.created_at.iso8601, title: l(migration.created_at) }= l(migration.created_at)
+
+%hr.spacer/
+
+%h3= t 'migrations.incoming_migrations'
+%p.muted-hint= t('migrations.incoming_migrations_html', path: settings_aliases_path)
diff --git a/app/views/settings/profiles/show.html.haml b/app/views/settings/profiles/show.html.haml
index f042011d6..6929f54f3 100644
--- a/app/views/settings/profiles/show.html.haml
+++ b/app/views/settings/profiles/show.html.haml
@@ -60,6 +60,11 @@
 %h6= t('auth.migrate_account')
 %p.muted-hint= t('auth.migrate_account_html', path: settings_migration_path)
 
+%hr.spacer/
+
+%h6= t 'migrations.incoming_migrations'
+%p.muted-hint= t('migrations.incoming_migrations_html', path: settings_aliases_path)
+
 - if open_deletion?
   %hr.spacer/
 
diff --git a/app/workers/activitypub/move_distribution_worker.rb b/app/workers/activitypub/move_distribution_worker.rb
new file mode 100644
index 000000000..396d5258f
--- /dev/null
+++ b/app/workers/activitypub/move_distribution_worker.rb
@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+class ActivityPub::MoveDistributionWorker
+  include Sidekiq::Worker
+  include Payloadable
+
+  sidekiq_options queue: 'push'
+
+  def perform(migration_id)
+    @migration = AccountMigration.find(migration_id)
+
+    ActivityPub::DeliveryWorker.push_bulk(inboxes) do |inbox_url|
+      [signed_payload, @account.id, inbox_url]
+    end
+
+    ActivityPub::DeliveryWorker.push_bulk(Relay.enabled.pluck(:inbox_url)) do |inbox_url|
+      [signed_payload, @account.id, inbox_url]
+    end
+  rescue ActiveRecord::RecordNotFound
+    true
+  end
+
+  private
+
+  def inboxes
+    @inboxes ||= @migration.account.followers.inboxes
+  end
+
+  def signed_payload
+    @signed_payload ||= Oj.dump(serialize_payload(@migration, ActivityPub::MoveSerializer, signer: @account))
+  end
+end
diff --git a/config/locales/en.yml b/config/locales/en.yml
index dabb679e7..c29c7f871 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -554,6 +554,12 @@ en:
     new_trending_tag:
       body: 'The hashtag #%{name} is trending today, but has not been previously reviewed. It will not be displayed publicly unless you allow it to, or just save the form as it is to never hear about it again.'
       subject: New hashtag up for review on %{instance} (#%{name})
+  aliases:
+    add_new: Create alias
+    created_msg: Successfully created a new alias. You can now initiate the move from the old account.
+    deleted_msg: Successfully remove the alias. Moving from that account to this one will no longer be possible.
+    hint_html: If you want to move from another account to this one, here you can create an alias, which is required before you can proceed with moving followers from the old account to this one. This action by itself is <strong>harmless and reversible</strong>. <strong>The account migration is initiated from the old account</strong>.
+    remove: Unlink alias
   appearance:
     advanced_web_interface: Advanced web interface
     advanced_web_interface_hint: 'If you want to make use of your entire screen width, the advanced web interface allows you to configure many different columns to see as much information at the same time as you want: Home, notifications, federated timeline, any number of lists and hashtags.'
@@ -613,6 +619,7 @@ en:
       confirming: Waiting for e-mail confirmation to be completed.
       functional: Your account is fully operational.
       pending: Your application is pending review by our staff. This may take some time. You will receive an e-mail if your application is approved.
+      redirecting_to: Your account is inactive because it is currently redirecting to %{acct}.
     trouble_logging_in: Trouble logging in?
   authorize_follow:
     already_following: You are already following this account
@@ -801,10 +808,32 @@ en:
       images_and_video: Cannot attach a video to a status that already contains images
       too_many: Cannot attach more than 4 files
   migrations:
-    acct: username@domain of the new account
-    currently_redirecting: 'Your profile is set to redirect to:'
-    proceed: Save
-    updated_msg: Your account migration setting successfully updated!
+    acct: Moved to
+    cancel: Cancel redirect
+    cancel_explanation: Cancelling the redirect will re-activate your current account, but will not bring back followers that have been moved to that account.
+    cancelled_msg: Successfully cancelled the redirect.
+    errors:
+      already_moved: is the same account you have already moved to
+      missing_also_known_as: is not back-referencing this account
+      move_to_self: cannot be current account
+      not_found: could not be found
+      on_cooldown: You are on cooldown
+    followers_count: Followers at time of move
+    incoming_migrations: Moving from a different account
+    incoming_migrations_html: To move from another account to this one, first you need to <a href="%{path}">create an account alias</a>.
+    moved_msg: Your account is now redirecting to %{acct} and your followers are being moved over.
+    not_redirecting: Your account is not redirecting to any other account currently.
+    on_cooldown: You have recently migrated your account. This function will become available again in %{count} days.
+    past_migrations: Past migrations
+    proceed_with_move: Move followers
+    redirecting_to: Your account is redirecting to %{acct}.
+    warning:
+      backreference_required: The new account must first be configured to back-reference this one
+      before: 'Before proceeding, please read these notes carefully:'
+      cooldown: After moving there is a cooldown period during which you will not be able to move again
+      disabled_account: Your current account will not be fully usable afterwards. However, you will have access to data export as well as re-activation.
+      followers: This action will move all followers from the current account to the new account
+      other_data: No other data will be moved automatically
   moderation:
     title: Moderation
   notification_mailer:
@@ -950,6 +979,7 @@ en:
   settings:
     account: Account
     account_settings: Account settings
+    aliases: Account aliases
     appearance: Appearance
     authorized_apps: Authorized apps
     back: Back to Mastodon
diff --git a/config/locales/simple_form.en.yml b/config/locales/simple_form.en.yml
index c9ffcfc13..3d909e999 100644
--- a/config/locales/simple_form.en.yml
+++ b/config/locales/simple_form.en.yml
@@ -2,6 +2,10 @@
 en:
   simple_form:
     hints:
+      account_alias:
+        acct: Specify the username@domain of the account you want to move from
+      account_migration:
+        acct: Specify the username@domain of the account you want to move to
       account_warning_preset:
         text: You can use toot syntax, such as URLs, hashtags and mentions
       admin_account_action:
@@ -15,6 +19,8 @@ en:
         avatar: PNG, GIF or JPG. At most %{size}. Will be downscaled to %{dimensions}px
         bot: This account mainly performs automated actions and might not be monitored
         context: One or multiple contexts where the filter should apply
+        current_password: For security purposes please enter the password of the current account
+        current_username: To confirm, please enter the username of the current account
         digest: Only sent after a long period of inactivity and only if you have received any personal messages in your absence
         discoverable: The profile directory is another way by which your account can reach a wider audience
         email: You will be sent a confirmation e-mail
@@ -60,6 +66,10 @@ en:
         fields:
           name: Label
           value: Content
+      account_alias:
+        acct: Handle of the old account
+      account_migration:
+        acct: Handle of the new account
       account_warning_preset:
         text: Preset text
       admin_account_action:
diff --git a/config/navigation.rb b/config/navigation.rb
index 38668bbf7..32c299143 100644
--- a/config/navigation.rb
+++ b/config/navigation.rb
@@ -5,7 +5,7 @@ SimpleNavigation::Configuration.run do |navigation|
     n.item :web, safe_join([fa_icon('chevron-left fw'), t('settings.back')]), root_url
 
     n.item :profile, safe_join([fa_icon('user fw'), t('settings.profile')]), settings_profile_url, if: -> { current_user.functional? } do |s|
-      s.item :profile, safe_join([fa_icon('pencil fw'), t('settings.appearance')]), settings_profile_url, highlights_on: %r{/settings/profile|/settings/migration}
+      s.item :profile, safe_join([fa_icon('pencil fw'), t('settings.appearance')]), settings_profile_url
       s.item :featured_tags, safe_join([fa_icon('hashtag fw'), t('settings.featured_tags')]), settings_featured_tags_url
       s.item :identity_proofs, safe_join([fa_icon('key fw'), t('settings.identity_proofs')]), settings_identity_proofs_path, highlights_on: %r{/settings/identity_proofs*}, if: proc { current_account.identity_proofs.exists? }
     end
@@ -20,13 +20,13 @@ SimpleNavigation::Configuration.run do |navigation|
     n.item :filters, safe_join([fa_icon('filter fw'), t('filters.index.title')]), filters_path, highlights_on: %r{/filters}, if: -> { current_user.functional? }
 
     n.item :security, safe_join([fa_icon('lock fw'), t('settings.account')]), edit_user_registration_url do |s|
-      s.item :password, safe_join([fa_icon('lock fw'), t('settings.account_settings')]), edit_user_registration_url, highlights_on: %r{/auth/edit|/settings/delete}
+      s.item :password, safe_join([fa_icon('lock fw'), t('settings.account_settings')]), edit_user_registration_url, highlights_on: %r{/auth/edit|/settings/delete|/settings/migration|/settings/aliases}
       s.item :two_factor_authentication, safe_join([fa_icon('mobile fw'), t('settings.two_factor_authentication')]), settings_two_factor_authentication_url, highlights_on: %r{/settings/two_factor_authentication}
       s.item :authorized_apps, safe_join([fa_icon('list fw'), t('settings.authorized_apps')]), oauth_authorized_applications_url
     end
 
-    n.item :data, safe_join([fa_icon('cloud-download fw'), t('settings.import_and_export')]), settings_export_url, if: -> { current_user.functional? } do |s|
-      s.item :import, safe_join([fa_icon('cloud-upload fw'), t('settings.import')]), settings_import_url
+    n.item :data, safe_join([fa_icon('cloud-download fw'), t('settings.import_and_export')]), settings_export_url do |s|
+      s.item :import, safe_join([fa_icon('cloud-upload fw'), t('settings.import')]), settings_import_url, if: -> { current_user.functional? }
       s.item :export, safe_join([fa_icon('cloud-download fw'), t('settings.export')]), settings_export_url
     end
 
diff --git a/config/routes.rb b/config/routes.rb
index dcfa079a0..37e0cbdee 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -134,8 +134,14 @@ Rails.application.routes.draw do
     end
 
     resource :delete, only: [:show, :destroy]
-    resource :migration, only: [:show, :update]
 
+    resource :migration, only: [:show, :create] do
+      collection do
+        post :cancel
+      end
+    end
+
+    resources :aliases, only: [:index, :create, :destroy]
     resources :sessions, only: [:destroy]
     resources :featured_tags, only: [:index, :create, :destroy]
   end
diff --git a/db/migrate/20190914202517_create_account_migrations.rb b/db/migrate/20190914202517_create_account_migrations.rb
new file mode 100644
index 000000000..cb9d71c09
--- /dev/null
+++ b/db/migrate/20190914202517_create_account_migrations.rb
@@ -0,0 +1,12 @@
+class CreateAccountMigrations < ActiveRecord::Migration[5.2]
+  def change
+    create_table :account_migrations do |t|
+      t.belongs_to :account, foreign_key: { on_delete: :cascade }
+      t.string :acct, null: false, default: ''
+      t.bigint :followers_count, null: false, default: 0
+      t.belongs_to :target_account, foreign_key: { to_table: :accounts, on_delete: :nullify }
+
+      t.timestamps
+    end
+  end
+end
diff --git a/db/migrate/20190915194355_create_account_aliases.rb b/db/migrate/20190915194355_create_account_aliases.rb
new file mode 100644
index 000000000..32ce031d9
--- /dev/null
+++ b/db/migrate/20190915194355_create_account_aliases.rb
@@ -0,0 +1,11 @@
+class CreateAccountAliases < ActiveRecord::Migration[5.2]
+  def change
+    create_table :account_aliases do |t|
+      t.belongs_to :account, foreign_key: { on_delete: :cascade }
+      t.string :acct, null: false, default: ''
+      t.string :uri, null: false, default: ''
+
+      t.timestamps
+    end
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 749f79dee..fabeb16f3 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -15,6 +15,15 @@ ActiveRecord::Schema.define(version: 2019_09_17_213523) do
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
 
+  create_table "account_aliases", force: :cascade do |t|
+    t.bigint "account_id"
+    t.string "acct", default: "", null: false
+    t.string "uri", default: "", null: false
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["account_id"], name: "index_account_aliases_on_account_id"
+  end
+
   create_table "account_conversations", force: :cascade do |t|
     t.bigint "account_id"
     t.bigint "conversation_id"
@@ -49,6 +58,17 @@ ActiveRecord::Schema.define(version: 2019_09_17_213523) do
     t.index ["account_id"], name: "index_account_identity_proofs_on_account_id"
   end
 
+  create_table "account_migrations", force: :cascade do |t|
+    t.bigint "account_id"
+    t.string "acct", default: "", null: false
+    t.bigint "followers_count", default: 0, null: false
+    t.bigint "target_account_id"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["account_id"], name: "index_account_migrations_on_account_id"
+    t.index ["target_account_id"], name: "index_account_migrations_on_target_account_id"
+  end
+
   create_table "account_moderation_notes", force: :cascade do |t|
     t.text "content", null: false
     t.bigint "account_id", null: false
@@ -768,10 +788,13 @@ ActiveRecord::Schema.define(version: 2019_09_17_213523) do
     t.index ["user_id"], name: "index_web_settings_on_user_id", unique: true
   end
 
+  add_foreign_key "account_aliases", "accounts", on_delete: :cascade
   add_foreign_key "account_conversations", "accounts", on_delete: :cascade
   add_foreign_key "account_conversations", "conversations", on_delete: :cascade
   add_foreign_key "account_domain_blocks", "accounts", name: "fk_206c6029bd", on_delete: :cascade
   add_foreign_key "account_identity_proofs", "accounts", on_delete: :cascade
+  add_foreign_key "account_migrations", "accounts", column: "target_account_id", on_delete: :nullify
+  add_foreign_key "account_migrations", "accounts", on_delete: :cascade
   add_foreign_key "account_moderation_notes", "accounts"
   add_foreign_key "account_moderation_notes", "accounts", column: "target_account_id"
   add_foreign_key "account_pins", "accounts", column: "target_account_id", on_delete: :cascade
diff --git a/spec/controllers/settings/migrations_controller_spec.rb b/spec/controllers/settings/migrations_controller_spec.rb
index 4d814a45e..36e4ba86e 100644
--- a/spec/controllers/settings/migrations_controller_spec.rb
+++ b/spec/controllers/settings/migrations_controller_spec.rb
@@ -21,6 +21,7 @@ describe Settings::MigrationsController do
 
       let(:user) { Fabricate(:user, account: account) }
       let(:account) { Fabricate(:account, moved_to_account: moved_to_account) }
+
       before { sign_in user, scope: :user }
 
       context 'when user does not have moved to account' do
@@ -32,7 +33,7 @@ describe Settings::MigrationsController do
         end
       end
 
-      context 'when user does not have moved to account' do
+      context 'when user has a moved to account' do
         let(:moved_to_account) { Fabricate(:account) }
 
         it 'renders show page' do
@@ -43,21 +44,22 @@ describe Settings::MigrationsController do
     end
   end
 
-  describe 'PUT #update' do
+  describe 'POST #create' do
     context 'when user is not sign in' do
-      subject { put :update }
+      subject { post :create }
 
       it_behaves_like 'authenticate user'
     end
 
     context 'when user is sign in' do
-      subject { put :update, params: { migration: { acct: acct } } }
+      subject { post :create, params: { account_migration: { acct: acct, current_password: '12345678' } } }
+
+      let(:user) { Fabricate(:user, password: '12345678') }
 
-      let(:user) { Fabricate(:user) }
       before { sign_in user, scope: :user }
 
       context 'when migration account is changed' do
-        let(:acct) { Fabricate(:account) }
+        let(:acct) { Fabricate(:account, also_known_as: [ActivityPub::TagManager.instance.uri_for(user.account)]) }
 
         it 'updates moved to account' do
           is_expected.to redirect_to settings_migration_path
diff --git a/spec/fabricators/account_alias_fabricator.rb b/spec/fabricators/account_alias_fabricator.rb
new file mode 100644
index 000000000..94dde9bb8
--- /dev/null
+++ b/spec/fabricators/account_alias_fabricator.rb
@@ -0,0 +1,5 @@
+Fabricator(:account_alias) do
+  account
+  acct 'test@example.com'
+  uri 'https://example.com/users/test'
+end
diff --git a/spec/fabricators/account_migration_fabricator.rb b/spec/fabricators/account_migration_fabricator.rb
new file mode 100644
index 000000000..3b3fc2077
--- /dev/null
+++ b/spec/fabricators/account_migration_fabricator.rb
@@ -0,0 +1,6 @@
+Fabricator(:account_migration) do
+  account
+  target_account
+  followers_count 1234
+  acct 'test@example.com'
+end
diff --git a/spec/models/account_alias_spec.rb b/spec/models/account_alias_spec.rb
new file mode 100644
index 000000000..27ec215aa
--- /dev/null
+++ b/spec/models/account_alias_spec.rb
@@ -0,0 +1,5 @@
+require 'rails_helper'
+
+RSpec.describe AccountAlias, type: :model do
+
+end
diff --git a/spec/models/account_migration_spec.rb b/spec/models/account_migration_spec.rb
new file mode 100644
index 000000000..8461b4b28
--- /dev/null
+++ b/spec/models/account_migration_spec.rb
@@ -0,0 +1,5 @@
+require 'rails_helper'
+
+RSpec.describe AccountMigration, type: :model do
+
+end
-- 
cgit 


From 27719a4001db8b4804ae301f0f262c7bce8cbcfa Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Sat, 28 Sep 2019 01:05:26 +0200
Subject: Fix older migrations not working due to new default scope (#11983)

Fix #11952, regression from #11623
---
 db/migrate/20161202132159_add_in_reply_to_account_id_to_statuses.rb | 2 +-
 db/migrate/20170209184350_add_reply_to_statuses.rb                  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

(limited to 'db/migrate')

diff --git a/db/migrate/20161202132159_add_in_reply_to_account_id_to_statuses.rb b/db/migrate/20161202132159_add_in_reply_to_account_id_to_statuses.rb
index 2c24b53d0..3a559ccd6 100644
--- a/db/migrate/20161202132159_add_in_reply_to_account_id_to_statuses.rb
+++ b/db/migrate/20161202132159_add_in_reply_to_account_id_to_statuses.rb
@@ -3,7 +3,7 @@ class AddInReplyToAccountIdToStatuses < ActiveRecord::Migration[5.0]
     add_column :statuses, :in_reply_to_account_id, :integer, null: true, default: nil
 
     ActiveRecord::Base.transaction do
-      Status.where.not(in_reply_to_id: nil).includes(:thread).find_each do |status|
+      Status.unscoped.where.not(in_reply_to_id: nil).includes(:thread).find_each do |status|
         next if status.thread.nil?
 
         status.in_reply_to_account_id = status.thread.account_id
diff --git a/db/migrate/20170209184350_add_reply_to_statuses.rb b/db/migrate/20170209184350_add_reply_to_statuses.rb
index c5074728b..b8b5c1306 100644
--- a/db/migrate/20170209184350_add_reply_to_statuses.rb
+++ b/db/migrate/20170209184350_add_reply_to_statuses.rb
@@ -1,7 +1,7 @@
 class AddReplyToStatuses < ActiveRecord::Migration[5.0]
   def up
     add_column :statuses, :reply, :boolean, nil: false, default: false
-    Status.update_all('reply = (in_reply_to_id IS NOT NULL)')
+    Status.unscoped.update_all('reply = (in_reply_to_id IS NOT NULL)')
   end
 
   def down
-- 
cgit 


From 3babf8464b0903b854ec16d355909444ef3ca0bc Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Sun, 29 Sep 2019 22:58:01 +0200
Subject: Add voters count support (#11917)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* Add voters count to polls

* Add ActivityPub serialization and parsing of voters count

* Add support for voters count in WebUI

* Move incrementation of voters count out of redis lock

* Reword “voters” to “people”
---
 app/javascript/mastodon/components/poll.js         | 19 +++++++---
 app/lib/activitypub/activity/create.rb             | 40 +++++++++++++++++++---
 app/lib/activitypub/adapter.rb                     |  1 +
 app/models/poll.rb                                 |  1 +
 app/serializers/activitypub/note_serializer.rb     | 12 ++++++-
 app/serializers/rest/poll_serializer.rb            |  2 +-
 app/services/activitypub/process_poll_service.rb   |  5 ++-
 app/services/post_status_service.rb                |  2 +-
 app/services/vote_service.rb                       | 32 +++++++++++++++--
 app/views/statuses/_poll.html.haml                 |  8 +++--
 config/locales/en.yml                              |  3 ++
 .../20190927232842_add_voters_count_to_polls.rb    |  5 +++
 db/schema.rb                                       |  3 +-
 13 files changed, 113 insertions(+), 20 deletions(-)
 create mode 100644 db/migrate/20190927232842_add_voters_count_to_polls.rb

(limited to 'db/migrate')

diff --git a/app/javascript/mastodon/components/poll.js b/app/javascript/mastodon/components/poll.js
index f88d260f2..cdbcf8f70 100644
--- a/app/javascript/mastodon/components/poll.js
+++ b/app/javascript/mastodon/components/poll.js
@@ -102,10 +102,11 @@ class Poll extends ImmutablePureComponent {
 
   renderOption (option, optionIndex, showResults) {
     const { poll, disabled, intl } = this.props;
-    const percent = poll.get('votes_count') === 0 ? 0 : (option.get('votes_count') / poll.get('votes_count')) * 100;
-    const leading = poll.get('options').filterNot(other => other.get('title') === option.get('title')).every(other => option.get('votes_count') >= other.get('votes_count'));
-    const active  = !!this.state.selected[`${optionIndex}`];
-    const voted   = option.get('voted') || (poll.get('own_votes') && poll.get('own_votes').includes(optionIndex));
+    const pollVotesCount  = poll.get('voters_count') || poll.get('votes_count');
+    const percent         = pollVotesCount === 0 ? 0 : (option.get('votes_count') / pollVotesCount) * 100;
+    const leading         = poll.get('options').filterNot(other => other.get('title') === option.get('title')).every(other => option.get('votes_count') >= other.get('votes_count'));
+    const active          = !!this.state.selected[`${optionIndex}`];
+    const voted           = option.get('voted') || (poll.get('own_votes') && poll.get('own_votes').includes(optionIndex));
 
     let titleEmojified = option.get('title_emojified');
     if (!titleEmojified) {
@@ -157,6 +158,14 @@ class Poll extends ImmutablePureComponent {
     const showResults   = poll.get('voted') || expired;
     const disabled      = this.props.disabled || Object.entries(this.state.selected).every(item => !item);
 
+    let votesCount = null;
+
+    if (poll.get('voters_count') !== null && poll.get('voters_count') !== undefined) {
+      votesCount = <FormattedMessage id='poll.total_people' defaultMessage='{count, plural, one {# person} other {# people}}' values={{ count: poll.get('voters_count') }} />;
+    } else {
+      votesCount = <FormattedMessage id='poll.total_votes' defaultMessage='{count, plural, one {# vote} other {# votes}}' values={{ count: poll.get('votes_count') }} />;
+    }
+
     return (
       <div className='poll'>
         <ul>
@@ -166,7 +175,7 @@ class Poll extends ImmutablePureComponent {
         <div className='poll__footer'>
           {!showResults && <button className='button button-secondary' disabled={disabled} onClick={this.handleVote}><FormattedMessage id='poll.vote' defaultMessage='Vote' /></button>}
           {showResults && !this.props.disabled && <span><button className='poll__link' onClick={this.handleRefresh}><FormattedMessage id='poll.refresh' defaultMessage='Refresh' /></button> · </span>}
-          <FormattedMessage id='poll.total_votes' defaultMessage='{count, plural, one {# vote} other {# votes}}' values={{ count: poll.get('votes_count') }} />
+          {votesCount}
           {poll.get('expires_at') && <span> · {timeRemaining}</span>}
         </div>
       </div>
diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index e69193b71..76bf9b2e5 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -232,25 +232,40 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
       items    = @object['oneOf']
     end
 
+    voters_count = @object['votersCount']
+
     @account.polls.new(
       multiple: multiple,
       expires_at: expires_at,
       options: items.map { |item| item['name'].presence || item['content'] }.compact,
-      cached_tallies: items.map { |item| item.dig('replies', 'totalItems') || 0 }
+      cached_tallies: items.map { |item| item.dig('replies', 'totalItems') || 0 },
+      voters_count: voters_count
     )
   end
 
   def poll_vote?
     return false if replied_to_status.nil? || replied_to_status.preloadable_poll.nil? || !replied_to_status.local? || !replied_to_status.preloadable_poll.options.include?(@object['name'])
 
-    unless replied_to_status.preloadable_poll.expired?
-      replied_to_status.preloadable_poll.votes.create!(account: @account, choice: replied_to_status.preloadable_poll.options.index(@object['name']), uri: @object['id'])
-      ActivityPub::DistributePollUpdateWorker.perform_in(3.minutes, replied_to_status.id) unless replied_to_status.preloadable_poll.hide_totals?
-    end
+    poll_vote! unless replied_to_status.preloadable_poll.expired?
 
     true
   end
 
+  def poll_vote!
+    poll = replied_to_status.preloadable_poll
+    already_voted = true
+    RedisLock.acquire(poll_lock_options) do |lock|
+      if lock.acquired?
+        already_voted = poll.votes.where(account: @account).exists?
+        poll.votes.create!(account: @account, choice: poll.options.index(@object['name']), uri: @object['id'])
+      else
+        raise Mastodon::RaceConditionError
+      end
+    end
+    increment_voters_count! unless already_voted
+    ActivityPub::DistributePollUpdateWorker.perform_in(3.minutes, replied_to_status.id) unless replied_to_status.preloadable_poll.hide_totals?
+  end
+
   def resolve_thread(status)
     return unless status.reply? && status.thread.nil? && Request.valid_url?(in_reply_to_uri)
     ThreadResolveWorker.perform_async(status.id, in_reply_to_uri)
@@ -416,7 +431,22 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     ActivityPub::RawDistributionWorker.perform_async(Oj.dump(@json), replied_to_status.account_id, [@account.preferred_inbox_url])
   end
 
+  def increment_voters_count!
+    poll = replied_to_status.preloadable_poll
+    unless poll.voters_count.nil?
+      poll.voters_count = poll.voters_count + 1
+      poll.save
+    end
+  rescue ActiveRecord::StaleObjectError
+    poll.reload
+    retry
+  end
+
   def lock_options
     { redis: Redis.current, key: "create:#{@object['id']}" }
   end
+
+  def poll_lock_options
+    { redis: Redis.current, key: "vote:#{replied_to_status.poll_id}:#{@account.id}" }
+  end
 end
diff --git a/app/lib/activitypub/adapter.rb b/app/lib/activitypub/adapter.rb
index cb2ac72d4..2a8f72333 100644
--- a/app/lib/activitypub/adapter.rb
+++ b/app/lib/activitypub/adapter.rb
@@ -21,6 +21,7 @@ class ActivityPub::Adapter < ActiveModelSerializers::Adapter::Base
     identity_proof: { 'toot' => 'http://joinmastodon.org/ns#', 'IdentityProof' => 'toot:IdentityProof' },
     blurhash: { 'toot' => 'http://joinmastodon.org/ns#', 'blurhash' => 'toot:blurhash' },
     discoverable: { 'toot' => 'http://joinmastodon.org/ns#', 'discoverable' => 'toot:discoverable' },
+    voters_count: { 'toot' => 'http://joinmastodon.org/ns#', 'votersCount' => 'toot:votersCount' },
   }.freeze
 
   def self.default_key_transform
diff --git a/app/models/poll.rb b/app/models/poll.rb
index 55a8f13a6..5427368fd 100644
--- a/app/models/poll.rb
+++ b/app/models/poll.rb
@@ -16,6 +16,7 @@
 #  created_at      :datetime         not null
 #  updated_at      :datetime         not null
 #  lock_version    :integer          default(0), not null
+#  voters_count    :bigint(8)
 #
 
 class Poll < ApplicationRecord
diff --git a/app/serializers/activitypub/note_serializer.rb b/app/serializers/activitypub/note_serializer.rb
index 364d3eda5..110621a28 100644
--- a/app/serializers/activitypub/note_serializer.rb
+++ b/app/serializers/activitypub/note_serializer.rb
@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 class ActivityPub::NoteSerializer < ActivityPub::Serializer
-  context_extensions :atom_uri, :conversation, :sensitive
+  context_extensions :atom_uri, :conversation, :sensitive, :voters_count
 
   attributes :id, :type, :summary,
              :in_reply_to, :published, :url,
@@ -23,6 +23,8 @@ class ActivityPub::NoteSerializer < ActivityPub::Serializer
   attribute :end_time, if: :poll_and_expires?
   attribute :closed, if: :poll_and_expired?
 
+  attribute :voters_count, if: :poll_and_voters_count?
+
   def id
     ActivityPub::TagManager.instance.uri_for(object)
   end
@@ -141,6 +143,10 @@ class ActivityPub::NoteSerializer < ActivityPub::Serializer
 
   alias end_time closed
 
+  def voters_count
+    object.preloadable_poll.voters_count
+  end
+
   def poll_and_expires?
     object.preloadable_poll&.expires_at&.present?
   end
@@ -149,6 +155,10 @@ class ActivityPub::NoteSerializer < ActivityPub::Serializer
     object.preloadable_poll&.expired?
   end
 
+  def poll_and_voters_count?
+    object.preloadable_poll&.voters_count
+  end
+
   class MediaAttachmentSerializer < ActivityPub::Serializer
     context_extensions :blurhash, :focal_point
 
diff --git a/app/serializers/rest/poll_serializer.rb b/app/serializers/rest/poll_serializer.rb
index eb98bb2d2..df6ebd0d4 100644
--- a/app/serializers/rest/poll_serializer.rb
+++ b/app/serializers/rest/poll_serializer.rb
@@ -2,7 +2,7 @@
 
 class REST::PollSerializer < ActiveModel::Serializer
   attributes :id, :expires_at, :expired,
-             :multiple, :votes_count
+             :multiple, :votes_count, :voters_count
 
   has_many :loaded_options, key: :options
   has_many :emojis, serializer: REST::CustomEmojiSerializer
diff --git a/app/services/activitypub/process_poll_service.rb b/app/services/activitypub/process_poll_service.rb
index 2fbce65b9..cb4a0d460 100644
--- a/app/services/activitypub/process_poll_service.rb
+++ b/app/services/activitypub/process_poll_service.rb
@@ -28,6 +28,8 @@ class ActivityPub::ProcessPollService < BaseService
       end
     end
 
+    voters_count = @json['votersCount']
+
     latest_options = items.map { |item| item['name'].presence || item['content'] }
 
     # If for some reasons the options were changed, it invalidates all previous
@@ -39,7 +41,8 @@ class ActivityPub::ProcessPollService < BaseService
         last_fetched_at: Time.now.utc,
         expires_at: expires_at,
         options: latest_options,
-        cached_tallies: items.map { |item| item.dig('replies', 'totalItems') || 0 }
+        cached_tallies: items.map { |item| item.dig('replies', 'totalItems') || 0 },
+        voters_count: voters_count
       )
     rescue ActiveRecord::StaleObjectError
       poll.reload
diff --git a/app/services/post_status_service.rb b/app/services/post_status_service.rb
index 34ec6d504..a0a650d62 100644
--- a/app/services/post_status_service.rb
+++ b/app/services/post_status_service.rb
@@ -174,7 +174,7 @@ class PostStatusService < BaseService
   def poll_attributes
     return if @options[:poll].blank?
 
-    @options[:poll].merge(account: @account)
+    @options[:poll].merge(account: @account, voters_count: 0)
   end
 
   def scheduled_options
diff --git a/app/services/vote_service.rb b/app/services/vote_service.rb
index 0eeb8fd56..cb7dce6e8 100644
--- a/app/services/vote_service.rb
+++ b/app/services/vote_service.rb
@@ -12,12 +12,24 @@ class VoteService < BaseService
     @choices = choices
     @votes   = []
 
-    ApplicationRecord.transaction do
-      @choices.each do |choice|
-        @votes << @poll.votes.create!(account: @account, choice: choice)
+    already_voted = true
+
+    RedisLock.acquire(lock_options) do |lock|
+      if lock.acquired?
+        already_voted = @poll.votes.where(account: @account).exists?
+
+        ApplicationRecord.transaction do
+          @choices.each do |choice|
+            @votes << @poll.votes.create!(account: @account, choice: choice)
+          end
+        end
+      else
+        raise Mastodon::RaceConditionError
       end
     end
 
+    increment_voters_count! unless already_voted
+
     ActivityTracker.increment('activity:interactions')
 
     if @poll.account.local?
@@ -53,4 +65,18 @@ class VoteService < BaseService
   def build_json(vote)
     Oj.dump(serialize_payload(vote, ActivityPub::VoteSerializer))
   end
+
+  def increment_voters_count!
+    unless @poll.voters_count.nil?
+      @poll.voters_count = @poll.voters_count + 1
+      @poll.save
+    end
+  rescue ActiveRecord::StaleObjectError
+    @poll.reload
+    retry
+  end
+
+  def lock_options
+    { redis: Redis.current, key: "vote:#{@poll.id}:#{@account.id}" }
+  end
 end
diff --git a/app/views/statuses/_poll.html.haml b/app/views/statuses/_poll.html.haml
index d6b36a5d1..d1aba6ef9 100644
--- a/app/views/statuses/_poll.html.haml
+++ b/app/views/statuses/_poll.html.haml
@@ -1,12 +1,13 @@
 - show_results = (user_signed_in? && poll.voted?(current_account)) || poll.expired?
 - own_votes = user_signed_in? ? poll.own_votes(current_account) : []
+- total_votes_count = poll.voters_count || poll.votes_count
 
 .poll
   %ul
     - poll.loaded_options.each_with_index do |option, index|
       %li
         - if show_results
-          - percent = poll.votes_count > 0 ? 100 * option.votes_count / poll.votes_count : 0
+          - percent = total_votes_count > 0 ? 100 * option.votes_count / total_votes_count : 0
           %span.poll__chart{ style: "width: #{percent}%" }
 
           %label.poll__text><
@@ -24,7 +25,10 @@
       %button.button.button-secondary{ disabled: true }
         = t('statuses.poll.vote')
 
-    %span= t('statuses.poll.total_votes', count: poll.votes_count)
+    - if poll.voters_count.nil?
+      %span= t('statuses.poll.total_votes', count: poll.votes_count)
+    - else
+      %span= t('statuses.poll.total_people', count: poll.voters_count)
 
     - unless poll.expires_at.nil?
       ·
diff --git a/config/locales/en.yml b/config/locales/en.yml
index dbdfe0ca0..82e20cb1f 100644
--- a/config/locales/en.yml
+++ b/config/locales/en.yml
@@ -1030,6 +1030,9 @@ en:
       private: Non-public toot cannot be pinned
       reblog: A boost cannot be pinned
     poll:
+      total_people:
+        one: "%{count} person"
+        other: "%{count} people"
       total_votes:
         one: "%{count} vote"
         other: "%{count} votes"
diff --git a/db/migrate/20190927232842_add_voters_count_to_polls.rb b/db/migrate/20190927232842_add_voters_count_to_polls.rb
new file mode 100644
index 000000000..846385700
--- /dev/null
+++ b/db/migrate/20190927232842_add_voters_count_to_polls.rb
@@ -0,0 +1,5 @@
+class AddVotersCountToPolls < ActiveRecord::Migration[5.2]
+  def change
+    add_column :polls, :voters_count, :bigint
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 8eeaf48a0..557b777e0 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_09_27_124642) do
+ActiveRecord::Schema.define(version: 2019_09_27_232842) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -529,6 +529,7 @@ ActiveRecord::Schema.define(version: 2019_09_27_124642) do
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
     t.integer "lock_version", default: 0, null: false
+    t.bigint "voters_count"
     t.index ["account_id"], name: "index_polls_on_account_id"
     t.index ["status_id"], name: "index_polls_on_status_id"
   end
-- 
cgit 


From 62f60e86c281ae1cd0356a7630dbb76069e2ffde Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Wed, 2 Oct 2019 04:59:37 +0200
Subject: Fix account counters being overwritten by parallel writes (#12045)

---
 app/models/account_stat.rb                         | 22 +++++++++
 ...1001213028_add_lock_version_to_account_stats.rb | 15 ++++++
 db/schema.rb                                       |  3 +-
 spec/models/account_stat_spec.rb                   | 53 ++++++++++++++++++++++
 4 files changed, 92 insertions(+), 1 deletion(-)
 create mode 100644 db/migrate/20191001213028_add_lock_version_to_account_stats.rb

(limited to 'db/migrate')

diff --git a/app/models/account_stat.rb b/app/models/account_stat.rb
index 1351f7d8a..c84e4217c 100644
--- a/app/models/account_stat.rb
+++ b/app/models/account_stat.rb
@@ -11,6 +11,7 @@
 #  created_at      :datetime         not null
 #  updated_at      :datetime         not null
 #  last_status_at  :datetime
+#  lock_version    :integer          default(0), not null
 #
 
 class AccountStat < ApplicationRecord
@@ -20,10 +21,26 @@ class AccountStat < ApplicationRecord
 
   def increment_count!(key)
     update(attributes_for_increment(key))
+  rescue ActiveRecord::StaleObjectError
+    begin
+      reload_with_id
+    rescue ActiveRecord::RecordNotFound
+      # Nothing to do
+    else
+      retry
+    end
   end
 
   def decrement_count!(key)
     update(key => [public_send(key) - 1, 0].max)
+  rescue ActiveRecord::StaleObjectError
+    begin
+      reload_with_id
+    rescue ActiveRecord::RecordNotFound
+      # Nothing to do
+    else
+      retry
+    end
   end
 
   private
@@ -33,4 +50,9 @@ class AccountStat < ApplicationRecord
     attrs[:last_status_at] = Time.now.utc if key == :statuses_count
     attrs
   end
+
+  def reload_with_id
+    self.id = find_by!(account: account).id if new_record?
+    reload
+  end
 end
diff --git a/db/migrate/20191001213028_add_lock_version_to_account_stats.rb b/db/migrate/20191001213028_add_lock_version_to_account_stats.rb
new file mode 100644
index 000000000..47f37cca2
--- /dev/null
+++ b/db/migrate/20191001213028_add_lock_version_to_account_stats.rb
@@ -0,0 +1,15 @@
+require Rails.root.join('lib', 'mastodon', 'migration_helpers')
+
+class AddLockVersionToAccountStats < ActiveRecord::Migration[5.2]
+  include Mastodon::MigrationHelpers
+
+  disable_ddl_transaction!
+
+  def up
+    safety_assured { add_column_with_default :account_stats, :lock_version, :integer, allow_null: false, default: 0 }
+  end
+
+  def down
+    remove_column :account_stats, :lock_version
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 557b777e0..2d516965c 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_09_27_232842) do
+ActiveRecord::Schema.define(version: 2019_10_01_213028) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -97,6 +97,7 @@ ActiveRecord::Schema.define(version: 2019_09_27_232842) do
     t.datetime "created_at", null: false
     t.datetime "updated_at", null: false
     t.datetime "last_status_at"
+    t.integer "lock_version", default: 0, null: false
     t.index ["account_id"], name: "index_account_stats_on_account_id", unique: true
   end
 
diff --git a/spec/models/account_stat_spec.rb b/spec/models/account_stat_spec.rb
index a94185109..8adc0d1d6 100644
--- a/spec/models/account_stat_spec.rb
+++ b/spec/models/account_stat_spec.rb
@@ -1,4 +1,57 @@
 require 'rails_helper'
 
 RSpec.describe AccountStat, type: :model do
+  describe '#increment_count!' do
+    it 'increments the count' do
+      account_stat = AccountStat.create(account: Fabricate(:account))
+      expect(account_stat.followers_count).to eq 0
+      account_stat.increment_count!(:followers_count)
+      expect(account_stat.followers_count).to eq 1
+    end
+
+    it 'increments the count in multi-threaded an environment' do
+      account_stat   = AccountStat.create(account: Fabricate(:account), statuses_count: 0)
+      increment_by   = 15
+      wait_for_start = true
+
+      threads = Array.new(increment_by) do
+        Thread.new do
+          true while wait_for_start
+          AccountStat.find(account_stat.id).increment_count!(:statuses_count)
+        end
+      end
+
+      wait_for_start = false
+      threads.each(&:join)
+
+      expect(account_stat.reload.statuses_count).to eq increment_by
+    end
+  end
+
+  describe '#decrement_count!' do
+    it 'decrements the count' do
+      account_stat = AccountStat.create(account: Fabricate(:account), followers_count: 15)
+      expect(account_stat.followers_count).to eq 15
+      account_stat.decrement_count!(:followers_count)
+      expect(account_stat.followers_count).to eq 14
+    end
+
+    it 'decrements the count in multi-threaded an environment' do
+      account_stat   = AccountStat.create(account: Fabricate(:account), statuses_count: 15)
+      decrement_by   = 10
+      wait_for_start = true
+
+      threads = Array.new(decrement_by) do
+        Thread.new do
+          true while wait_for_start
+          AccountStat.find(account_stat.id).decrement_count!(:statuses_count)
+        end
+      end
+
+      wait_for_start = false
+      threads.each(&:join)
+
+      expect(account_stat.reload.statuses_count).to eq 5
+    end
+  end
 end
-- 
cgit 


From b5be067c88151f75e09fc7d8b558ad4ccd51867b Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Mon, 7 Oct 2019 04:14:36 +0200
Subject: Fix existing user records with now-renamed `pt` locale (#12092)

Fix #12082
---
 db/migrate/20191007013357_update_pt_locales.rb | 11 +++++++++++
 db/schema.rb                                   | 26 +++++++++++++++++++++++++-
 2 files changed, 36 insertions(+), 1 deletion(-)
 create mode 100644 db/migrate/20191007013357_update_pt_locales.rb

(limited to 'db/migrate')

diff --git a/db/migrate/20191007013357_update_pt_locales.rb b/db/migrate/20191007013357_update_pt_locales.rb
new file mode 100644
index 000000000..b7288d38a
--- /dev/null
+++ b/db/migrate/20191007013357_update_pt_locales.rb
@@ -0,0 +1,11 @@
+class UpdatePtLocales < ActiveRecord::Migration[5.2]
+  disable_ddl_transaction!
+
+  def up
+    User.where(locale: 'pt').in_batches.update_all(locale: 'pt-PT')
+  end
+
+  def down
+    User.where(locale: 'pt-PT').in_batches.update_all(locale: 'pt')
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 2d516965c..62e2d3a6d 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_10_01_213028) do
+ActiveRecord::Schema.define(version: 2019_10_07_013357) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -693,6 +693,30 @@ ActiveRecord::Schema.define(version: 2019_10_01_213028) do
     t.index ["tag_id", "status_id"], name: "index_statuses_tags_on_tag_id_and_status_id", unique: true
   end
 
+  create_table "stream_entries", force: :cascade do |t|
+    t.bigint "activity_id"
+    t.string "activity_type"
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.boolean "hidden", default: false, null: false
+    t.bigint "account_id"
+    t.index ["account_id", "activity_type", "id"], name: "index_stream_entries_on_account_id_and_activity_type_and_id"
+    t.index ["activity_id", "activity_type"], name: "index_stream_entries_on_activity_id_and_activity_type"
+  end
+
+  create_table "subscriptions", force: :cascade do |t|
+    t.string "callback_url", default: "", null: false
+    t.string "secret"
+    t.datetime "expires_at"
+    t.boolean "confirmed", default: false, null: false
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.datetime "last_successful_delivery_at"
+    t.string "domain"
+    t.bigint "account_id", null: false
+    t.index ["account_id", "callback_url"], name: "index_subscriptions_on_account_id_and_callback_url", unique: true
+  end
+
   create_table "tags", force: :cascade do |t|
     t.string "name", default: "", null: false
     t.datetime "created_at", null: false
-- 
cgit 


From ebe574d5b57c647450cab3d4ca2e8ca52c4fb3fe Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Mon, 7 Oct 2019 06:05:14 +0200
Subject: Fix old migration trying to use new column due to default status
 scope (#12095)

Fix #12087
---
 db/migrate/20181024224956_migrate_account_conversations.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

(limited to 'db/migrate')

diff --git a/db/migrate/20181024224956_migrate_account_conversations.rb b/db/migrate/20181024224956_migrate_account_conversations.rb
index b718f9e1d..d4bc3b91d 100644
--- a/db/migrate/20181024224956_migrate_account_conversations.rb
+++ b/db/migrate/20181024224956_migrate_account_conversations.rb
@@ -52,6 +52,6 @@ class MigrateAccountConversations < ActiveRecord::Migration[5.2]
   end
 
   def notifications_about_direct_statuses
-    Notification.joins(mention: :status).where(activity_type: 'Mention', statuses: { visibility: :direct })
+    Notification.joins('INNER JOIN mentions ON mentions.id = notifications.activity_id INNER JOIN statuses ON statuses.id = mentions.status_id').where(activity_type: 'Mention', statuses: { visibility: :direct })
   end
 end
-- 
cgit 


From 65e13cfacf0d1a862fbc4da44ad84b1522b01b34 Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Mon, 4 Nov 2019 13:02:01 +0100
Subject: Add abilityto add oneself to lists (#12271)

* Add ability to add oneself to lists

* Change search results to include oneself when searching through followers

* Mark follow relation as optional in ListAccount
---
 app/models/account.rb                                            | 4 +++-
 app/models/list_account.rb                                       | 6 +++---
 app/services/account_search_service.rb                           | 2 +-
 db/migrate/20191031163205_change_list_account_follow_nullable.rb | 5 +++++
 db/schema.rb                                                     | 4 ++--
 5 files changed, 14 insertions(+), 7 deletions(-)
 create mode 100644 db/migrate/20191031163205_change_list_account_follow_nullable.rb

(limited to 'db/migrate')

diff --git a/app/models/account.rb b/app/models/account.rb
index 05936def3..cae82da16 100644
--- a/app/models/account.rb
+++ b/app/models/account.rb
@@ -429,6 +429,8 @@ class Account < ApplicationRecord
             SELECT target_account_id
             FROM follows
             WHERE account_id = ?
+            UNION ALL
+            SELECT ?
           )
           SELECT
             accounts.*,
@@ -444,7 +446,7 @@ class Account < ApplicationRecord
           LIMIT ? OFFSET ?
         SQL
 
-        records = find_by_sql([sql, account.id, account.id, account.id, limit, offset])
+        records = find_by_sql([sql, account.id, account.id, account.id, account.id, limit, offset])
       else
         sql = <<-SQL.squish
           SELECT
diff --git a/app/models/list_account.rb b/app/models/list_account.rb
index 87b498224..785923c4c 100644
--- a/app/models/list_account.rb
+++ b/app/models/list_account.rb
@@ -6,13 +6,13 @@
 #  id         :bigint(8)        not null, primary key
 #  list_id    :bigint(8)        not null
 #  account_id :bigint(8)        not null
-#  follow_id  :bigint(8)        not null
+#  follow_id  :bigint(8)
 #
 
 class ListAccount < ApplicationRecord
   belongs_to :list
   belongs_to :account
-  belongs_to :follow
+  belongs_to :follow, optional: true
 
   validates :account_id, uniqueness: { scope: :list_id }
 
@@ -21,6 +21,6 @@ class ListAccount < ApplicationRecord
   private
 
   def set_follow
-    self.follow = Follow.find_by(account_id: list.account_id, target_account_id: account.id)
+    self.follow = Follow.find_by!(account_id: list.account_id, target_account_id: account.id) unless list.account_id == account.id
   end
 end
diff --git a/app/services/account_search_service.rb b/app/services/account_search_service.rb
index 40c5f8590..7e74cc893 100644
--- a/app/services/account_search_service.rb
+++ b/app/services/account_search_service.rb
@@ -127,7 +127,7 @@ class AccountSearchService < BaseService
   end
 
   def following_ids
-    @following_ids ||= account.active_relationships.pluck(:target_account_id)
+    @following_ids ||= account.active_relationships.pluck(:target_account_id) + [account.id]
   end
 
   def limit_for_non_exact_results
diff --git a/db/migrate/20191031163205_change_list_account_follow_nullable.rb b/db/migrate/20191031163205_change_list_account_follow_nullable.rb
new file mode 100644
index 000000000..ff8911546
--- /dev/null
+++ b/db/migrate/20191031163205_change_list_account_follow_nullable.rb
@@ -0,0 +1,5 @@
+class ChangeListAccountFollowNullable < ActiveRecord::Migration[5.1]
+  def change
+    change_column_null :list_accounts, :follow_id, true
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 62e2d3a6d..1bcc003a5 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -10,7 +10,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 2019_10_07_013357) do
+ActiveRecord::Schema.define(version: 2019_10_31_163205) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -373,7 +373,7 @@ ActiveRecord::Schema.define(version: 2019_10_07_013357) do
   create_table "list_accounts", force: :cascade do |t|
     t.bigint "list_id", null: false
     t.bigint "account_id", null: false
-    t.bigint "follow_id", null: false
+    t.bigint "follow_id"
     t.index ["account_id", "list_id"], name: "index_list_accounts_on_account_id_and_list_id", unique: true
     t.index ["follow_id"], name: "index_list_accounts_on_follow_id"
     t.index ["list_id", "account_id"], name: "index_list_accounts_on_list_id_and_account_id"
-- 
cgit 


From dfea7368c934f600bd0b6b93b4a6c008a4e265b0 Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Wed, 13 Nov 2019 23:02:10 +0100
Subject: Add bookmarks (#7107)

* Add backend support for bookmarks

Bookmarks behave like favourites, except they aren't shared with other
users and do not have an associated counter.

* Add spec for bookmark endpoints

* Add front-end support for bookmarks

* Introduce OAuth scopes for bookmarks

* Add bookmarks to archive takeout

* Fix migration

* Coding style fixes

* Fix rebase issue

* Update bookmarked_statuses to latest UI changes

* Update bookmark actions to properly reflect status changes in state

* Add bookmarks item to single-column layout

* Make active bookmarks red
---
 app/controllers/api/v1/bookmarks_controller.rb     |  67 +++++++++++++
 .../api/v1/statuses/bookmarks_controller.rb        |  39 ++++++++
 app/javascript/mastodon/actions/bookmarks.js       |  90 ++++++++++++++++++
 app/javascript/mastodon/actions/interactions.js    |  80 ++++++++++++++++
 .../mastodon/components/status_action_bar.js       |   7 ++
 .../mastodon/containers/status_container.js        |  10 ++
 .../mastodon/features/bookmarked_statuses/index.js | 104 +++++++++++++++++++++
 .../mastodon/features/getting_started/index.js     |   4 +-
 .../features/status/components/action_bar.js       |   7 ++
 app/javascript/mastodon/features/status/index.js   |  11 +++
 .../features/ui/components/columns_area.js         |   2 +
 .../features/ui/components/navigation_panel.js     |   1 +
 app/javascript/mastodon/features/ui/index.js       |   2 +
 .../mastodon/features/ui/util/async-components.js  |   4 +
 app/javascript/mastodon/reducers/status_lists.js   |  29 ++++++
 app/javascript/mastodon/reducers/statuses.js       |   6 ++
 app/javascript/styles/mastodon/components.scss     |   4 +
 app/javascript/styles/mastodon/variables.scss      |   2 +
 app/models/bookmark.rb                             |  26 ++++++
 app/models/concerns/account_associations.rb        |   1 +
 app/models/concerns/account_interactions.rb        |   4 +
 app/models/status.rb                               |   5 +
 app/presenters/status_relationships_presenter.rb   |   2 +
 app/serializers/rest/status_serializer.rb          |   9 ++
 app/services/backup_service.rb                     |  21 +++++
 config/initializers/doorkeeper.rb                  |   2 +
 config/locales/doorkeeper.en.yml                   |   2 +
 config/routes.rb                                   |   4 +
 db/migrate/20180831171112_create_bookmarks.rb      |  17 ++++
 db/schema.rb                                       |  12 +++
 .../api/v1/bookmarks_controller_spec.rb            |  78 ++++++++++++++++
 .../api/v1/statuses/bookmarks_controller_spec.rb   |  57 +++++++++++
 spec/fabricators/bookmark_fabricator.rb            |   4 +
 33 files changed, 712 insertions(+), 1 deletion(-)
 create mode 100644 app/controllers/api/v1/bookmarks_controller.rb
 create mode 100644 app/controllers/api/v1/statuses/bookmarks_controller.rb
 create mode 100644 app/javascript/mastodon/actions/bookmarks.js
 create mode 100644 app/javascript/mastodon/features/bookmarked_statuses/index.js
 create mode 100644 app/models/bookmark.rb
 create mode 100644 db/migrate/20180831171112_create_bookmarks.rb
 create mode 100644 spec/controllers/api/v1/bookmarks_controller_spec.rb
 create mode 100644 spec/controllers/api/v1/statuses/bookmarks_controller_spec.rb
 create mode 100644 spec/fabricators/bookmark_fabricator.rb

(limited to 'db/migrate')

diff --git a/app/controllers/api/v1/bookmarks_controller.rb b/app/controllers/api/v1/bookmarks_controller.rb
new file mode 100644
index 000000000..cf4cba8dd
--- /dev/null
+++ b/app/controllers/api/v1/bookmarks_controller.rb
@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+class Api::V1::BookmarksController < Api::BaseController
+  before_action -> { doorkeeper_authorize! :read, :'read:bookmarks' }
+  before_action :require_user!
+  after_action :insert_pagination_headers
+
+  respond_to :json
+
+  def index
+    @statuses = load_statuses
+    render json: @statuses, each_serializer: REST::StatusSerializer, relationships: StatusRelationshipsPresenter.new(@statuses, current_user&.account_id)
+  end
+
+  private
+
+  def load_statuses
+    cached_bookmarks
+  end
+
+  def cached_bookmarks
+    cache_collection(
+      Status.reorder(nil).joins(:bookmarks).merge(results),
+      Status
+    )
+  end
+
+  def results
+    @_results ||= account_bookmarks.paginate_by_max_id(
+      limit_param(DEFAULT_STATUSES_LIMIT),
+      params[:max_id],
+      params[:since_id]
+    )
+  end
+
+  def account_bookmarks
+    current_account.bookmarks
+  end
+
+  def insert_pagination_headers
+    set_pagination_headers(next_path, prev_path)
+  end
+
+  def next_path
+    api_v1_bookmarks_url pagination_params(max_id: pagination_max_id) if records_continue?
+  end
+
+  def prev_path
+    api_v1_bookmarks_url pagination_params(since_id: pagination_since_id) unless results.empty?
+  end
+
+  def pagination_max_id
+    results.last.id
+  end
+
+  def pagination_since_id
+    results.first.id
+  end
+
+  def records_continue?
+    results.size == limit_param(DEFAULT_STATUSES_LIMIT)
+  end
+
+  def pagination_params(core_params)
+    params.slice(:limit).permit(:limit).merge(core_params)
+  end
+end
diff --git a/app/controllers/api/v1/statuses/bookmarks_controller.rb b/app/controllers/api/v1/statuses/bookmarks_controller.rb
new file mode 100644
index 000000000..bb9729cf5
--- /dev/null
+++ b/app/controllers/api/v1/statuses/bookmarks_controller.rb
@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+class Api::V1::Statuses::BookmarksController < Api::BaseController
+  include Authorization
+
+  before_action -> { doorkeeper_authorize! :write, :'write:bookmarks' }
+  before_action :require_user!
+
+  respond_to :json
+
+  def create
+    @status = bookmarked_status
+    render json: @status, serializer: REST::StatusSerializer
+  end
+
+  def destroy
+    @status = requested_status
+    @bookmarks_map = { @status.id => false }
+
+    bookmark = Bookmark.find_by!(account: current_user.account, status: @status)
+    bookmark.destroy!
+
+    render json: @status, serializer: REST::StatusSerializer, relationships: StatusRelationshipsPresenter.new([@status], current_user&.account_id, bookmarks_map: @bookmarks_map)
+  end
+
+  private
+
+  def bookmarked_status
+    authorize_with current_user.account, requested_status, :show?
+
+    bookmark = Bookmark.find_or_create_by!(account: current_user.account, status: requested_status)
+
+    bookmark.status.reload
+  end
+
+  def requested_status
+    Status.find(params[:status_id])
+  end
+end
diff --git a/app/javascript/mastodon/actions/bookmarks.js b/app/javascript/mastodon/actions/bookmarks.js
new file mode 100644
index 000000000..544ed2ff2
--- /dev/null
+++ b/app/javascript/mastodon/actions/bookmarks.js
@@ -0,0 +1,90 @@
+import api, { getLinks } from '../api';
+import { importFetchedStatuses } from './importer';
+
+export const BOOKMARKED_STATUSES_FETCH_REQUEST = 'BOOKMARKED_STATUSES_FETCH_REQUEST';
+export const BOOKMARKED_STATUSES_FETCH_SUCCESS = 'BOOKMARKED_STATUSES_FETCH_SUCCESS';
+export const BOOKMARKED_STATUSES_FETCH_FAIL    = 'BOOKMARKED_STATUSES_FETCH_FAIL';
+
+export const BOOKMARKED_STATUSES_EXPAND_REQUEST = 'BOOKMARKED_STATUSES_EXPAND_REQUEST';
+export const BOOKMARKED_STATUSES_EXPAND_SUCCESS = 'BOOKMARKED_STATUSES_EXPAND_SUCCESS';
+export const BOOKMARKED_STATUSES_EXPAND_FAIL    = 'BOOKMARKED_STATUSES_EXPAND_FAIL';
+
+export function fetchBookmarkedStatuses() {
+  return (dispatch, getState) => {
+    if (getState().getIn(['status_lists', 'bookmarks', 'isLoading'])) {
+      return;
+    }
+
+    dispatch(fetchBookmarkedStatusesRequest());
+
+    api(getState).get('/api/v1/bookmarks').then(response => {
+      const next = getLinks(response).refs.find(link => link.rel === 'next');
+      dispatch(importFetchedStatuses(response.data));
+      dispatch(fetchBookmarkedStatusesSuccess(response.data, next ? next.uri : null));
+    }).catch(error => {
+      dispatch(fetchBookmarkedStatusesFail(error));
+    });
+  };
+};
+
+export function fetchBookmarkedStatusesRequest() {
+  return {
+    type: BOOKMARKED_STATUSES_FETCH_REQUEST,
+  };
+};
+
+export function fetchBookmarkedStatusesSuccess(statuses, next) {
+  return {
+    type: BOOKMARKED_STATUSES_FETCH_SUCCESS,
+    statuses,
+    next,
+  };
+};
+
+export function fetchBookmarkedStatusesFail(error) {
+  return {
+    type: BOOKMARKED_STATUSES_FETCH_FAIL,
+    error,
+  };
+};
+
+export function expandBookmarkedStatuses() {
+  return (dispatch, getState) => {
+    const url = getState().getIn(['status_lists', 'bookmarks', 'next'], null);
+
+    if (url === null || getState().getIn(['status_lists', 'bookmarks', 'isLoading'])) {
+      return;
+    }
+
+    dispatch(expandBookmarkedStatusesRequest());
+
+    api(getState).get(url).then(response => {
+      const next = getLinks(response).refs.find(link => link.rel === 'next');
+      dispatch(importFetchedStatuses(response.data));
+      dispatch(expandBookmarkedStatusesSuccess(response.data, next ? next.uri : null));
+    }).catch(error => {
+      dispatch(expandBookmarkedStatusesFail(error));
+    });
+  };
+};
+
+export function expandBookmarkedStatusesRequest() {
+  return {
+    type: BOOKMARKED_STATUSES_EXPAND_REQUEST,
+  };
+};
+
+export function expandBookmarkedStatusesSuccess(statuses, next) {
+  return {
+    type: BOOKMARKED_STATUSES_EXPAND_SUCCESS,
+    statuses,
+    next,
+  };
+};
+
+export function expandBookmarkedStatusesFail(error) {
+  return {
+    type: BOOKMARKED_STATUSES_EXPAND_FAIL,
+    error,
+  };
+};
diff --git a/app/javascript/mastodon/actions/interactions.js b/app/javascript/mastodon/actions/interactions.js
index 2dc4c574c..28c6b1a62 100644
--- a/app/javascript/mastodon/actions/interactions.js
+++ b/app/javascript/mastodon/actions/interactions.js
@@ -33,6 +33,14 @@ export const UNPIN_REQUEST = 'UNPIN_REQUEST';
 export const UNPIN_SUCCESS = 'UNPIN_SUCCESS';
 export const UNPIN_FAIL    = 'UNPIN_FAIL';
 
+export const BOOKMARK_REQUEST = 'BOOKMARK_REQUEST';
+export const BOOKMARK_SUCCESS = 'BOOKMARKED_SUCCESS';
+export const BOOKMARK_FAIL    = 'BOOKMARKED_FAIL';
+
+export const UNBOOKMARK_REQUEST = 'UNBOOKMARKED_REQUEST';
+export const UNBOOKMARK_SUCCESS = 'UNBOOKMARKED_SUCCESS';
+export const UNBOOKMARK_FAIL    = 'UNBOOKMARKED_FAIL';
+
 export function reblog(status) {
   return function (dispatch, getState) {
     dispatch(reblogRequest(status));
@@ -187,6 +195,78 @@ export function unfavouriteFail(status, error) {
   };
 };
 
+export function bookmark(status) {
+  return function (dispatch, getState) {
+    dispatch(bookmarkRequest(status));
+
+    api(getState).post(`/api/v1/statuses/${status.get('id')}/bookmark`).then(function (response) {
+      dispatch(importFetchedStatus(response.data));
+      dispatch(bookmarkSuccess(status, response.data));
+    }).catch(function (error) {
+      dispatch(bookmarkFail(status, error));
+    });
+  };
+};
+
+export function unbookmark(status) {
+  return (dispatch, getState) => {
+    dispatch(unbookmarkRequest(status));
+
+    api(getState).post(`/api/v1/statuses/${status.get('id')}/unbookmark`).then(response => {
+      dispatch(importFetchedStatus(response.data));
+      dispatch(unbookmarkSuccess(status, response.data));
+    }).catch(error => {
+      dispatch(unbookmarkFail(status, error));
+    });
+  };
+};
+
+export function bookmarkRequest(status) {
+  return {
+    type: BOOKMARK_REQUEST,
+    status: status,
+  };
+};
+
+export function bookmarkSuccess(status, response) {
+  return {
+    type: BOOKMARK_SUCCESS,
+    status: status,
+    response: response,
+  };
+};
+
+export function bookmarkFail(status, error) {
+  return {
+    type: BOOKMARK_FAIL,
+    status: status,
+    error: error,
+  };
+};
+
+export function unbookmarkRequest(status) {
+  return {
+    type: UNBOOKMARK_REQUEST,
+    status: status,
+  };
+};
+
+export function unbookmarkSuccess(status, response) {
+  return {
+    type: UNBOOKMARK_SUCCESS,
+    status: status,
+    response: response,
+  };
+};
+
+export function unbookmarkFail(status, error) {
+  return {
+    type: UNBOOKMARK_FAIL,
+    status: status,
+    error: error,
+  };
+};
+
 export function fetchReblogs(id) {
   return (dispatch, getState) => {
     dispatch(fetchReblogsRequest(id));
diff --git a/app/javascript/mastodon/components/status_action_bar.js b/app/javascript/mastodon/components/status_action_bar.js
index 0bfbd8879..4fa2c1158 100644
--- a/app/javascript/mastodon/components/status_action_bar.js
+++ b/app/javascript/mastodon/components/status_action_bar.js
@@ -23,6 +23,7 @@ const messages = defineMessages({
   cancel_reblog_private: { id: 'status.cancel_reblog_private', defaultMessage: 'Unboost' },
   cannot_reblog: { id: 'status.cannot_reblog', defaultMessage: 'This post cannot be boosted' },
   favourite: { id: 'status.favourite', defaultMessage: 'Favourite' },
+  bookmark: { id: 'status.bookmark', defaultMessage: 'Bookmark' },
   open: { id: 'status.open', defaultMessage: 'Expand this status' },
   report: { id: 'status.report', defaultMessage: 'Report @{name}' },
   muteConversation: { id: 'status.mute_conversation', defaultMessage: 'Mute conversation' },
@@ -66,6 +67,7 @@ class StatusActionBar extends ImmutablePureComponent {
     onEmbed: PropTypes.func,
     onMuteConversation: PropTypes.func,
     onPin: PropTypes.func,
+    onBookmark: PropTypes.func,
     withDismiss: PropTypes.bool,
     intl: PropTypes.object.isRequired,
   };
@@ -114,6 +116,10 @@ class StatusActionBar extends ImmutablePureComponent {
     window.open(`/interact/${this.props.status.get('id')}?type=${type}`, 'mastodon-intent', 'width=445,height=600,resizable=no,menubar=no,status=no,scrollbars=yes');
   }
 
+  handleBookmarkClick = () => {
+    this.props.onBookmark(this.props.status);
+  }
+
   handleDeleteClick = () => {
     this.props.onDelete(this.props.status, this.context.router.history);
   }
@@ -253,6 +259,7 @@ class StatusActionBar extends ImmutablePureComponent {
         <IconButton className='status__action-bar-button' disabled={!publicStatus} active={status.get('reblogged')} pressed={status.get('reblogged')} title={!publicStatus ? intl.formatMessage(messages.cannot_reblog) : intl.formatMessage(messages.reblog)} icon={reblogIcon} onClick={this.handleReblogClick} />
         <IconButton className='status__action-bar-button star-icon' animate active={status.get('favourited')} pressed={status.get('favourited')} title={intl.formatMessage(messages.favourite)} icon='star' onClick={this.handleFavouriteClick} />
         {shareButton}
+        <IconButton className='status__action-bar-button bookmark-icon' disabled={anonymousAccess} active={status.get('bookmarked')} pressed={status.get('bookmarked')} title={intl.formatMessage(messages.bookmark)} icon='bookmark' onClick={this.handleBookmarkClick} />
 
         <div className='status__action-bar-dropdown'>
           <DropdownMenuContainer disabled={anonymousAccess} status={status} items={menu} icon='ellipsis-h' size={18} direction='right' title={intl.formatMessage(messages.more)} />
diff --git a/app/javascript/mastodon/containers/status_container.js b/app/javascript/mastodon/containers/status_container.js
index fb22676e0..16ba02e12 100644
--- a/app/javascript/mastodon/containers/status_container.js
+++ b/app/javascript/mastodon/containers/status_container.js
@@ -9,8 +9,10 @@ import {
 import {
   reblog,
   favourite,
+  bookmark,
   unreblog,
   unfavourite,
+  unbookmark,
   pin,
   unpin,
 } from '../actions/interactions';
@@ -90,6 +92,14 @@ const mapDispatchToProps = (dispatch, { intl }) => ({
     }
   },
 
+  onBookmark (status) {
+    if (status.get('bookmarked')) {
+      dispatch(unbookmark(status));
+    } else {
+      dispatch(bookmark(status));
+    }
+  },
+
   onPin (status) {
     if (status.get('pinned')) {
       dispatch(unpin(status));
diff --git a/app/javascript/mastodon/features/bookmarked_statuses/index.js b/app/javascript/mastodon/features/bookmarked_statuses/index.js
new file mode 100644
index 000000000..c37cb9176
--- /dev/null
+++ b/app/javascript/mastodon/features/bookmarked_statuses/index.js
@@ -0,0 +1,104 @@
+import React from 'react';
+import { connect } from 'react-redux';
+import PropTypes from 'prop-types';
+import ImmutablePropTypes from 'react-immutable-proptypes';
+import { fetchBookmarkedStatuses, expandBookmarkedStatuses } from '../../actions/bookmarks';
+import Column from '../ui/components/column';
+import ColumnHeader from '../../components/column_header';
+import { addColumn, removeColumn, moveColumn } from '../../actions/columns';
+import StatusList from '../../components/status_list';
+import { defineMessages, injectIntl, FormattedMessage } from 'react-intl';
+import ImmutablePureComponent from 'react-immutable-pure-component';
+import { debounce } from 'lodash';
+
+const messages = defineMessages({
+  heading: { id: 'column.bookmarks', defaultMessage: 'Bookmarks' },
+});
+
+const mapStateToProps = state => ({
+  statusIds: state.getIn(['status_lists', 'bookmarks', 'items']),
+  isLoading: state.getIn(['status_lists', 'bookmarks', 'isLoading'], true),
+  hasMore: !!state.getIn(['status_lists', 'bookmarks', 'next']),
+});
+
+export default @connect(mapStateToProps)
+@injectIntl
+class Bookmarks extends ImmutablePureComponent {
+
+  static propTypes = {
+    dispatch: PropTypes.func.isRequired,
+    shouldUpdateScroll: PropTypes.func,
+    statusIds: ImmutablePropTypes.list.isRequired,
+    intl: PropTypes.object.isRequired,
+    columnId: PropTypes.string,
+    multiColumn: PropTypes.bool,
+    hasMore: PropTypes.bool,
+    isLoading: PropTypes.bool,
+  };
+
+  componentWillMount () {
+    this.props.dispatch(fetchBookmarkedStatuses());
+  }
+
+  handlePin = () => {
+    const { columnId, dispatch } = this.props;
+
+    if (columnId) {
+      dispatch(removeColumn(columnId));
+    } else {
+      dispatch(addColumn('BOOKMARKS', {}));
+    }
+  }
+
+  handleMove = (dir) => {
+    const { columnId, dispatch } = this.props;
+    dispatch(moveColumn(columnId, dir));
+  }
+
+  handleHeaderClick = () => {
+    this.column.scrollTop();
+  }
+
+  setRef = c => {
+    this.column = c;
+  }
+
+  handleLoadMore = debounce(() => {
+    this.props.dispatch(expandBookmarkedStatuses());
+  }, 300, { leading: true })
+
+  render () {
+    const { intl, shouldUpdateScroll, statusIds, columnId, multiColumn, hasMore, isLoading } = this.props;
+    const pinned = !!columnId;
+
+    const emptyMessage = <FormattedMessage id='empty_column.bookmarked_statuses' defaultMessage="You don't have any bookmarked toots yet. When you bookmark one, it will show up here." />;
+
+    return (
+      <Column bindToDocument={!multiColumn} ref={this.setRef} label={intl.formatMessage(messages.heading)}>
+        <ColumnHeader
+          icon='bookmark'
+          title={intl.formatMessage(messages.heading)}
+          onPin={this.handlePin}
+          onMove={this.handleMove}
+          onClick={this.handleHeaderClick}
+          pinned={pinned}
+          multiColumn={multiColumn}
+          showBackButton
+        />
+
+        <StatusList
+          trackScroll={!pinned}
+          statusIds={statusIds}
+          scrollKey={`bookmarked_statuses-${columnId}`}
+          hasMore={hasMore}
+          isLoading={isLoading}
+          onLoadMore={this.handleLoadMore}
+          shouldUpdateScroll={shouldUpdateScroll}
+          emptyMessage={emptyMessage}
+          bindToDocument={!multiColumn}
+        />
+      </Column>
+    );
+  }
+
+}
diff --git a/app/javascript/mastodon/features/getting_started/index.js b/app/javascript/mastodon/features/getting_started/index.js
index 67ec7665b..adbc147d1 100644
--- a/app/javascript/mastodon/features/getting_started/index.js
+++ b/app/javascript/mastodon/features/getting_started/index.js
@@ -22,6 +22,7 @@ const messages = defineMessages({
   settings_subheading: { id: 'column_subheading.settings', defaultMessage: 'Settings' },
   community_timeline: { id: 'navigation_bar.community_timeline', defaultMessage: 'Local timeline' },
   direct: { id: 'navigation_bar.direct', defaultMessage: 'Direct messages' },
+  bookmarks: { id: 'navigation_bar.bookmarks', defaultMessage: 'Bookmarks' },
   preferences: { id: 'navigation_bar.preferences', defaultMessage: 'Preferences' },
   follow_requests: { id: 'navigation_bar.follow_requests', defaultMessage: 'Follow requests' },
   favourites: { id: 'navigation_bar.favourites', defaultMessage: 'Favourites' },
@@ -126,11 +127,12 @@ class GettingStarted extends ImmutablePureComponent {
 
     navItems.push(
       <ColumnLink key={i++} icon='envelope' text={intl.formatMessage(messages.direct)} to='/timelines/direct' />,
+      <ColumnLink key={i++} icon='bookmark' text={intl.formatMessage(messages.bookmarks)} to='/bookmarks' />,
       <ColumnLink key={i++} icon='star' text={intl.formatMessage(messages.favourites)} to='/favourites' />,
       <ColumnLink key={i++} icon='list-ul' text={intl.formatMessage(messages.lists)} to='/lists' />
     );
 
-    height += 48*3;
+    height += 48*4;
 
     if (myAccount.get('locked') || unreadFollowRequests > 0) {
       navItems.push(<ColumnLink key={i++} icon='user-plus' text={intl.formatMessage(messages.follow_requests)} badge={badgeDisplay(unreadFollowRequests, 40)} to='/follow_requests' />);
diff --git a/app/javascript/mastodon/features/status/components/action_bar.js b/app/javascript/mastodon/features/status/components/action_bar.js
index 3e511b7a6..1b81cd245 100644
--- a/app/javascript/mastodon/features/status/components/action_bar.js
+++ b/app/javascript/mastodon/features/status/components/action_bar.js
@@ -17,6 +17,7 @@ const messages = defineMessages({
   cancel_reblog_private: { id: 'status.cancel_reblog_private', defaultMessage: 'Unboost' },
   cannot_reblog: { id: 'status.cannot_reblog', defaultMessage: 'This post cannot be boosted' },
   favourite: { id: 'status.favourite', defaultMessage: 'Favourite' },
+  bookmark: { id: 'status.bookmark', defaultMessage: 'Bookmark' },
   mute: { id: 'status.mute', defaultMessage: 'Mute @{name}' },
   muteConversation: { id: 'status.mute_conversation', defaultMessage: 'Mute conversation' },
   unmuteConversation: { id: 'status.unmute_conversation', defaultMessage: 'Unmute conversation' },
@@ -43,6 +44,7 @@ class ActionBar extends React.PureComponent {
     onReply: PropTypes.func.isRequired,
     onReblog: PropTypes.func.isRequired,
     onFavourite: PropTypes.func.isRequired,
+    onBookmark: PropTypes.func.isRequired,
     onDelete: PropTypes.func.isRequired,
     onDirect: PropTypes.func.isRequired,
     onMention: PropTypes.func.isRequired,
@@ -67,6 +69,10 @@ class ActionBar extends React.PureComponent {
     this.props.onFavourite(this.props.status);
   }
 
+  handleBookmarkClick = (e) => {
+    this.props.onBookmark(this.props.status, e);
+  }
+
   handleDeleteClick = () => {
     this.props.onDelete(this.props.status, this.context.router.history);
   }
@@ -198,6 +204,7 @@ class ActionBar extends React.PureComponent {
         <div className='detailed-status__button'><IconButton disabled={reblog_disabled} active={status.get('reblogged')} title={reblog_disabled ? intl.formatMessage(messages.cannot_reblog) : intl.formatMessage(messages.reblog)} icon={reblogIcon} onClick={this.handleReblogClick} /></div>
         <div className='detailed-status__button'><IconButton className='star-icon' animate active={status.get('favourited')} title={intl.formatMessage(messages.favourite)} icon='star' onClick={this.handleFavouriteClick} /></div>
         {shareButton}
+        <div className='detailed-status__button'><IconButton className='bookmark-icon' active={status.get('bookmarked')} title={intl.formatMessage(messages.bookmark)} icon='bookmark' onClick={this.handleBookmarkClick} /></div>
 
         <div className='detailed-status__action-bar-dropdown'>
           <DropdownMenuContainer size={18} icon='ellipsis-h' items={menu} direction='left' title='More' />
diff --git a/app/javascript/mastodon/features/status/index.js b/app/javascript/mastodon/features/status/index.js
index 029057d40..9fb3fe305 100644
--- a/app/javascript/mastodon/features/status/index.js
+++ b/app/javascript/mastodon/features/status/index.js
@@ -13,6 +13,8 @@ import Column from '../ui/components/column';
 import {
   favourite,
   unfavourite,
+  bookmark,
+  unbookmark,
   reblog,
   unreblog,
   pin,
@@ -232,6 +234,14 @@ class Status extends ImmutablePureComponent {
     }
   }
 
+  handleBookmarkClick = (status) => {
+    if (status.get('bookmarked')) {
+      this.props.dispatch(unbookmark(status));
+    } else {
+      this.props.dispatch(bookmark(status));
+    }
+  }
+
   handleDeleteClick = (status, history, withRedraft = false) => {
     const { dispatch, intl } = this.props;
 
@@ -499,6 +509,7 @@ class Status extends ImmutablePureComponent {
                   onReply={this.handleReplyClick}
                   onFavourite={this.handleFavouriteClick}
                   onReblog={this.handleReblogClick}
+                  onBookmark={this.handleBookmarkClick}
                   onDelete={this.handleDeleteClick}
                   onDirect={this.handleDirectClick}
                   onMention={this.handleMentionClick}
diff --git a/app/javascript/mastodon/features/ui/components/columns_area.js b/app/javascript/mastodon/features/ui/components/columns_area.js
index f31425c70..8bc4bfc0e 100644
--- a/app/javascript/mastodon/features/ui/components/columns_area.js
+++ b/app/javascript/mastodon/features/ui/components/columns_area.js
@@ -21,6 +21,7 @@ import {
   HashtagTimeline,
   DirectTimeline,
   FavouritedStatuses,
+  BookmarkedStatuses,
   ListTimeline,
   Directory,
 } from '../../ui/util/async-components';
@@ -40,6 +41,7 @@ const componentMap = {
   'HASHTAG': HashtagTimeline,
   'DIRECT': DirectTimeline,
   'FAVOURITES': FavouritedStatuses,
+  'BOOKMARKS': BookmarkedStatuses,
   'LIST': ListTimeline,
   'DIRECTORY': Directory,
 };
diff --git a/app/javascript/mastodon/features/ui/components/navigation_panel.js b/app/javascript/mastodon/features/ui/components/navigation_panel.js
index 51e3ec037..0c12852f5 100644
--- a/app/javascript/mastodon/features/ui/components/navigation_panel.js
+++ b/app/javascript/mastodon/features/ui/components/navigation_panel.js
@@ -17,6 +17,7 @@ const NavigationPanel = () => (
     <NavLink className='column-link column-link--transparent' exact to='/timelines/public' data-preview-title-id='column.public' data-preview-icon='globe' ><Icon className='column-link__icon' id='globe' fixedWidth /><FormattedMessage id='tabs_bar.federated_timeline' defaultMessage='Federated' /></NavLink>
     <NavLink className='column-link column-link--transparent' to='/timelines/direct'><Icon className='column-link__icon' id='envelope' fixedWidth /><FormattedMessage id='navigation_bar.direct' defaultMessage='Direct messages' /></NavLink>
     <NavLink className='column-link column-link--transparent' to='/favourites'><Icon className='column-link__icon' id='star' fixedWidth /><FormattedMessage id='navigation_bar.favourites' defaultMessage='Favourites' /></NavLink>
+    <NavLink className='column-link column-link--transparent' to='/bookmarks'><Icon className='column-link__icon' id='bookmark' fixedWidth /><FormattedMessage id='navigation_bar.bookmarks' defaultMessage='Bookmarks' /></NavLink>
     <NavLink className='column-link column-link--transparent' to='/lists'><Icon className='column-link__icon' id='list-ul' fixedWidth /><FormattedMessage id='navigation_bar.lists' defaultMessage='Lists' /></NavLink>
     {profile_directory && <NavLink className='column-link column-link--transparent' to='/directory'><Icon className='column-link__icon' id='address-book-o' fixedWidth /><FormattedMessage id='getting_started.directory' defaultMessage='Profile directory' /></NavLink>}
 
diff --git a/app/javascript/mastodon/features/ui/index.js b/app/javascript/mastodon/features/ui/index.js
index a45ba91eb..b0e38c5cb 100644
--- a/app/javascript/mastodon/features/ui/index.js
+++ b/app/javascript/mastodon/features/ui/index.js
@@ -41,6 +41,7 @@ import {
   FollowRequests,
   GenericNotFound,
   FavouritedStatuses,
+  BookmarkedStatuses,
   ListTimeline,
   Blocks,
   DomainBlocks,
@@ -190,6 +191,7 @@ class SwitchingColumnsArea extends React.PureComponent {
 
           <WrappedRoute path='/notifications' component={Notifications} content={children} componentParams={{ shouldUpdateScroll: this.shouldUpdateScroll }} />
           <WrappedRoute path='/favourites' component={FavouritedStatuses} content={children} componentParams={{ shouldUpdateScroll: this.shouldUpdateScroll }} />
+          <WrappedRoute path='/bookmarks' component={BookmarkedStatuses} content={children} />
           <WrappedRoute path='/pinned' component={PinnedStatuses} content={children} componentParams={{ shouldUpdateScroll: this.shouldUpdateScroll }} />
 
           <WrappedRoute path='/search' component={Search} content={children} />
diff --git a/app/javascript/mastodon/features/ui/util/async-components.js b/app/javascript/mastodon/features/ui/util/async-components.js
index bb0fcb859..986efda1e 100644
--- a/app/javascript/mastodon/features/ui/util/async-components.js
+++ b/app/javascript/mastodon/features/ui/util/async-components.js
@@ -90,6 +90,10 @@ export function FavouritedStatuses () {
   return import(/* webpackChunkName: "features/favourited_statuses" */'../../favourited_statuses');
 }
 
+export function BookmarkedStatuses () {
+  return import(/* webpackChunkName: "features/bookmarked_statuses" */'../../bookmarked_statuses');
+}
+
 export function Blocks () {
   return import(/* webpackChunkName: "features/blocks" */'../../blocks');
 }
diff --git a/app/javascript/mastodon/reducers/status_lists.js b/app/javascript/mastodon/reducers/status_lists.js
index 6c5f33557..9f8f28dee 100644
--- a/app/javascript/mastodon/reducers/status_lists.js
+++ b/app/javascript/mastodon/reducers/status_lists.js
@@ -6,6 +6,14 @@ import {
   FAVOURITED_STATUSES_EXPAND_SUCCESS,
   FAVOURITED_STATUSES_EXPAND_FAIL,
 } from '../actions/favourites';
+import {
+  BOOKMARKED_STATUSES_FETCH_REQUEST,
+  BOOKMARKED_STATUSES_FETCH_SUCCESS,
+  BOOKMARKED_STATUSES_FETCH_FAIL,
+  BOOKMARKED_STATUSES_EXPAND_REQUEST,
+  BOOKMARKED_STATUSES_EXPAND_SUCCESS,
+  BOOKMARKED_STATUSES_EXPAND_FAIL,
+} from '../actions/bookmarks';
 import {
   PINNED_STATUSES_FETCH_SUCCESS,
 } from '../actions/pin_statuses';
@@ -13,6 +21,8 @@ import { Map as ImmutableMap, List as ImmutableList } from 'immutable';
 import {
   FAVOURITE_SUCCESS,
   UNFAVOURITE_SUCCESS,
+  BOOKMARK_SUCCESS,
+  UNBOOKMARK_SUCCESS,
   PIN_SUCCESS,
   UNPIN_SUCCESS,
 } from '../actions/interactions';
@@ -23,6 +33,11 @@ const initialState = ImmutableMap({
     loaded: false,
     items: ImmutableList(),
   }),
+  bookmarks: ImmutableMap({
+    next: null,
+    loaded: false,
+    items: ImmutableList(),
+  }),
   pins: ImmutableMap({
     next: null,
     loaded: false,
@@ -71,10 +86,24 @@ export default function statusLists(state = initialState, action) {
     return normalizeList(state, 'favourites', action.statuses, action.next);
   case FAVOURITED_STATUSES_EXPAND_SUCCESS:
     return appendToList(state, 'favourites', action.statuses, action.next);
+  case BOOKMARKED_STATUSES_FETCH_REQUEST:
+  case BOOKMARKED_STATUSES_EXPAND_REQUEST:
+    return state.setIn(['bookmarks', 'isLoading'], true);
+  case BOOKMARKED_STATUSES_FETCH_FAIL:
+  case BOOKMARKED_STATUSES_EXPAND_FAIL:
+    return state.setIn(['bookmarks', 'isLoading'], false);
+  case BOOKMARKED_STATUSES_FETCH_SUCCESS:
+    return normalizeList(state, 'bookmarks', action.statuses, action.next);
+  case BOOKMARKED_STATUSES_EXPAND_SUCCESS:
+    return appendToList(state, 'bookmarks', action.statuses, action.next);
   case FAVOURITE_SUCCESS:
     return prependOneToList(state, 'favourites', action.status);
   case UNFAVOURITE_SUCCESS:
     return removeOneFromList(state, 'favourites', action.status);
+  case BOOKMARK_SUCCESS:
+    return prependOneToList(state, 'bookmarks', action.status);
+  case UNBOOKMARK_SUCCESS:
+    return removeOneFromList(state, 'bookmarks', action.status);
   case PINNED_STATUSES_FETCH_SUCCESS:
     return normalizeList(state, 'pins', action.statuses, action.next);
   case PIN_SUCCESS:
diff --git a/app/javascript/mastodon/reducers/statuses.js b/app/javascript/mastodon/reducers/statuses.js
index 372673bc0..772f98bcb 100644
--- a/app/javascript/mastodon/reducers/statuses.js
+++ b/app/javascript/mastodon/reducers/statuses.js
@@ -4,6 +4,8 @@ import {
   FAVOURITE_REQUEST,
   FAVOURITE_FAIL,
   UNFAVOURITE_SUCCESS,
+  BOOKMARK_REQUEST,
+  BOOKMARK_FAIL,
 } from '../actions/interactions';
 import {
   STATUS_MUTE_SUCCESS,
@@ -43,6 +45,10 @@ export default function statuses(state = initialState, action) {
     return state.setIn([action.status.get('id'), 'favourites_count'], favouritesCount - 1);
   case FAVOURITE_FAIL:
     return state.get(action.status.get('id')) === undefined ? state : state.setIn([action.status.get('id'), 'favourited'], false);
+  case BOOKMARK_REQUEST:
+    return state.get(action.status.get('id')) === undefined ? state : state.setIn([action.status.get('id'), 'bookmarked'], true);
+  case BOOKMARK_FAIL:
+    return state.get(action.status.get('id')) === undefined ? state : state.setIn([action.status.get('id'), 'bookmarked'], false);
   case REBLOG_REQUEST:
     return state.setIn([action.status.get('id'), 'reblogged'], true);
   case REBLOG_FAIL:
diff --git a/app/javascript/styles/mastodon/components.scss b/app/javascript/styles/mastodon/components.scss
index ded668aea..98ffe1c20 100644
--- a/app/javascript/styles/mastodon/components.scss
+++ b/app/javascript/styles/mastodon/components.scss
@@ -1592,6 +1592,10 @@ a.account__display-name {
   color: $gold-star;
 }
 
+.bookmark-icon.active {
+  color: $red-bookmark;
+}
+
 .no-reduce-motion .icon-button.star-icon {
   &.activate {
     & > .fa-star {
diff --git a/app/javascript/styles/mastodon/variables.scss b/app/javascript/styles/mastodon/variables.scss
index a82c44229..8602c3dde 100644
--- a/app/javascript/styles/mastodon/variables.scss
+++ b/app/javascript/styles/mastodon/variables.scss
@@ -6,6 +6,8 @@ $error-red: #df405a !default;        // Cerise
 $warning-red: #ff5050 !default;      // Sunset Orange
 $gold-star: #ca8f04 !default;        // Dark Goldenrod
 
+$red-bookmark: $warning-red;
+
 // Values from the classic Mastodon UI
 $classic-base-color: #282c37;         // Midnight Express
 $classic-primary-color: #9baec8;      // Echo Blue
diff --git a/app/models/bookmark.rb b/app/models/bookmark.rb
new file mode 100644
index 000000000..01dc48ee7
--- /dev/null
+++ b/app/models/bookmark.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+# == Schema Information
+#
+# Table name: bookmarks
+#
+#  id         :integer          not null, primary key
+#  created_at :datetime         not null
+#  updated_at :datetime         not null
+#  account_id :integer          not null
+#  status_id  :integer          not null
+#
+
+class Bookmark < ApplicationRecord
+  include Paginable
+
+  update_index('statuses#status', :status) if Chewy.enabled?
+
+  belongs_to :account, inverse_of: :bookmarks
+  belongs_to :status,  inverse_of: :bookmarks
+
+  validates :status_id, uniqueness: { scope: :account_id }
+
+  before_validation do
+    self.status = status.reblog if status&.reblog?
+  end
+end
diff --git a/app/models/concerns/account_associations.rb b/app/models/concerns/account_associations.rb
index c9cc5c610..499edbf4e 100644
--- a/app/models/concerns/account_associations.rb
+++ b/app/models/concerns/account_associations.rb
@@ -13,6 +13,7 @@ module AccountAssociations
     # Timelines
     has_many :statuses, inverse_of: :account, dependent: :destroy
     has_many :favourites, inverse_of: :account, dependent: :destroy
+    has_many :bookmarks, inverse_of: :account, dependent: :destroy
     has_many :mentions, inverse_of: :account, dependent: :destroy
     has_many :notifications, inverse_of: :account, dependent: :destroy
     has_many :conversations, class_name: 'AccountConversation', dependent: :destroy, inverse_of: :account
diff --git a/app/models/concerns/account_interactions.rb b/app/models/concerns/account_interactions.rb
index ad2909d91..f27d39483 100644
--- a/app/models/concerns/account_interactions.rb
+++ b/app/models/concerns/account_interactions.rb
@@ -186,6 +186,10 @@ module AccountInteractions
     status.proper.favourites.where(account: self).exists?
   end
 
+  def bookmarked?(status)
+    status.proper.bookmarks.where(account: self).exists?
+  end
+
   def reblogged?(status)
     status.proper.reblogs.where(account: self).exists?
   end
diff --git a/app/models/status.rb b/app/models/status.rb
index 0c01a5389..1cb381400 100644
--- a/app/models/status.rb
+++ b/app/models/status.rb
@@ -54,6 +54,7 @@ class Status < ApplicationRecord
   belongs_to :reblog, foreign_key: 'reblog_of_id', class_name: 'Status', inverse_of: :reblogs, optional: true
 
   has_many :favourites, inverse_of: :status, dependent: :destroy
+  has_many :bookmarks, inverse_of: :status, dependent: :destroy
   has_many :reblogs, foreign_key: 'reblog_of_id', class_name: 'Status', inverse_of: :reblog, dependent: :destroy
   has_many :replies, foreign_key: 'in_reply_to_id', class_name: 'Status', inverse_of: :thread
   has_many :mentions, dependent: :destroy, inverse_of: :status
@@ -302,6 +303,10 @@ class Status < ApplicationRecord
       Favourite.select('status_id').where(status_id: status_ids).where(account_id: account_id).each_with_object({}) { |f, h| h[f.status_id] = true }
     end
 
+    def bookmarks_map(status_ids, account_id)
+      Bookmark.select('status_id').where(status_id: status_ids).where(account_id: account_id).map { |f| [f.status_id, true] }.to_h
+    end
+
     def reblogs_map(status_ids, account_id)
       unscoped.select('reblog_of_id').where(reblog_of_id: status_ids).where(account_id: account_id).each_with_object({}) { |s, h| h[s.reblog_of_id] = true }
     end
diff --git a/app/presenters/status_relationships_presenter.rb b/app/presenters/status_relationships_presenter.rb
index b04e10e2f..64e688d87 100644
--- a/app/presenters/status_relationships_presenter.rb
+++ b/app/presenters/status_relationships_presenter.rb
@@ -7,6 +7,7 @@ class StatusRelationshipsPresenter
     if current_account_id.nil?
       @reblogs_map    = {}
       @favourites_map = {}
+      @bookmarks_map  = {}
       @mutes_map      = {}
       @pins_map       = {}
     else
@@ -17,6 +18,7 @@ class StatusRelationshipsPresenter
 
       @reblogs_map     = Status.reblogs_map(status_ids, current_account_id).merge(options[:reblogs_map] || {})
       @favourites_map  = Status.favourites_map(status_ids, current_account_id).merge(options[:favourites_map] || {})
+      @bookmarks_map   = Status.bookmarks_map(status_ids, current_account_id).merge(options[:bookmarks_map] || {})
       @mutes_map       = Status.mutes_map(conversation_ids, current_account_id).merge(options[:mutes_map] || {})
       @pins_map        = Status.pins_map(pinnable_status_ids, current_account_id).merge(options[:pins_map] || {})
     end
diff --git a/app/serializers/rest/status_serializer.rb b/app/serializers/rest/status_serializer.rb
index 2dc4a1b61..08bc4d82a 100644
--- a/app/serializers/rest/status_serializer.rb
+++ b/app/serializers/rest/status_serializer.rb
@@ -9,6 +9,7 @@ class REST::StatusSerializer < ActiveModel::Serializer
   attribute :favourited, if: :current_user?
   attribute :reblogged, if: :current_user?
   attribute :muted, if: :current_user?
+  attribute :bookmarked, if: :current_user?
   attribute :pinned, if: :pinnable?
 
   attribute :content, unless: :source_requested?
@@ -93,6 +94,14 @@ class REST::StatusSerializer < ActiveModel::Serializer
     end
   end
 
+  def bookmarked
+    if instance_options && instance_options[:bookmarks]
+      instance_options[:bookmarks].bookmarks_map[object.id] || false
+    else
+      current_user.account.bookmarked?(object)
+    end
+  end
+
   def pinned
     if instance_options && instance_options[:relationships]
       instance_options[:relationships].pins_map[object.id] || false
diff --git a/app/services/backup_service.rb b/app/services/backup_service.rb
index 12e4fa8b4..fe26d7aa0 100644
--- a/app/services/backup_service.rb
+++ b/app/services/backup_service.rb
@@ -45,6 +45,7 @@ class BackupService < BaseService
           dump_media_attachments!(tar)
           dump_outbox!(tar)
           dump_likes!(tar)
+          dump_bookmarks!(tar)
           dump_actor!(tar)
         end
       end
@@ -85,6 +86,7 @@ class BackupService < BaseService
     actor[:image][:url] = 'header' + File.extname(actor[:image][:url]) if actor[:image]
     actor[:outbox]      = 'outbox.json'
     actor[:likes]       = 'likes.json'
+    actor[:bookmarks]   = 'bookmarks.json'
 
     download_to_tar(tar, account.avatar, 'avatar' + File.extname(account.avatar.path)) if account.avatar.exists?
     download_to_tar(tar, account.header, 'header' + File.extname(account.header.path)) if account.header.exists?
@@ -115,6 +117,25 @@ class BackupService < BaseService
     end
   end
 
+  def dump_bookmarks!(tar)
+    collection = serialize(ActivityPub::CollectionPresenter.new(id: 'bookmarks.json', type: :ordered, size: 0, items: []), ActivityPub::CollectionSerializer)
+
+    Status.reorder(nil).joins(:bookmarks).includes(:account).merge(account.bookmarks).find_in_batches do |statuses|
+      statuses.each do |status|
+        collection[:totalItems] += 1
+        collection[:orderedItems] << ActivityPub::TagManager.instance.uri_for(status)
+      end
+
+      GC.start
+    end
+
+    json = Oj.dump(collection)
+
+    tar.add_file_simple('bookmarks.json', 0o444, json.bytesize) do |io|
+      io.write(json)
+    end
+  end
+
   def collection_presenter
     ActivityPub::CollectionPresenter.new(
       id: 'outbox.json',
diff --git a/config/initializers/doorkeeper.rb b/config/initializers/doorkeeper.rb
index 914b3c001..a5c9caa4a 100644
--- a/config/initializers/doorkeeper.rb
+++ b/config/initializers/doorkeeper.rb
@@ -58,6 +58,7 @@ Doorkeeper.configure do
   optional_scopes :write,
                   :'write:accounts',
                   :'write:blocks',
+                  :'write:bookmarks',
                   :'write:conversations',
                   :'write:favourites',
                   :'write:filters',
@@ -71,6 +72,7 @@ Doorkeeper.configure do
                   :read,
                   :'read:accounts',
                   :'read:blocks',
+                  :'read:bookmarks',
                   :'read:favourites',
                   :'read:filters',
                   :'read:follows',
diff --git a/config/locales/doorkeeper.en.yml b/config/locales/doorkeeper.en.yml
index d9b7c2c8e..4e9c83a8f 100644
--- a/config/locales/doorkeeper.en.yml
+++ b/config/locales/doorkeeper.en.yml
@@ -125,6 +125,7 @@ en:
       read: read all your account's data
       read:accounts: see accounts information
       read:blocks: see your blocks
+      read:bookmarks: see your bookmarks
       read:favourites: see your favourites
       read:filters: see your filters
       read:follows: see your follows
@@ -137,6 +138,7 @@ en:
       write: modify all your account's data
       write:accounts: modify your profile
       write:blocks: block accounts and domains
+      write:bookmarks: bookmark statuses
       write:favourites: favourite statuses
       write:filters: create filters
       write:follows: follow people
diff --git a/config/routes.rb b/config/routes.rb
index e43e201a5..5411cff58 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -289,6 +289,9 @@ Rails.application.routes.draw do
           resource :favourite, only: :create
           post :unfavourite, to: 'favourites#destroy'
 
+          resource :bookmark, only: :create
+          post :unbookmark, to: 'bookmarks#destroy'
+
           resource :mute, only: :create
           post :unmute, to: 'mutes#destroy'
 
@@ -324,6 +327,7 @@ Rails.application.routes.draw do
       resources :blocks,       only: [:index]
       resources :mutes,        only: [:index]
       resources :favourites,   only: [:index]
+      resources :bookmarks,    only: [:index]
       resources :reports,      only: [:create]
       resources :trends,       only: [:index]
       resources :filters,      only: [:index, :create, :show, :update, :destroy]
diff --git a/db/migrate/20180831171112_create_bookmarks.rb b/db/migrate/20180831171112_create_bookmarks.rb
new file mode 100644
index 000000000..27c7339c9
--- /dev/null
+++ b/db/migrate/20180831171112_create_bookmarks.rb
@@ -0,0 +1,17 @@
+class CreateBookmarks < ActiveRecord::Migration[5.1]
+  def change
+    create_table :bookmarks do |t|
+      t.references :account, null: false
+      t.references :status, null: false
+
+      t.timestamps
+    end
+
+    safety_assured do
+      add_foreign_key :bookmarks, :accounts, column: :account_id, on_delete: :cascade
+      add_foreign_key :bookmarks, :statuses, column: :status_id, on_delete: :cascade
+    end
+
+    add_index :bookmarks, [:account_id, :status_id], unique: true
+  end
+end
diff --git a/db/schema.rb b/db/schema.rb
index 1bcc003a5..84c76e4ce 100644
--- a/db/schema.rb
+++ b/db/schema.rb
@@ -217,6 +217,16 @@ ActiveRecord::Schema.define(version: 2019_10_31_163205) do
     t.index ["target_account_id"], name: "index_blocks_on_target_account_id"
   end
 
+  create_table "bookmarks", force: :cascade do |t|
+    t.bigint "account_id", null: false
+    t.bigint "status_id", null: false
+    t.datetime "created_at", null: false
+    t.datetime "updated_at", null: false
+    t.index ["account_id", "status_id"], name: "index_bookmarks_on_account_id_and_status_id", unique: true
+    t.index ["account_id"], name: "index_bookmarks_on_account_id"
+    t.index ["status_id"], name: "index_bookmarks_on_status_id"
+  end
+
   create_table "conversation_mutes", force: :cascade do |t|
     t.bigint "conversation_id", null: false
     t.bigint "account_id", null: false
@@ -834,6 +844,8 @@ ActiveRecord::Schema.define(version: 2019_10_31_163205) do
   add_foreign_key "backups", "users", on_delete: :nullify
   add_foreign_key "blocks", "accounts", column: "target_account_id", name: "fk_9571bfabc1", on_delete: :cascade
   add_foreign_key "blocks", "accounts", name: "fk_4269e03e65", on_delete: :cascade
+  add_foreign_key "bookmarks", "accounts", on_delete: :cascade
+  add_foreign_key "bookmarks", "statuses", on_delete: :cascade
   add_foreign_key "conversation_mutes", "accounts", name: "fk_225b4212bb", on_delete: :cascade
   add_foreign_key "conversation_mutes", "conversations", on_delete: :cascade
   add_foreign_key "custom_filters", "accounts", on_delete: :cascade
diff --git a/spec/controllers/api/v1/bookmarks_controller_spec.rb b/spec/controllers/api/v1/bookmarks_controller_spec.rb
new file mode 100644
index 000000000..79601b6e6
--- /dev/null
+++ b/spec/controllers/api/v1/bookmarks_controller_spec.rb
@@ -0,0 +1,78 @@
+require 'rails_helper'
+
+RSpec.describe Api::V1::BookmarksController, type: :controller do
+  render_views
+
+  let(:user)  { Fabricate(:user) }
+  let(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'read:bookmarks') }
+
+  describe 'GET #index' do
+    context 'without token' do
+      it 'returns http unauthorized' do
+        get :index
+        expect(response).to have_http_status :unauthorized
+      end
+    end
+
+    context 'with token' do
+      context 'without read scope' do
+        before do
+          allow(controller).to receive(:doorkeeper_token) do
+            Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: '')
+          end
+        end
+
+        it 'returns http forbidden' do
+          get :index
+          expect(response).to have_http_status :forbidden
+        end
+      end
+
+      context 'without valid resource owner' do
+        before do
+          token = Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'read')
+          user.destroy!
+
+          allow(controller).to receive(:doorkeeper_token) { token }
+        end
+
+        it 'returns http unprocessable entity' do
+          get :index
+          expect(response).to have_http_status :unprocessable_entity
+        end
+      end
+
+      context 'with read scope and valid resource owner' do
+        before do
+          allow(controller).to receive(:doorkeeper_token) do
+            Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'read')
+          end
+        end
+
+        it 'shows bookmarks owned by the user' do
+          bookmarked_by_user = Fabricate(:bookmark, account: user.account)
+          bookmarked_by_others = Fabricate(:bookmark)
+
+          get :index
+
+          expect(assigns(:statuses)).to match_array [bookmarked_by_user.status]
+        end
+
+        it 'adds pagination headers if necessary' do
+          bookmark = Fabricate(:bookmark, account: user.account)
+
+          get :index, params: { limit: 1 }
+
+          expect(response.headers['Link'].find_link(['rel', 'next']).href).to eq "http://test.host/api/v1/bookmarks?limit=1&max_id=#{bookmark.id}"
+          expect(response.headers['Link'].find_link(['rel', 'prev']).href).to eq "http://test.host/api/v1/bookmarks?limit=1&since_id=#{bookmark.id}"
+        end
+
+        it 'does not add pagination headers if not necessary' do
+          get :index
+
+          expect(response.headers['Link']).to eq nil
+        end
+      end
+    end
+  end
+end
diff --git a/spec/controllers/api/v1/statuses/bookmarks_controller_spec.rb b/spec/controllers/api/v1/statuses/bookmarks_controller_spec.rb
new file mode 100644
index 000000000..b79853718
--- /dev/null
+++ b/spec/controllers/api/v1/statuses/bookmarks_controller_spec.rb
@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+describe Api::V1::Statuses::BookmarksController do
+  render_views
+
+  let(:user)  { Fabricate(:user, account: Fabricate(:account, username: 'alice')) }
+  let(:app)   { Fabricate(:application, name: 'Test app', website: 'http://testapp.com') }
+  let(:token) { Fabricate(:accessible_access_token, resource_owner_id: user.id, scopes: 'write:bookmarks', application: app) }
+
+  context 'with an oauth token' do
+    before do
+      allow(controller).to receive(:doorkeeper_token) { token }
+    end
+
+    describe 'POST #create' do
+      let(:status) { Fabricate(:status, account: user.account) }
+
+      before do
+        post :create, params: { status_id: status.id }
+      end
+
+      it 'returns http success' do
+        expect(response).to have_http_status(:success)
+      end
+
+      it 'updates the bookmarked attribute' do
+        expect(user.account.bookmarked?(status)).to be true
+      end
+
+      it 'return json with updated attributes' do
+        hash_body = body_as_json
+
+        expect(hash_body[:id]).to eq status.id.to_s
+        expect(hash_body[:bookmarked]).to be true
+      end
+    end
+
+    describe 'POST #destroy' do
+      let(:status) { Fabricate(:status, account: user.account) }
+
+      before do
+        Bookmark.find_or_create_by!(account: user.account, status: status)
+        post :destroy, params: { status_id: status.id }
+      end
+
+      it 'returns http success' do
+        expect(response).to have_http_status(:success)
+      end
+
+      it 'updates the bookmarked attribute' do
+        expect(user.account.bookmarked?(status)).to be false
+      end
+    end
+  end
+end
diff --git a/spec/fabricators/bookmark_fabricator.rb b/spec/fabricators/bookmark_fabricator.rb
new file mode 100644
index 000000000..12cbc5bfa
--- /dev/null
+++ b/spec/fabricators/bookmark_fabricator.rb
@@ -0,0 +1,4 @@
+Fabricator(:bookmark) do
+  account
+  status
+end
-- 
cgit 


From 33c2a7e23c3e53f05ac93ec533e2c9b238e2cc34 Mon Sep 17 00:00:00 2001
From: Thibaut Girka <thib@sitedethib.com>
Date: Wed, 20 Nov 2019 17:18:00 +0100
Subject: Add documentation about the migration hack

---
 config/initializers/0_duplicate_migrations.rb | 14 ++++++++++++++
 db/migrate/20180410220657_create_bookmarks.rb | 10 ++++++++--
 db/migrate/20180831171112_create_bookmarks.rb |  3 +++
 3 files changed, 25 insertions(+), 2 deletions(-)

(limited to 'db/migrate')

diff --git a/config/initializers/0_duplicate_migrations.rb b/config/initializers/0_duplicate_migrations.rb
index 4c7440fa4..d15d2b24a 100644
--- a/config/initializers/0_duplicate_migrations.rb
+++ b/config/initializers/0_duplicate_migrations.rb
@@ -1,3 +1,17 @@
+# Some migrations have been present in glitch-soc for a long time and have then
+# been merged in upstream Mastodon, under a different version number.
+#
+# This puts us in an uneasy situation in which if we remove upstream's
+# migration file, people migrating from upstream will end up having a conflict
+# with their already-ran migration.
+#
+# On the other hand, if we keep upstream's migration and remove our own,
+# any current glitch-soc user will have a conflict during migration.
+#
+# For lack of a better solution, as those migrations are indeed identical,
+# we decided monkey-patching Rails' Migrator to completely ignore the duplicate,
+# keeping only the one that has run, or an arbitrary one.
+
 ALLOWED_DUPLICATES = [20180410220657, 20180831171112].freeze
 
 module ActiveRecord
diff --git a/db/migrate/20180410220657_create_bookmarks.rb b/db/migrate/20180410220657_create_bookmarks.rb
index 08d22c10d..bc79022e4 100644
--- a/db/migrate/20180410220657_create_bookmarks.rb
+++ b/db/migrate/20180410220657_create_bookmarks.rb
@@ -1,3 +1,6 @@
+# This migration is a duplicate of 20180831171112 and may get ignored, see
+# config/initializers/0_duplicate_migrations.rb
+
 class CreateBookmarks < ActiveRecord::Migration[5.1]
   def change
     create_table :bookmarks do |t|
@@ -7,8 +10,11 @@ class CreateBookmarks < ActiveRecord::Migration[5.1]
       t.timestamps
     end
 
-    safety_assured { add_foreign_key :bookmarks, :accounts, column: :account_id, on_delete: :cascade }
-    safety_assured { add_foreign_key :bookmarks, :statuses, column: :status_id, on_delete: :cascade }
+    safety_assured do
+      add_foreign_key :bookmarks, :accounts, column: :account_id, on_delete: :cascade
+      add_foreign_key :bookmarks, :statuses, column: :status_id, on_delete: :cascade
+    end
+
     add_index :bookmarks, [:account_id, :status_id], unique: true
   end
 end
diff --git a/db/migrate/20180831171112_create_bookmarks.rb b/db/migrate/20180831171112_create_bookmarks.rb
index 27c7339c9..5d587b7e9 100644
--- a/db/migrate/20180831171112_create_bookmarks.rb
+++ b/db/migrate/20180831171112_create_bookmarks.rb
@@ -1,3 +1,6 @@
+# This migration is a duplicate of 20180410220657 and may get ignored, see
+# config/initializers/0_duplicate_migrations.rb
+
 class CreateBookmarks < ActiveRecord::Migration[5.1]
   def change
     create_table :bookmarks do |t|
-- 
cgit 


From f682387aae0222e4cd77ef92a2b8f9e0e99c2818 Mon Sep 17 00:00:00 2001
From: ThibG <thib@sitedethib.com>
Date: Wed, 4 Dec 2019 04:34:31 +0100
Subject: Fix old migration failing with new status default scope (#12493)

---
 ...20181024224956_migrate_account_conversations.rb | 60 ++++++++++++++++++++++
 1 file changed, 60 insertions(+)

(limited to 'db/migrate')

diff --git a/db/migrate/20181024224956_migrate_account_conversations.rb b/db/migrate/20181024224956_migrate_account_conversations.rb
index d4bc3b91d..9f6c94fd1 100644
--- a/db/migrate/20181024224956_migrate_account_conversations.rb
+++ b/db/migrate/20181024224956_migrate_account_conversations.rb
@@ -1,6 +1,66 @@
 class MigrateAccountConversations < ActiveRecord::Migration[5.2]
   disable_ddl_transaction!
 
+  class Mention < ApplicationRecord
+    belongs_to :account, inverse_of: :mentions
+    belongs_to :status, -> { unscope(where: :deleted_at) }
+
+    delegate(
+      :username,
+      :acct,
+      to: :account,
+      prefix: true
+    )
+  end
+
+  class Notification < ApplicationRecord
+    belongs_to :account, optional: true
+    belongs_to :activity, polymorphic: true, optional: true
+
+    belongs_to :status,         foreign_type: 'Status',        foreign_key: 'activity_id', optional: true
+    belongs_to :mention,        foreign_type: 'Mention',       foreign_key: 'activity_id', optional: true
+
+    def target_status
+      mention&.status
+    end
+  end
+
+  class AccountConversation < ApplicationRecord
+    belongs_to :account
+    belongs_to :conversation
+    belongs_to :last_status, -> { unscope(where: :deleted_at) }, class_name: 'Status'
+
+    before_validation :set_last_status
+
+    class << self
+      def add_status(recipient, status)
+        conversation = find_or_initialize_by(account: recipient, conversation_id: status.conversation_id, participant_account_ids: participants_from_status(recipient, status))
+
+        return conversation if conversation.status_ids.include?(status.id)
+
+        conversation.status_ids << status.id
+        conversation.unread = status.account_id != recipient.id
+        conversation.save
+        conversation
+      rescue ActiveRecord::StaleObjectError
+        retry
+      end
+
+      private
+
+      def participants_from_status(recipient, status)
+        ((status.active_mentions.pluck(:account_id) + [status.account_id]).uniq - [recipient.id]).sort
+      end
+    end
+
+    private
+
+    def set_last_status
+      self.status_ids     = status_ids.sort
+      self.last_status_id = status_ids.last
+    end
+  end
+
   def up
     say ''
     say 'WARNING: This migration may take a *long* time for large instances'
-- 
cgit