Lists (#5703)

* Add structure for lists

* Add list timeline streaming API

* Add list APIs, bind list-account relation to follow relation

* Add API for adding/removing accounts from lists

* Add pagination to lists API

* Add pagination to list accounts API

* Adjust scopes for new APIs

- Creating and modifying lists merely requires "write" scope
- Fetching information about lists merely requires "read" scope

* Add test for wrong user context on list timeline

* Clean up tests

34 files changed, 161 insertions, 86 deletions

diff --git a/app/models/account.rb b/app/models/account.rb
index bc01d2448..9353c40da 100644
--- a/app/models/account.rb
+++ b/app/models/account.rb
@@ -3,7 +3,7 @@
 #
 # Table name: accounts
 #
-#  id                      :bigint           not null, primary key
+#  id                      :integer          not null, primary key
 #  username                :string           default(""), not null
 #  domain                  :string
 #  secret                  :string           default(""), not null
@@ -53,6 +53,7 @@ class Account < ApplicationRecord
   include AccountInteractions
   include Attachmentable
   include Remotable
+  include Paginable
 
   enum protocol: [:ostatus, :activitypub]
 
@@ -95,6 +96,10 @@ class Account < ApplicationRecord
   has_many :account_moderation_notes, dependent: :destroy
   has_many :targeted_moderation_notes, class_name: 'AccountModerationNote', foreign_key: :target_account_id, dependent: :destroy
 
+  # Lists
+  has_many :list_accounts, inverse_of: :account, dependent: :destroy
+  has_many :lists, through: :list_accounts
+
   scope :remote, -> { where.not(domain: nil) }
   scope :local, -> { where(domain: nil) }
   scope :without_followers, -> { where(followers_count: 0) }
diff --git a/app/models/account_domain_block.rb b/app/models/account_domain_block.rb
index 9c98ec2a6..35810b6c2 100644
--- a/app/models/account_domain_block.rb
+++ b/app/models/account_domain_block.rb
@@ -3,11 +3,11 @@
 #
 # Table name: account_domain_blocks
 #
+#  id         :integer          not null, primary key
 #  domain     :string
 #  created_at :datetime         not null
 #  updated_at :datetime         not null
-#  account_id :bigint
-#  id         :bigint           not null, primary key
+#  account_id :integer
 #
 
 class AccountDomainBlock < ApplicationRecord
diff --git a/app/models/account_moderation_note.rb b/app/models/account_moderation_note.rb
index 06f464850..3ac9b1ac1 100644
--- a/app/models/account_moderation_note.rb
+++ b/app/models/account_moderation_note.rb
@@ -3,10 +3,10 @@
 #
 # Table name: account_moderation_notes
 #
-#  id                :bigint           not null, primary key
+#  id                :integer          not null, primary key
 #  content           :text             not null
-#  account_id        :bigint           not null
-#  target_account_id :bigint           not null
+#  account_id        :integer          not null
+#  target_account_id :integer          not null
 #  created_at        :datetime         not null
 #  updated_at        :datetime         not null
 #
diff --git a/app/models/block.rb b/app/models/block.rb
index 5778f7e90..284abfe4c 100644
--- a/app/models/block.rb
+++ b/app/models/block.rb
@@ -3,11 +3,11 @@
 #
 # Table name: blocks
 #
+#  id                :integer          not null, primary key
 #  created_at        :datetime         not null
 #  updated_at        :datetime         not null
-#  account_id        :bigint           not null
-#  id                :bigint           not null, primary key
-#  target_account_id :bigint           not null
+#  account_id        :integer          not null
+#  target_account_id :integer          not null
 #
 
 class Block < ApplicationRecord
diff --git a/app/models/conversation.rb b/app/models/conversation.rb
index e08532522..08c1ce945 100644
--- a/app/models/conversation.rb
+++ b/app/models/conversation.rb
@@ -3,7 +3,7 @@
 #
 # Table name: conversations
 #
-#  id         :bigint           not null, primary key
+#  id         :integer          not null, primary key
 #  uri        :string
 #  created_at :datetime         not null
 #  updated_at :datetime         not null
diff --git a/app/models/conversation_mute.rb b/app/models/conversation_mute.rb
index 316865bd2..248cdfe6e 100644
--- a/app/models/conversation_mute.rb
+++ b/app/models/conversation_mute.rb
@@ -3,9 +3,9 @@
 #
 # Table name: conversation_mutes
 #
-#  conversation_id :bigint           not null
-#  account_id      :bigint           not null
-#  id              :bigint           not null, primary key
+#  id              :integer          not null, primary key
+#  conversation_id :integer          not null
+#  account_id      :integer          not null
 #
 
 class ConversationMute < ApplicationRecord
diff --git a/app/models/custom_emoji.rb b/app/models/custom_emoji.rb
index 5723ebd5d..a77b53c98 100644
--- a/app/models/custom_emoji.rb
+++ b/app/models/custom_emoji.rb
@@ -3,7 +3,7 @@
 #
 # Table name: custom_emojis
 #
-#  id                 :bigint           not null, primary key
+#  id                 :integer          not null, primary key
 #  shortcode          :string           default(""), not null
 #  domain             :string
 #  image_file_name    :string
diff --git a/app/models/domain_block.rb b/app/models/domain_block.rb
index 557d0a19c..aea8919af 100644
--- a/app/models/domain_block.rb
+++ b/app/models/domain_block.rb
@@ -3,12 +3,12 @@
 #
 # Table name: domain_blocks
 #
+#  id           :integer          not null, primary key
 #  domain       :string           default(""), not null
 #  created_at   :datetime         not null
 #  updated_at   :datetime         not null
 #  severity     :integer          default("silence")
 #  reject_media :boolean          default(FALSE), not null
-#  id           :bigint           not null, primary key
 #
 
 class DomainBlock < ApplicationRecord
diff --git a/app/models/email_domain_block.rb b/app/models/email_domain_block.rb
index 2c348197c..a104810d1 100644
--- a/app/models/email_domain_block.rb
+++ b/app/models/email_domain_block.rb
@@ -3,7 +3,7 @@
 #
 # Table name: email_domain_blocks
 #
-#  id         :bigint           not null, primary key
+#  id         :integer          not null, primary key
 #  domain     :string           default(""), not null
 #  created_at :datetime         not null
 #  updated_at :datetime         not null
diff --git a/app/models/favourite.rb b/app/models/favourite.rb
index f611aa6a9..c38838f2a 100644
--- a/app/models/favourite.rb
+++ b/app/models/favourite.rb
@@ -3,11 +3,11 @@
 #
 # Table name: favourites
 #
+#  id         :integer          not null, primary key
 #  created_at :datetime         not null
 #  updated_at :datetime         not null
-#  account_id :bigint           not null
-#  id         :bigint           not null, primary key
-#  status_id  :bigint           not null
+#  account_id :integer          not null
+#  status_id  :integer          not null
 #
 
 class Favourite < ApplicationRecord
diff --git a/app/models/feed.rb b/app/models/feed.rb
index 5f7b7877a..d99f1ffb2 100644
--- a/app/models/feed.rb
+++ b/app/models/feed.rb
@@ -1,36 +1,27 @@
 # frozen_string_literal: true
 
 class Feed
-  def initialize(type, account)
-    @type    = type
-    @account = account
+  def initialize(type, id)
+    @type = type
+    @id   = id
   end
 
   def get(limit, max_id = nil, since_id = nil)
-    if redis.exists("account:#{@account.id}:regeneration")
-      from_database(limit, max_id, since_id)
-    else
-      from_redis(limit, max_id, since_id)
-    end
+    from_redis(limit, max_id, since_id)
   end
 
-  private
+  protected
 
   def from_redis(limit, max_id, since_id)
     max_id     = '+inf' if max_id.blank?
     since_id   = '-inf' if since_id.blank?
     unhydrated = redis.zrevrangebyscore(key, "(#{max_id}", "(#{since_id}", limit: [0, limit], with_scores: true).map(&:first).map(&:to_i)
-    Status.where(id: unhydrated).cache_ids
-  end
 
-  def from_database(limit, max_id, since_id)
-    Status.as_home_timeline(@account)
-          .paginate_by_max_id(limit, max_id, since_id)
-          .reject { |status| FeedManager.instance.filter?(:home, status, @account.id) }
+    Status.where(id: unhydrated).cache_ids
   end
 
   def key
-    FeedManager.instance.key(@type, @account.id)
+    FeedManager.instance.key(@type, @id)
   end
 
   def redis
diff --git a/app/models/follow.rb b/app/models/follow.rb
index 3d5447fb1..795ecf55a 100644
--- a/app/models/follow.rb
+++ b/app/models/follow.rb
@@ -3,11 +3,11 @@
 #
 # Table name: follows
 #
+#  id                :integer          not null, primary key
 #  created_at        :datetime         not null
 #  updated_at        :datetime         not null
-#  account_id        :bigint           not null
-#  id                :bigint           not null, primary key
-#  target_account_id :bigint           not null
+#  account_id        :integer          not null
+#  target_account_id :integer          not null
 #
 
 class Follow < ApplicationRecord
diff --git a/app/models/follow_request.rb b/app/models/follow_request.rb
index ce27fc921..fac91b513 100644
--- a/app/models/follow_request.rb
+++ b/app/models/follow_request.rb
@@ -3,11 +3,11 @@
 #
 # Table name: follow_requests
 #
+#  id                :integer          not null, primary key
 #  created_at        :datetime         not null
 #  updated_at        :datetime         not null
-#  account_id        :bigint           not null
-#  id                :bigint           not null, primary key
-#  target_account_id :bigint           not null
+#  account_id        :integer          not null
+#  target_account_id :integer          not null
 #
 
 class FollowRequest < ApplicationRecord
diff --git a/app/models/home_feed.rb b/app/models/home_feed.rb
new file mode 100644
index 000000000..b943a34ce
--- /dev/null
+++ b/app/models/home_feed.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+class HomeFeed < Feed
+  def initialize(account)
+    @type    = :home
+    @id      = account.id
+    @account = account
+  end
+
+  def get(limit, max_id = nil, since_id = nil)
+    if redis.exists("account:#{@account.id}:regeneration")
+      from_database(limit, max_id, since_id)
+    else
+      super
+    end
+  end
+
+  private
+
+  def from_database(limit, max_id, since_id)
+    Status.as_home_timeline(@account)
+          .paginate_by_max_id(limit, max_id, since_id)
+          .reject { |status| FeedManager.instance.filter?(:home, status, @account.id) }
+  end
+end
diff --git a/app/models/import.rb b/app/models/import.rb
index 6f1278556..091fb3044 100644
--- a/app/models/import.rb
+++ b/app/models/import.rb
@@ -3,6 +3,7 @@
 #
 # Table name: imports
 #
+#  id                :integer          not null, primary key
 #  type              :integer          not null
 #  approved          :boolean          default(FALSE), not null
 #  created_at        :datetime         not null
@@ -11,8 +12,7 @@
 #  data_content_type :string
 #  data_file_size    :integer
 #  data_updated_at   :datetime
-#  account_id        :bigint           not null
-#  id                :bigint           not null, primary key
+#  account_id        :integer          not null
 #
 
 class Import < ApplicationRecord
diff --git a/app/models/list.rb b/app/models/list.rb
new file mode 100644
index 000000000..5d7ba0065
--- /dev/null
+++ b/app/models/list.rb
@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+# == Schema Information
+#
+# Table name: lists
+#
+#  id         :integer          not null, primary key
+#  account_id :integer
+#  title      :string           default(""), not null
+#  created_at :datetime         not null
+#  updated_at :datetime         not null
+#
+
+class List < ApplicationRecord
+  include Paginable
+
+  belongs_to :account
+
+  has_many :list_accounts, inverse_of: :list, dependent: :destroy
+  has_many :accounts, through: :list_accounts
+
+  validates :title, presence: true
+end
diff --git a/app/models/list_account.rb b/app/models/list_account.rb
new file mode 100644
index 000000000..c08239aa0
--- /dev/null
+++ b/app/models/list_account.rb
@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+# == Schema Information
+#
+# Table name: list_accounts
+#
+#  id         :integer          not null, primary key
+#  list_id    :integer          not null
+#  account_id :integer          not null
+#  follow_id  :integer          not null
+#
+
+class ListAccount < ApplicationRecord
+  belongs_to :list, required: true
+  belongs_to :account, required: true
+  belongs_to :follow, required: true
+
+  before_validation :set_follow
+
+  private
+
+  def set_follow
+    self.follow = Follow.find_by(account_id: list.account_id, target_account_id: account.id)
+  end
+end
diff --git a/app/models/list_feed.rb b/app/models/list_feed.rb
new file mode 100644
index 000000000..f371e4ed9
--- /dev/null
+++ b/app/models/list_feed.rb
@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+class ListFeed < Feed
+  def initialize(list)
+    @type    = :list
+    @id      = list.id
+  end
+end
diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb
index f05418925..abc5ab854 100644
--- a/app/models/media_attachment.rb
+++ b/app/models/media_attachment.rb
@@ -3,19 +3,19 @@
 #
 # Table name: media_attachments
 #
-#  id                :bigint           not null, primary key
-#  status_id         :bigint
+#  id                :integer          not null, primary key
+#  status_id         :integer
 #  file_file_name    :string
 #  file_content_type :string
 #  file_file_size    :integer
 #  file_updated_at   :datetime
 #  remote_url        :string           default(""), not null
-#  account_id        :bigint
 #  created_at        :datetime         not null
 #  updated_at        :datetime         not null
 #  shortcode         :string
 #  type              :integer          default("image"), not null
 #  file_meta         :json
+#  account_id        :integer
 #  description       :text
 #
 
diff --git a/app/models/mention.rb b/app/models/mention.rb
index fc089d365..14533e6a9 100644
--- a/app/models/mention.rb
+++ b/app/models/mention.rb
@@ -3,11 +3,11 @@
 #
 # Table name: mentions
 #
-#  status_id  :bigint
+#  id         :integer          not null, primary key
+#  status_id  :integer
 #  created_at :datetime         not null
 #  updated_at :datetime         not null
-#  account_id :bigint
-#  id         :bigint           not null, primary key
+#  account_id :integer
 #
 
 class Mention < ApplicationRecord
diff --git a/app/models/notification.rb b/app/models/notification.rb
index c88af9021..a3ffb1f45 100644
--- a/app/models/notification.rb
+++ b/app/models/notification.rb
@@ -3,13 +3,13 @@
 #
 # Table name: notifications
 #
-#  id              :bigint           not null, primary key
-#  account_id      :bigint
-#  activity_id     :bigint
+#  id              :integer          not null, primary key
+#  activity_id     :integer
 #  activity_type   :string
 #  created_at      :datetime         not null
 #  updated_at      :datetime         not null
-#  from_account_id :bigint
+#  account_id      :integer
+#  from_account_id :integer
 #
 
 class Notification < ApplicationRecord
diff --git a/app/models/preview_card.rb b/app/models/preview_card.rb
index 63c04b410..e2bf65d94 100644
--- a/app/models/preview_card.rb
+++ b/app/models/preview_card.rb
@@ -3,7 +3,7 @@
 #
 # Table name: preview_cards
 #
-#  id                 :bigint           not null, primary key
+#  id                 :integer          not null, primary key
 #  url                :string           default(""), not null
 #  title              :string           default(""), not null
 #  description        :string           default(""), not null
diff --git a/app/models/report.rb b/app/models/report.rb
index 99c90b7dd..c36f8db0a 100644
--- a/app/models/report.rb
+++ b/app/models/report.rb
@@ -3,15 +3,15 @@
 #
 # Table name: reports
 #
+#  id                         :integer          not null, primary key
 #  status_ids                 :integer          default([]), not null, is an Array
 #  comment                    :text             default(""), not null
 #  action_taken               :boolean          default(FALSE), not null
 #  created_at                 :datetime         not null
 #  updated_at                 :datetime         not null
-#  account_id                 :bigint           not null
-#  action_taken_by_account_id :bigint
-#  id                         :bigint           not null, primary key
-#  target_account_id          :bigint           not null
+#  account_id                 :integer          not null
+#  action_taken_by_account_id :integer
+#  target_account_id          :integer          not null
 #
 
 class Report < ApplicationRecord
diff --git a/app/models/session_activation.rb b/app/models/session_activation.rb
index 59565f877..d19489b36 100644
--- a/app/models/session_activation.rb
+++ b/app/models/session_activation.rb
@@ -3,15 +3,15 @@
 #
 # Table name: session_activations
 #
-#  id                       :bigint           not null, primary key
-#  user_id                  :bigint           not null
+#  id                       :integer          not null, primary key
 #  session_id               :string           not null
 #  created_at               :datetime         not null
 #  updated_at               :datetime         not null
 #  user_agent               :string           default(""), not null
 #  ip                       :inet
-#  access_token_id          :bigint
-#  web_push_subscription_id :bigint
+#  access_token_id          :integer
+#  user_id                  :integer          not null
+#  web_push_subscription_id :integer
 #
 
 #  id              :bigint           not null, primary key
diff --git a/app/models/setting.rb b/app/models/setting.rb
index be68d3123..df93590ce 100644
--- a/app/models/setting.rb
+++ b/app/models/setting.rb
@@ -3,13 +3,13 @@
 #
 # Table name: settings
 #
+#  id         :integer          not null, primary key
 #  var        :string           not null
 #  value      :text
 #  thing_type :string
 #  created_at :datetime
 #  updated_at :datetime
-#  id         :bigint           not null, primary key
-#  thing_id   :bigint
+#  thing_id   :integer
 #
 
 class Setting < RailsSettings::Base
diff --git a/app/models/site_upload.rb b/app/models/site_upload.rb
index ba2ca777b..8ffdc8313 100644
--- a/app/models/site_upload.rb
+++ b/app/models/site_upload.rb
@@ -3,7 +3,7 @@
 #
 # Table name: site_uploads
 #
-#  id                :bigint           not null, primary key
+#  id                :integer          not null, primary key
 #  var               :string           default(""), not null
 #  file_file_name    :string
 #  file_content_type :string
diff --git a/app/models/status.rb b/app/models/status.rb
index b4f314311..26095070f 100644
--- a/app/models/status.rb
+++ b/app/models/status.rb
@@ -3,26 +3,26 @@
 #
 # Table name: statuses
 #
-#  id                     :bigint           not null, primary key
+#  id                     :integer          not null, primary key
 #  uri                    :string
-#  account_id             :bigint           not null
 #  text                   :text             default(""), not null
 #  created_at             :datetime         not null
 #  updated_at             :datetime         not null
-#  in_reply_to_id         :bigint
-#  reblog_of_id           :bigint
+#  in_reply_to_id         :integer
+#  reblog_of_id           :integer
 #  url                    :string
 #  sensitive              :boolean          default(FALSE), not null
 #  visibility             :integer          default("public"), not null
-#  in_reply_to_account_id :bigint
-#  application_id         :bigint
 #  spoiler_text           :text             default(""), not null
 #  reply                  :boolean          default(FALSE), not null
 #  favourites_count       :integer          default(0), not null
 #  reblogs_count          :integer          default(0), not null
 #  language               :string
-#  conversation_id        :bigint
+#  conversation_id        :integer
 #  local                  :boolean
+#  account_id             :integer          not null
+#  application_id         :integer
+#  in_reply_to_account_id :integer
 #
 
 class Status < ApplicationRecord
diff --git a/app/models/status_pin.rb b/app/models/status_pin.rb
index 5795d07bf..a72c19750 100644
--- a/app/models/status_pin.rb
+++ b/app/models/status_pin.rb
@@ -3,9 +3,9 @@
 #
 # Table name: status_pins
 #
-#  id         :bigint           not null, primary key
-#  account_id :bigint           not null
-#  status_id  :bigint           not null
+#  id         :integer          not null, primary key
+#  account_id :integer          not null
+#  status_id  :integer          not null
 #  created_at :datetime         not null
 #  updated_at :datetime         not null
 #
diff --git a/app/models/stream_entry.rb b/app/models/stream_entry.rb
index 50b900c3c..2ae034d93 100644
--- a/app/models/stream_entry.rb
+++ b/app/models/stream_entry.rb
@@ -3,13 +3,13 @@
 #
 # Table name: stream_entries
 #
-#  activity_id   :bigint
+#  id            :integer          not null, primary key
+#  activity_id   :integer
 #  activity_type :string
 #  created_at    :datetime         not null
 #  updated_at    :datetime         not null
 #  hidden        :boolean          default(FALSE), not null
-#  account_id    :bigint
-#  id            :bigint           not null, primary key
+#  account_id    :integer
 #
 
 class StreamEntry < ApplicationRecord
diff --git a/app/models/subscription.rb b/app/models/subscription.rb
index bc50c5317..7f2eeab91 100644
--- a/app/models/subscription.rb
+++ b/app/models/subscription.rb
@@ -3,6 +3,7 @@
 #
 # Table name: subscriptions
 #
+#  id                          :integer          not null, primary key
 #  callback_url                :string           default(""), not null
 #  secret                      :string
 #  expires_at                  :datetime
@@ -11,8 +12,7 @@
 #  updated_at                  :datetime         not null
 #  last_successful_delivery_at :datetime
 #  domain                      :string
-#  account_id                  :bigint           not null
-#  id                          :bigint           not null, primary key
+#  account_id                  :integer          not null
 #
 
 class Subscription < ApplicationRecord
diff --git a/app/models/tag.rb b/app/models/tag.rb
index 6ebaf1145..0fa08e157 100644
--- a/app/models/tag.rb
+++ b/app/models/tag.rb
@@ -3,7 +3,7 @@
 #
 # Table name: tags
 #
-#  id         :bigint           not null, primary key
+#  id         :integer          not null, primary key
 #  name       :string           default(""), not null
 #  created_at :datetime         not null
 #  updated_at :datetime         not null
diff --git a/app/models/user.rb b/app/models/user.rb
index 326b871a1..b9b228c00 100644
--- a/app/models/user.rb
+++ b/app/models/user.rb
@@ -3,7 +3,7 @@
 #
 # Table name: users
 #
-#  id                        :bigint           not null, primary key
+#  id                        :integer          not null, primary key
 #  email                     :string           default(""), not null
 #  created_at                :datetime         not null
 #  updated_at                :datetime         not null
@@ -30,7 +30,7 @@
 #  last_emailed_at           :datetime
 #  otp_backup_codes          :string           is an Array
 #  filtered_languages        :string           default([]), not null, is an Array
-#  account_id                :bigint           not null
+#  account_id                :integer          not null
 #  disabled                  :boolean          default(FALSE), not null
 #  moderator                 :boolean          default(FALSE), not null
 #
diff --git a/app/models/web/push_subscription.rb b/app/models/web/push_subscription.rb
index a41906227..5aee92d27 100644
--- a/app/models/web/push_subscription.rb
+++ b/app/models/web/push_subscription.rb
@@ -3,7 +3,7 @@
 #
 # Table name: web_push_subscriptions
 #
-#  id         :bigint           not null, primary key
+#  id         :integer          not null, primary key
 #  endpoint   :string           not null
 #  key_p256dh :string           not null
 #  key_auth   :string           not null
diff --git a/app/models/web/setting.rb b/app/models/web/setting.rb
index 6d08c4d35..12b9d1226 100644
--- a/app/models/web/setting.rb
+++ b/app/models/web/setting.rb
@@ -3,11 +3,11 @@
 #
 # Table name: web_settings
 #
+#  id         :integer          not null, primary key
 #  data       :json
 #  created_at :datetime         not null
 #  updated_at :datetime         not null
-#  id         :bigint           not null, primary key
-#  user_id    :bigint
+#  user_id    :integer
 #
 
 class Web::Setting < ApplicationRecord