From 24cafd73a2b644025e9aeaadf4fed46dd3ecea4d Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sat, 18 Nov 2017 00:16:48 +0100 Subject: Lists (#5703) * Add structure for lists * Add list timeline streaming API * Add list APIs, bind list-account relation to follow relation * Add API for adding/removing accounts from lists * Add pagination to lists API * Add pagination to list accounts API * Adjust scopes for new APIs - Creating and modifying lists merely requires "write" scope - Fetching information about lists merely requires "read" scope * Add test for wrong user context on list timeline * Clean up tests --- .../api/v1/lists/accounts_controller.rb | 81 ++++++++++++++++++++++ app/controllers/api/v1/lists_controller.rb | 79 +++++++++++++++++++++ .../api/v1/timelines/home_controller.rb | 2 +- .../api/v1/timelines/list_controller.rb | 66 ++++++++++++++++++ app/lib/feed_manager.rb | 73 ++++++++++--------- app/models/account.rb | 7 +- app/models/account_domain_block.rb | 4 +- app/models/account_moderation_note.rb | 6 +- app/models/block.rb | 6 +- app/models/conversation.rb | 2 +- app/models/conversation_mute.rb | 6 +- app/models/custom_emoji.rb | 2 +- app/models/domain_block.rb | 2 +- app/models/email_domain_block.rb | 2 +- app/models/favourite.rb | 6 +- app/models/feed.rb | 23 ++---- app/models/follow.rb | 6 +- app/models/follow_request.rb | 6 +- app/models/home_feed.rb | 25 +++++++ app/models/import.rb | 4 +- app/models/list.rb | 22 ++++++ app/models/list_account.rb | 24 +++++++ app/models/list_feed.rb | 8 +++ app/models/media_attachment.rb | 6 +- app/models/mention.rb | 6 +- app/models/notification.rb | 8 +-- app/models/preview_card.rb | 2 +- app/models/report.rb | 8 +-- app/models/session_activation.rb | 8 +-- app/models/setting.rb | 4 +- app/models/site_upload.rb | 2 +- app/models/status.rb | 14 ++-- app/models/status_pin.rb | 6 +- app/models/stream_entry.rb | 6 +- app/models/subscription.rb | 4 +- app/models/tag.rb | 2 +- app/models/user.rb | 4 +- app/models/web/push_subscription.rb | 2 +- app/models/web/setting.rb | 4 +- app/serializers/rest/list_serializer.rb | 5 ++ app/services/batched_remove_status_service.rb | 11 ++- app/services/fan_out_on_write_service.rb | 17 ++++- app/services/remove_status_service.rb | 15 ++-- app/workers/feed_insert_worker.rb | 39 ++++++----- app/workers/push_update_worker.rb | 11 +-- 45 files changed, 496 insertions(+), 150 deletions(-) create mode 100644 app/controllers/api/v1/lists/accounts_controller.rb create mode 100644 app/controllers/api/v1/lists_controller.rb create mode 100644 app/controllers/api/v1/timelines/list_controller.rb create mode 100644 app/models/home_feed.rb create mode 100644 app/models/list.rb create mode 100644 app/models/list_account.rb create mode 100644 app/models/list_feed.rb create mode 100644 app/serializers/rest/list_serializer.rb (limited to 'app') diff --git a/app/controllers/api/v1/lists/accounts_controller.rb b/app/controllers/api/v1/lists/accounts_controller.rb new file mode 100644 index 000000000..40c485e8d --- /dev/null +++ b/app/controllers/api/v1/lists/accounts_controller.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +class Api::V1::Lists::AccountsController < Api::BaseController + before_action -> { doorkeeper_authorize! :read }, only: [:show] + before_action -> { doorkeeper_authorize! :write }, except: [:show] + + before_action :require_user! + before_action :set_list + + after_action :insert_pagination_headers, only: :show + + def show + @accounts = @list.accounts.paginate_by_max_id(limit_param(DEFAULT_ACCOUNTS_LIMIT), params[:max_id], params[:since_id]) + render json: @accounts, each_serializer: REST::AccountSerializer + end + + def create + ApplicationRecord.transaction do + list_accounts.each do |account| + @list.accounts << account + end + end + + render_empty + end + + def destroy + ListAccount.where(list: @list, account_id: account_ids).destroy_all + render_empty + end + + private + + def set_list + @list = List.where(account: current_account).find(params[:list_id]) + end + + def list_accounts + Account.find(account_ids) + end + + def account_ids + Array(resource_params[:account_ids]) + end + + def resource_params + params.permit(account_ids: []) + end + + def insert_pagination_headers + set_pagination_headers(next_path, prev_path) + end + + def next_path + if records_continue? + api_v1_list_accounts_url pagination_params(max_id: pagination_max_id) + end + end + + def prev_path + unless @accounts.empty? + api_v1_list_accounts_url pagination_params(since_id: pagination_since_id) + end + end + + def pagination_max_id + @accounts.last.id + end + + def pagination_since_id + @accounts.first.id + end + + def records_continue? + @accounts.size == limit_param(DEFAULT_ACCOUNTS_LIMIT) + end + + def pagination_params(core_params) + params.permit(:limit).merge(core_params) + end +end diff --git a/app/controllers/api/v1/lists_controller.rb b/app/controllers/api/v1/lists_controller.rb new file mode 100644 index 000000000..9437373bd --- /dev/null +++ b/app/controllers/api/v1/lists_controller.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true + +class Api::V1::ListsController < Api::BaseController + LISTS_LIMIT = 50 + + before_action -> { doorkeeper_authorize! :read }, only: [:index, :show] + before_action -> { doorkeeper_authorize! :write }, except: [:index, :show] + + before_action :require_user! + before_action :set_list, except: [:index, :create] + + after_action :insert_pagination_headers, only: :index + + def index + @lists = List.where(account: current_account).paginate_by_max_id(limit_param(LISTS_LIMIT), params[:max_id], params[:since_id]) + render json: @lists, each_serializer: REST::ListSerializer + end + + def show + render json: @list, serializer: REST::ListSerializer + end + + def create + @list = List.create!(list_params.merge(account: current_account)) + render json: @list, serializer: REST::ListSerializer + end + + def update + @list.update!(list_params) + render json: @list, serializer: REST::ListSerializer + end + + def destroy + @list.destroy! + render_empty + end + + private + + def set_list + @list = List.where(account: current_account).find(params[:id]) + end + + def list_params + params.permit(:title) + end + + def insert_pagination_headers + set_pagination_headers(next_path, prev_path) + end + + def next_path + if records_continue? + api_v1_lists_url pagination_params(max_id: pagination_max_id) + end + end + + def prev_path + unless @lists.empty? + api_v1_lists_url pagination_params(since_id: pagination_since_id) + end + end + + def pagination_max_id + @lists.last.id + end + + def pagination_since_id + @lists.first.id + end + + def records_continue? + @lists.size == limit_param(LISTS_LIMIT) + end + + def pagination_params(core_params) + params.permit(:limit).merge(core_params) + end +end diff --git a/app/controllers/api/v1/timelines/home_controller.rb b/app/controllers/api/v1/timelines/home_controller.rb index 3dd27710c..db6cd8568 100644 --- a/app/controllers/api/v1/timelines/home_controller.rb +++ b/app/controllers/api/v1/timelines/home_controller.rb @@ -31,7 +31,7 @@ class Api::V1::Timelines::HomeController < Api::BaseController end def account_home_feed - Feed.new(:home, current_account) + HomeFeed.new(current_account) end def insert_pagination_headers diff --git a/app/controllers/api/v1/timelines/list_controller.rb b/app/controllers/api/v1/timelines/list_controller.rb new file mode 100644 index 000000000..f5db71e46 --- /dev/null +++ b/app/controllers/api/v1/timelines/list_controller.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +class Api::V1::Timelines::ListController < Api::BaseController + before_action -> { doorkeeper_authorize! :read } + before_action :require_user! + before_action :set_list + before_action :set_statuses + + after_action :insert_pagination_headers, unless: -> { @statuses.empty? } + + def show + render json: @statuses, + each_serializer: REST::StatusSerializer, + relationships: StatusRelationshipsPresenter.new(@statuses, current_user.account_id) + end + + private + + def set_list + @list = List.where(account: current_account).find(params[:id]) + end + + def set_statuses + @statuses = cached_list_statuses + end + + def cached_list_statuses + cache_collection list_statuses, Status + end + + def list_statuses + list_feed.get( + limit_param(DEFAULT_STATUSES_LIMIT), + params[:max_id], + params[:since_id] + ) + end + + def list_feed + ListFeed.new(@list) + end + + def insert_pagination_headers + set_pagination_headers(next_path, prev_path) + end + + def pagination_params(core_params) + params.permit(:limit).merge(core_params) + end + + def next_path + api_v1_timelines_list_url params[:id], pagination_params(max_id: pagination_max_id) + end + + def prev_path + api_v1_timelines_list_url params[:id], pagination_params(since_id: pagination_since_id) + end + + def pagination_max_id + @statuses.last.id + end + + def pagination_since_id + @statuses.first.id + end +end diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb index 58650efb6..79fae6e96 100644 --- a/app/lib/feed_manager.rb +++ b/app/lib/feed_manager.rb @@ -26,34 +26,42 @@ class FeedManager end end - def push(timeline_type, account, status) - return false unless add_to_feed(timeline_type, account, status) - - trim(timeline_type, account.id) - - PushUpdateWorker.perform_async(account.id, status.id) if push_update_required?(timeline_type, account.id) - + def push_to_home(account, status) + return false unless add_to_feed(:home, account.id, status) + trim(:home, account.id) + PushUpdateWorker.perform_async(account.id, status.id, "timeline:#{account.id}") if push_update_required?("timeline:#{account.id}") true end - def unpush(timeline_type, account, status) - return false unless remove_from_feed(timeline_type, account, status) + def unpush_from_home(account, status) + return false unless remove_from_feed(:home, account.id, status) + Redis.current.publish("timeline:#{account.id}", Oj.dump(event: :delete, payload: status.id.to_s)) + true + end - payload = Oj.dump(event: :delete, payload: status.id.to_s) - Redis.current.publish("timeline:#{account.id}", payload) + def push_to_list(list, status) + return false unless add_to_feed(:list, list.id, status) + trim(:list, list.id) + PushUpdateWorker.perform_async(list.account_id, status.id, "timeline:list:#{list.id}") if push_update_required?("timeline:list:#{list.id}") + true + end + def unpush_from_list(list, status) + return false unless remove_from_feed(:list, list.id, status) + Redis.current.publish("timeline:list:#{list.id}", Oj.dump(event: :delete, payload: status.id.to_s)) true end def trim(type, account_id) timeline_key = key(type, account_id) - reblog_key = key(type, account_id, 'reblogs') + reblog_key = key(type, account_id, 'reblogs') + # Remove any items past the MAX_ITEMS'th entry in our feed redis.zremrangebyrank(timeline_key, '0', (-(FeedManager::MAX_ITEMS + 1)).to_s) # Get the score of the REBLOG_FALLOFF'th item in our feed, and stop # tracking anything after it for deduplication purposes. - falloff_rank = FeedManager::REBLOG_FALLOFF - 1 + falloff_rank = FeedManager::REBLOG_FALLOFF - 1 falloff_range = redis.zrevrange(timeline_key, falloff_rank, falloff_rank, with_scores: true) falloff_score = falloff_range&.first&.last&.to_i || 0 @@ -69,10 +77,6 @@ class FeedManager end end - def push_update_required?(timeline_type, account_id) - timeline_type != :home || redis.get("subscribed:timeline:#{account_id}").present? - end - def merge_into_timeline(from_account, into_account) timeline_key = key(:home, into_account.id) query = from_account.statuses.limit(FeedManager::MAX_ITEMS / 4) @@ -84,28 +88,28 @@ class FeedManager query.each do |status| next if status.direct_visibility? || filter?(:home, status, into_account) - add_to_feed(:home, into_account, status) + add_to_feed(:home, into_account.id, status) end trim(:home, into_account.id) end def unmerge_from_timeline(from_account, into_account) - timeline_key = key(:home, into_account.id) + timeline_key = key(:home, into_account.id) oldest_home_score = redis.zrange(timeline_key, 0, 0, with_scores: true)&.first&.last&.to_i || 0 from_account.statuses.select('id, reblog_of_id').where('id > ?', oldest_home_score).reorder(nil).find_each do |status| - remove_from_feed(:home, into_account, status) + remove_from_feed(:home, into_account.id, status) end end def clear_from_timeline(account, target_account) - timeline_key = key(:home, account.id) + timeline_key = key(:home, account.id) timeline_status_ids = redis.zrange(timeline_key, 0, -1) - target_statuses = Status.where(id: timeline_status_ids, account: target_account) + target_statuses = Status.where(id: timeline_status_ids, account: target_account) target_statuses.each do |status| - unpush(:home, account, status) + unpush_from_home(account, status) end end @@ -122,7 +126,7 @@ class FeedManager statuses.each do |status| next if filter_from_home?(status, account) - added += 1 if add_to_feed(:home, account, status) + added += 1 if add_to_feed(:home, account.id, status) end break unless added.zero? @@ -137,6 +141,10 @@ class FeedManager Redis.current end + def push_update_required?(timeline_id) + redis.exists("subscribed:#{timeline_id}") + end + def filter_from_home?(status, receiver_id) return false if receiver_id == status.account_id return true if status.reply? && (status.in_reply_to_id.nil? || status.in_reply_to_account_id.nil?) @@ -182,9 +190,9 @@ class FeedManager # added, and false if it was not added to the feed. Note that this is # an internal helper: callers must call trim or push updates if # either action is appropriate. - def add_to_feed(timeline_type, account, status) - timeline_key = key(timeline_type, account.id) - reblog_key = key(timeline_type, account.id, 'reblogs') + def add_to_feed(timeline_type, account_id, status) + timeline_key = key(timeline_type, account_id) + reblog_key = key(timeline_type, account_id, 'reblogs') if status.reblog? # If the original status or a reblog of it is within @@ -195,6 +203,7 @@ class FeedManager return false if !rank.nil? && rank < FeedManager::REBLOG_FALLOFF reblog_rank = redis.zrevrank(reblog_key, status.reblog_of_id) + if reblog_rank.nil? # This is not something we've already seen reblogged, so we # can just add it to the feed (and note that we're @@ -205,7 +214,7 @@ class FeedManager # Another reblog of the same status was already in the # REBLOG_FALLOFF most recent statuses, so we note that this # is an "extra" reblog, by storing it in reblog_set_key. - reblog_set_key = key(timeline_type, account.id, "reblogs:#{status.reblog_of_id}") + reblog_set_key = key(timeline_type, account_id, "reblogs:#{status.reblog_of_id}") redis.sadd(reblog_set_key, status.id) return false end @@ -220,8 +229,8 @@ class FeedManager # with reblogs, and returning true if a status was removed. As with # `add_to_feed`, this does not trigger push updates, so callers must # do so if appropriate. - def remove_from_feed(timeline_type, account, status) - timeline_key = key(timeline_type, account.id) + def remove_from_feed(timeline_type, account_id, status) + timeline_key = key(timeline_type, account_id) if status.reblog? # 1. If the reblogging status is not in the feed, stop. @@ -229,7 +238,7 @@ class FeedManager return false if status_rank.nil? # 2. Remove reblog from set of this status's reblogs. - reblog_set_key = key(timeline_type, account.id, "reblogs:#{status.reblog_of_id}") + reblog_set_key = key(timeline_type, account_id, "reblogs:#{status.reblog_of_id}") redis.srem(reblog_set_key, status.id) # 3. Re-insert another reblog or original into the feed if one @@ -244,7 +253,7 @@ class FeedManager # (outside conditional) else # If the original is getting deleted, no use for reblog references - redis.del(key(timeline_type, account.id, "reblogs:#{status.id}")) + redis.del(key(timeline_type, account_id, "reblogs:#{status.id}")) end redis.zrem(timeline_key, status.id) diff --git a/app/models/account.rb b/app/models/account.rb index bc01d2448..9353c40da 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -3,7 +3,7 @@ # # Table name: accounts # -# id :bigint not null, primary key +# id :integer not null, primary key # username :string default(""), not null # domain :string # secret :string default(""), not null @@ -53,6 +53,7 @@ class Account < ApplicationRecord include AccountInteractions include Attachmentable include Remotable + include Paginable enum protocol: [:ostatus, :activitypub] @@ -95,6 +96,10 @@ class Account < ApplicationRecord has_many :account_moderation_notes, dependent: :destroy has_many :targeted_moderation_notes, class_name: 'AccountModerationNote', foreign_key: :target_account_id, dependent: :destroy + # Lists + has_many :list_accounts, inverse_of: :account, dependent: :destroy + has_many :lists, through: :list_accounts + scope :remote, -> { where.not(domain: nil) } scope :local, -> { where(domain: nil) } scope :without_followers, -> { where(followers_count: 0) } diff --git a/app/models/account_domain_block.rb b/app/models/account_domain_block.rb index 9c98ec2a6..35810b6c2 100644 --- a/app/models/account_domain_block.rb +++ b/app/models/account_domain_block.rb @@ -3,11 +3,11 @@ # # Table name: account_domain_blocks # +# id :integer not null, primary key # domain :string # created_at :datetime not null # updated_at :datetime not null -# account_id :bigint -# id :bigint not null, primary key +# account_id :integer # class AccountDomainBlock < ApplicationRecord diff --git a/app/models/account_moderation_note.rb b/app/models/account_moderation_note.rb index 06f464850..3ac9b1ac1 100644 --- a/app/models/account_moderation_note.rb +++ b/app/models/account_moderation_note.rb @@ -3,10 +3,10 @@ # # Table name: account_moderation_notes # -# id :bigint not null, primary key +# id :integer not null, primary key # content :text not null -# account_id :bigint not null -# target_account_id :bigint not null +# account_id :integer not null +# target_account_id :integer not null # created_at :datetime not null # updated_at :datetime not null # diff --git a/app/models/block.rb b/app/models/block.rb index 5778f7e90..284abfe4c 100644 --- a/app/models/block.rb +++ b/app/models/block.rb @@ -3,11 +3,11 @@ # # Table name: blocks # +# id :integer not null, primary key # created_at :datetime not null # updated_at :datetime not null -# account_id :bigint not null -# id :bigint not null, primary key -# target_account_id :bigint not null +# account_id :integer not null +# target_account_id :integer not null # class Block < ApplicationRecord diff --git a/app/models/conversation.rb b/app/models/conversation.rb index e08532522..08c1ce945 100644 --- a/app/models/conversation.rb +++ b/app/models/conversation.rb @@ -3,7 +3,7 @@ # # Table name: conversations # -# id :bigint not null, primary key +# id :integer not null, primary key # uri :string # created_at :datetime not null # updated_at :datetime not null diff --git a/app/models/conversation_mute.rb b/app/models/conversation_mute.rb index 316865bd2..248cdfe6e 100644 --- a/app/models/conversation_mute.rb +++ b/app/models/conversation_mute.rb @@ -3,9 +3,9 @@ # # Table name: conversation_mutes # -# conversation_id :bigint not null -# account_id :bigint not null -# id :bigint not null, primary key +# id :integer not null, primary key +# conversation_id :integer not null +# account_id :integer not null # class ConversationMute < ApplicationRecord diff --git a/app/models/custom_emoji.rb b/app/models/custom_emoji.rb index 5723ebd5d..a77b53c98 100644 --- a/app/models/custom_emoji.rb +++ b/app/models/custom_emoji.rb @@ -3,7 +3,7 @@ # # Table name: custom_emojis # -# id :bigint not null, primary key +# id :integer not null, primary key # shortcode :string default(""), not null # domain :string # image_file_name :string diff --git a/app/models/domain_block.rb b/app/models/domain_block.rb index 557d0a19c..aea8919af 100644 --- a/app/models/domain_block.rb +++ b/app/models/domain_block.rb @@ -3,12 +3,12 @@ # # Table name: domain_blocks # +# id :integer not null, primary key # domain :string default(""), not null # created_at :datetime not null # updated_at :datetime not null # severity :integer default("silence") # reject_media :boolean default(FALSE), not null -# id :bigint not null, primary key # class DomainBlock < ApplicationRecord diff --git a/app/models/email_domain_block.rb b/app/models/email_domain_block.rb index 2c348197c..a104810d1 100644 --- a/app/models/email_domain_block.rb +++ b/app/models/email_domain_block.rb @@ -3,7 +3,7 @@ # # Table name: email_domain_blocks # -# id :bigint not null, primary key +# id :integer not null, primary key # domain :string default(""), not null # created_at :datetime not null # updated_at :datetime not null diff --git a/app/models/favourite.rb b/app/models/favourite.rb index f611aa6a9..c38838f2a 100644 --- a/app/models/favourite.rb +++ b/app/models/favourite.rb @@ -3,11 +3,11 @@ # # Table name: favourites # +# id :integer not null, primary key # created_at :datetime not null # updated_at :datetime not null -# account_id :bigint not null -# id :bigint not null, primary key -# status_id :bigint not null +# account_id :integer not null +# status_id :integer not null # class Favourite < ApplicationRecord diff --git a/app/models/feed.rb b/app/models/feed.rb index 5f7b7877a..d99f1ffb2 100644 --- a/app/models/feed.rb +++ b/app/models/feed.rb @@ -1,36 +1,27 @@ # frozen_string_literal: true class Feed - def initialize(type, account) - @type = type - @account = account + def initialize(type, id) + @type = type + @id = id end def get(limit, max_id = nil, since_id = nil) - if redis.exists("account:#{@account.id}:regeneration") - from_database(limit, max_id, since_id) - else - from_redis(limit, max_id, since_id) - end + from_redis(limit, max_id, since_id) end - private + protected def from_redis(limit, max_id, since_id) max_id = '+inf' if max_id.blank? since_id = '-inf' if since_id.blank? unhydrated = redis.zrevrangebyscore(key, "(#{max_id}", "(#{since_id}", limit: [0, limit], with_scores: true).map(&:first).map(&:to_i) - Status.where(id: unhydrated).cache_ids - end - def from_database(limit, max_id, since_id) - Status.as_home_timeline(@account) - .paginate_by_max_id(limit, max_id, since_id) - .reject { |status| FeedManager.instance.filter?(:home, status, @account.id) } + Status.where(id: unhydrated).cache_ids end def key - FeedManager.instance.key(@type, @account.id) + FeedManager.instance.key(@type, @id) end def redis diff --git a/app/models/follow.rb b/app/models/follow.rb index 3d5447fb1..795ecf55a 100644 --- a/app/models/follow.rb +++ b/app/models/follow.rb @@ -3,11 +3,11 @@ # # Table name: follows # +# id :integer not null, primary key # created_at :datetime not null # updated_at :datetime not null -# account_id :bigint not null -# id :bigint not null, primary key -# target_account_id :bigint not null +# account_id :integer not null +# target_account_id :integer not null # class Follow < ApplicationRecord diff --git a/app/models/follow_request.rb b/app/models/follow_request.rb index ce27fc921..fac91b513 100644 --- a/app/models/follow_request.rb +++ b/app/models/follow_request.rb @@ -3,11 +3,11 @@ # # Table name: follow_requests # +# id :integer not null, primary key # created_at :datetime not null # updated_at :datetime not null -# account_id :bigint not null -# id :bigint not null, primary key -# target_account_id :bigint not null +# account_id :integer not null +# target_account_id :integer not null # class FollowRequest < ApplicationRecord diff --git a/app/models/home_feed.rb b/app/models/home_feed.rb new file mode 100644 index 000000000..b943a34ce --- /dev/null +++ b/app/models/home_feed.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +class HomeFeed < Feed + def initialize(account) + @type = :home + @id = account.id + @account = account + end + + def get(limit, max_id = nil, since_id = nil) + if redis.exists("account:#{@account.id}:regeneration") + from_database(limit, max_id, since_id) + else + super + end + end + + private + + def from_database(limit, max_id, since_id) + Status.as_home_timeline(@account) + .paginate_by_max_id(limit, max_id, since_id) + .reject { |status| FeedManager.instance.filter?(:home, status, @account.id) } + end +end diff --git a/app/models/import.rb b/app/models/import.rb index 6f1278556..091fb3044 100644 --- a/app/models/import.rb +++ b/app/models/import.rb @@ -3,6 +3,7 @@ # # Table name: imports # +# id :integer not null, primary key # type :integer not null # approved :boolean default(FALSE), not null # created_at :datetime not null @@ -11,8 +12,7 @@ # data_content_type :string # data_file_size :integer # data_updated_at :datetime -# account_id :bigint not null -# id :bigint not null, primary key +# account_id :integer not null # class Import < ApplicationRecord diff --git a/app/models/list.rb b/app/models/list.rb new file mode 100644 index 000000000..5d7ba0065 --- /dev/null +++ b/app/models/list.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true +# == Schema Information +# +# Table name: lists +# +# id :integer not null, primary key +# account_id :integer +# title :string default(""), not null +# created_at :datetime not null +# updated_at :datetime not null +# + +class List < ApplicationRecord + include Paginable + + belongs_to :account + + has_many :list_accounts, inverse_of: :list, dependent: :destroy + has_many :accounts, through: :list_accounts + + validates :title, presence: true +end diff --git a/app/models/list_account.rb b/app/models/list_account.rb new file mode 100644 index 000000000..c08239aa0 --- /dev/null +++ b/app/models/list_account.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true +# == Schema Information +# +# Table name: list_accounts +# +# id :integer not null, primary key +# list_id :integer not null +# account_id :integer not null +# follow_id :integer not null +# + +class ListAccount < ApplicationRecord + belongs_to :list, required: true + belongs_to :account, required: true + belongs_to :follow, required: true + + before_validation :set_follow + + private + + def set_follow + self.follow = Follow.find_by(account_id: list.account_id, target_account_id: account.id) + end +end diff --git a/app/models/list_feed.rb b/app/models/list_feed.rb new file mode 100644 index 000000000..f371e4ed9 --- /dev/null +++ b/app/models/list_feed.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +class ListFeed < Feed + def initialize(list) + @type = :list + @id = list.id + end +end diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb index f05418925..abc5ab854 100644 --- a/app/models/media_attachment.rb +++ b/app/models/media_attachment.rb @@ -3,19 +3,19 @@ # # Table name: media_attachments # -# id :bigint not null, primary key -# status_id :bigint +# id :integer not null, primary key +# status_id :integer # file_file_name :string # file_content_type :string # file_file_size :integer # file_updated_at :datetime # remote_url :string default(""), not null -# account_id :bigint # created_at :datetime not null # updated_at :datetime not null # shortcode :string # type :integer default("image"), not null # file_meta :json +# account_id :integer # description :text # diff --git a/app/models/mention.rb b/app/models/mention.rb index fc089d365..14533e6a9 100644 --- a/app/models/mention.rb +++ b/app/models/mention.rb @@ -3,11 +3,11 @@ # # Table name: mentions # -# status_id :bigint +# id :integer not null, primary key +# status_id :integer # created_at :datetime not null # updated_at :datetime not null -# account_id :bigint -# id :bigint not null, primary key +# account_id :integer # class Mention < ApplicationRecord diff --git a/app/models/notification.rb b/app/models/notification.rb index c88af9021..a3ffb1f45 100644 --- a/app/models/notification.rb +++ b/app/models/notification.rb @@ -3,13 +3,13 @@ # # Table name: notifications # -# id :bigint not null, primary key -# account_id :bigint -# activity_id :bigint +# id :integer not null, primary key +# activity_id :integer # activity_type :string # created_at :datetime not null # updated_at :datetime not null -# from_account_id :bigint +# account_id :integer +# from_account_id :integer # class Notification < ApplicationRecord diff --git a/app/models/preview_card.rb b/app/models/preview_card.rb index 63c04b410..e2bf65d94 100644 --- a/app/models/preview_card.rb +++ b/app/models/preview_card.rb @@ -3,7 +3,7 @@ # # Table name: preview_cards # -# id :bigint not null, primary key +# id :integer not null, primary key # url :string default(""), not null # title :string default(""), not null # description :string default(""), not null diff --git a/app/models/report.rb b/app/models/report.rb index 99c90b7dd..c36f8db0a 100644 --- a/app/models/report.rb +++ b/app/models/report.rb @@ -3,15 +3,15 @@ # # Table name: reports # +# id :integer not null, primary key # status_ids :integer default([]), not null, is an Array # comment :text default(""), not null # action_taken :boolean default(FALSE), not null # created_at :datetime not null # updated_at :datetime not null -# account_id :bigint not null -# action_taken_by_account_id :bigint -# id :bigint not null, primary key -# target_account_id :bigint not null +# account_id :integer not null +# action_taken_by_account_id :integer +# target_account_id :integer not null # class Report < ApplicationRecord diff --git a/app/models/session_activation.rb b/app/models/session_activation.rb index 59565f877..d19489b36 100644 --- a/app/models/session_activation.rb +++ b/app/models/session_activation.rb @@ -3,15 +3,15 @@ # # Table name: session_activations # -# id :bigint not null, primary key -# user_id :bigint not null +# id :integer not null, primary key # session_id :string not null # created_at :datetime not null # updated_at :datetime not null # user_agent :string default(""), not null # ip :inet -# access_token_id :bigint -# web_push_subscription_id :bigint +# access_token_id :integer +# user_id :integer not null +# web_push_subscription_id :integer # # id :bigint not null, primary key diff --git a/app/models/setting.rb b/app/models/setting.rb index be68d3123..df93590ce 100644 --- a/app/models/setting.rb +++ b/app/models/setting.rb @@ -3,13 +3,13 @@ # # Table name: settings # +# id :integer not null, primary key # var :string not null # value :text # thing_type :string # created_at :datetime # updated_at :datetime -# id :bigint not null, primary key -# thing_id :bigint +# thing_id :integer # class Setting < RailsSettings::Base diff --git a/app/models/site_upload.rb b/app/models/site_upload.rb index ba2ca777b..8ffdc8313 100644 --- a/app/models/site_upload.rb +++ b/app/models/site_upload.rb @@ -3,7 +3,7 @@ # # Table name: site_uploads # -# id :bigint not null, primary key +# id :integer not null, primary key # var :string default(""), not null # file_file_name :string # file_content_type :string diff --git a/app/models/status.rb b/app/models/status.rb index b4f314311..26095070f 100644 --- a/app/models/status.rb +++ b/app/models/status.rb @@ -3,26 +3,26 @@ # # Table name: statuses # -# id :bigint not null, primary key +# id :integer not null, primary key # uri :string -# account_id :bigint not null # text :text default(""), not null # created_at :datetime not null # updated_at :datetime not null -# in_reply_to_id :bigint -# reblog_of_id :bigint +# in_reply_to_id :integer +# reblog_of_id :integer # url :string # sensitive :boolean default(FALSE), not null # visibility :integer default("public"), not null -# in_reply_to_account_id :bigint -# application_id :bigint # spoiler_text :text default(""), not null # reply :boolean default(FALSE), not null # favourites_count :integer default(0), not null # reblogs_count :integer default(0), not null # language :string -# conversation_id :bigint +# conversation_id :integer # local :boolean +# account_id :integer not null +# application_id :integer +# in_reply_to_account_id :integer # class Status < ApplicationRecord diff --git a/app/models/status_pin.rb b/app/models/status_pin.rb index 5795d07bf..a72c19750 100644 --- a/app/models/status_pin.rb +++ b/app/models/status_pin.rb @@ -3,9 +3,9 @@ # # Table name: status_pins # -# id :bigint not null, primary key -# account_id :bigint not null -# status_id :bigint not null +# id :integer not null, primary key +# account_id :integer not null +# status_id :integer not null # created_at :datetime not null # updated_at :datetime not null # diff --git a/app/models/stream_entry.rb b/app/models/stream_entry.rb index 50b900c3c..2ae034d93 100644 --- a/app/models/stream_entry.rb +++ b/app/models/stream_entry.rb @@ -3,13 +3,13 @@ # # Table name: stream_entries # -# activity_id :bigint +# id :integer not null, primary key +# activity_id :integer # activity_type :string # created_at :datetime not null # updated_at :datetime not null # hidden :boolean default(FALSE), not null -# account_id :bigint -# id :bigint not null, primary key +# account_id :integer # class StreamEntry < ApplicationRecord diff --git a/app/models/subscription.rb b/app/models/subscription.rb index bc50c5317..7f2eeab91 100644 --- a/app/models/subscription.rb +++ b/app/models/subscription.rb @@ -3,6 +3,7 @@ # # Table name: subscriptions # +# id :integer not null, primary key # callback_url :string default(""), not null # secret :string # expires_at :datetime @@ -11,8 +12,7 @@ # updated_at :datetime not null # last_successful_delivery_at :datetime # domain :string -# account_id :bigint not null -# id :bigint not null, primary key +# account_id :integer not null # class Subscription < ApplicationRecord diff --git a/app/models/tag.rb b/app/models/tag.rb index 6ebaf1145..0fa08e157 100644 --- a/app/models/tag.rb +++ b/app/models/tag.rb @@ -3,7 +3,7 @@ # # Table name: tags # -# id :bigint not null, primary key +# id :integer not null, primary key # name :string default(""), not null # created_at :datetime not null # updated_at :datetime not null diff --git a/app/models/user.rb b/app/models/user.rb index 326b871a1..b9b228c00 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -3,7 +3,7 @@ # # Table name: users # -# id :bigint not null, primary key +# id :integer not null, primary key # email :string default(""), not null # created_at :datetime not null # updated_at :datetime not null @@ -30,7 +30,7 @@ # last_emailed_at :datetime # otp_backup_codes :string is an Array # filtered_languages :string default([]), not null, is an Array -# account_id :bigint not null +# account_id :integer not null # disabled :boolean default(FALSE), not null # moderator :boolean default(FALSE), not null # diff --git a/app/models/web/push_subscription.rb b/app/models/web/push_subscription.rb index a41906227..5aee92d27 100644 --- a/app/models/web/push_subscription.rb +++ b/app/models/web/push_subscription.rb @@ -3,7 +3,7 @@ # # Table name: web_push_subscriptions # -# id :bigint not null, primary key +# id :integer not null, primary key # endpoint :string not null # key_p256dh :string not null # key_auth :string not null diff --git a/app/models/web/setting.rb b/app/models/web/setting.rb index 6d08c4d35..12b9d1226 100644 --- a/app/models/web/setting.rb +++ b/app/models/web/setting.rb @@ -3,11 +3,11 @@ # # Table name: web_settings # +# id :integer not null, primary key # data :json # created_at :datetime not null # updated_at :datetime not null -# id :bigint not null, primary key -# user_id :bigint +# user_id :integer # class Web::Setting < ApplicationRecord diff --git a/app/serializers/rest/list_serializer.rb b/app/serializers/rest/list_serializer.rb new file mode 100644 index 000000000..c0150888e --- /dev/null +++ b/app/serializers/rest/list_serializer.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +class REST::ListSerializer < ActiveModel::Serializer + attributes :id, :title +end diff --git a/app/services/batched_remove_status_service.rb b/app/services/batched_remove_status_service.rb index 676a5d04d..6b6b0c418 100644 --- a/app/services/batched_remove_status_service.rb +++ b/app/services/batched_remove_status_service.rb @@ -30,6 +30,7 @@ class BatchedRemoveStatusService < BaseService account = account_statuses.first.account unpush_from_home_timelines(account, account_statuses) + unpush_from_list_timelines(account, account_statuses) if account.local? batch_stream_entries(account, account_statuses) @@ -79,7 +80,15 @@ class BatchedRemoveStatusService < BaseService recipients.each do |follower| statuses.each do |status| - FeedManager.instance.unpush(:home, follower, status) + FeedManager.instance.unpush_from_home(follower, status) + end + end + end + + def unpush_from_list_timelines(account, statuses) + account.lists.select(:id, :account_id).each do |list| + statuses.each do |status| + FeedManager.instance.unpush_from_list(list, status) end end end diff --git a/app/services/fan_out_on_write_service.rb b/app/services/fan_out_on_write_service.rb index 47a47a735..bbaf3094b 100644 --- a/app/services/fan_out_on_write_service.rb +++ b/app/services/fan_out_on_write_service.rb @@ -14,6 +14,7 @@ class FanOutOnWriteService < BaseService deliver_to_mentioned_followers(status) else deliver_to_followers(status) + deliver_to_lists(status) end return if status.account.silenced? || !status.public_visibility? || status.reblog? @@ -30,7 +31,7 @@ class FanOutOnWriteService < BaseService def deliver_to_self(status) Rails.logger.debug "Delivering status #{status.id} to author" - FeedManager.instance.push(:home, status.account, status) + FeedManager.instance.push_to_home(status.account, status) end def deliver_to_followers(status) @@ -38,7 +39,17 @@ class FanOutOnWriteService < BaseService status.account.followers.where(domain: nil).joins(:user).where('users.current_sign_in_at > ?', 14.days.ago).select(:id).reorder(nil).find_in_batches do |followers| FeedInsertWorker.push_bulk(followers) do |follower| - [status.id, follower.id] + [status.id, follower.id, :home] + end + end + end + + def deliver_to_lists(status) + Rails.logger.debug "Delivering status #{status.id} to lists" + + status.account.lists.joins(account: :user).where('users.current_sign_in_at > ?', 14.days.ago).select(:id).reorder(nil).find_in_batches do |lists| + FeedInsertWorker.push_bulk(lists) do |list| + [status.id, list.id, :list] end end end @@ -49,7 +60,7 @@ class FanOutOnWriteService < BaseService status.mentions.includes(:account).each do |mention| mentioned_account = mention.account next if !mentioned_account.local? || !mentioned_account.following?(status.account) || FeedManager.instance.filter?(:home, status, mention.account_id) - FeedManager.instance.push(:home, mentioned_account, status) + FeedManager.instance.push_to_home(mentioned_account, status) end end diff --git a/app/services/remove_status_service.rb b/app/services/remove_status_service.rb index 96d9208cc..c75627205 100644 --- a/app/services/remove_status_service.rb +++ b/app/services/remove_status_service.rb @@ -14,6 +14,7 @@ class RemoveStatusService < BaseService remove_from_self if status.account.local? remove_from_followers + remove_from_lists remove_from_affected remove_reblogs remove_from_hashtags @@ -30,12 +31,18 @@ class RemoveStatusService < BaseService private def remove_from_self - unpush(:home, @account, @status) + FeedManager.instance.unpush_from_home(@account, @status) end def remove_from_followers @account.followers.local.find_each do |follower| - unpush(:home, follower, @status) + FeedManager.instance.unpush_from_home(follower, @status) + end + end + + def remove_from_lists + @account.lists.select(:id, :account_id).find_each do |list| + FeedManager.instance.unpush_from_list(list, @status) end end @@ -101,10 +108,6 @@ class RemoveStatusService < BaseService end end - def unpush(type, receiver, status) - FeedManager.instance.unpush(type, receiver, status) - end - def remove_from_hashtags return unless @status.public_visibility? diff --git a/app/workers/feed_insert_worker.rb b/app/workers/feed_insert_worker.rb index 65c02d3ef..1ae3c877b 100644 --- a/app/workers/feed_insert_worker.rb +++ b/app/workers/feed_insert_worker.rb @@ -3,34 +3,41 @@ class FeedInsertWorker include Sidekiq::Worker - attr_reader :status, :follower - - def perform(status_id, follower_id) - @status = Status.find_by(id: status_id) - @follower = Account.find_by(id: follower_id) + def perform(status_id, id, type = :home) + @type = type.to_sym + @status = Status.find(status_id) + + case @type + when :home + @follower = Account.find(id) + when :list + @list = List.find(id) + @follower = @list.account + end check_and_insert + rescue ActiveRecord::RecordNotFound + true end private def check_and_insert - if records_available? - perform_push unless feed_filtered? - else - true - end - end - - def records_available? - status.present? && follower.present? + perform_push unless feed_filtered? end def feed_filtered? - FeedManager.instance.filter?(:home, status, follower.id) + # Note: Lists are a variation of home, so the filtering rules + # of home apply to both + FeedManager.instance.filter?(:home, @status, @follower.id) end def perform_push - FeedManager.instance.push(:home, follower, status) + case @type + when :home + FeedManager.instance.push_to_home(@follower, @status) + when :list + FeedManager.instance.push_to_list(@list, @status) + end end end diff --git a/app/workers/push_update_worker.rb b/app/workers/push_update_worker.rb index 697cbd6a6..d76d73d96 100644 --- a/app/workers/push_update_worker.rb +++ b/app/workers/push_update_worker.rb @@ -3,12 +3,13 @@ class PushUpdateWorker include Sidekiq::Worker - def perform(account_id, status_id) - account = Account.find(account_id) - status = Status.find(status_id) - message = InlineRenderer.render(status, account, :status) + def perform(account_id, status_id, timeline_id = nil) + account = Account.find(account_id) + status = Status.find(status_id) + message = InlineRenderer.render(status, account, :status) + timeline_id = "timeline:#{account.id}" if timeline_id.nil? - Redis.current.publish("timeline:#{account.id}", Oj.dump(event: :update, payload: message, queued_at: (Time.now.to_f * 1000.0).to_i)) + Redis.current.publish(timeline_id, Oj.dump(event: :update, payload: message, queued_at: (Time.now.to_f * 1000.0).to_i)) rescue ActiveRecord::RecordNotFound true end -- cgit From 130aa90d5500f481c181a16012724b5f81d62616 Mon Sep 17 00:00:00 2001 From: David Yip Date: Fri, 17 Nov 2017 17:41:15 -0600 Subject: Update annotations on Follow, FollowRequest, and Mute. Follow and FollowRequest had conflicts in their schema annotations, so I ran latest migrations and let annotate_models fix them up. --- app/models/follow.rb | 5 ++--- app/models/follow_request.rb | 5 ++--- app/models/mute.rb | 6 +++--- db/schema.rb | 1 - 4 files changed, 7 insertions(+), 10 deletions(-) (limited to 'app') diff --git a/app/models/follow.rb b/app/models/follow.rb index b82da7227..3fb665afc 100644 --- a/app/models/follow.rb +++ b/app/models/follow.rb @@ -6,9 +6,8 @@ # id :integer not null, primary key # created_at :datetime not null # updated_at :datetime not null -# account_id :bigint not null -# id :bigint not null, primary key -# target_account_id :bigint not null +# account_id :integer not null +# target_account_id :integer not null # show_reblogs :boolean default(TRUE), not null # diff --git a/app/models/follow_request.rb b/app/models/follow_request.rb index 21f01e18b..ebf6959ce 100644 --- a/app/models/follow_request.rb +++ b/app/models/follow_request.rb @@ -6,9 +6,8 @@ # id :integer not null, primary key # created_at :datetime not null # updated_at :datetime not null -# account_id :bigint not null -# id :bigint not null, primary key -# target_account_id :bigint not null +# account_id :integer not null +# target_account_id :integer not null # show_reblogs :boolean default(TRUE), not null # diff --git a/app/models/mute.rb b/app/models/mute.rb index 74b445c0b..ca984641a 100644 --- a/app/models/mute.rb +++ b/app/models/mute.rb @@ -3,12 +3,12 @@ # # Table name: mutes # -# id :bigint not null, primary key +# id :integer not null, primary key # created_at :datetime not null # updated_at :datetime not null -# account_id :bigint not null -# target_account_id :bigint not null # hide_notifications :boolean default(TRUE), not null +# account_id :integer not null +# target_account_id :integer not null # class Mute < ApplicationRecord diff --git a/db/schema.rb b/db/schema.rb index c2de16885..10e35cd7d 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -234,7 +234,6 @@ ActiveRecord::Schema.define(version: 20171116161857) do t.boolean "hide_notifications", default: true, null: false t.bigint "account_id", null: false t.bigint "target_account_id", null: false - t.boolean "hide_notifications", default: true, null: false t.index ["account_id", "target_account_id"], name: "index_mutes_on_account_id_and_target_account_id", unique: true end -- cgit