diff options
Diffstat (limited to 'app/models')
47 files changed, 1530 insertions, 559 deletions
diff --git a/app/models/account.rb b/app/models/account.rb index 96f23979f..e41fdf003 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -58,15 +58,16 @@ class Account < ApplicationRecord hub_url ) - USERNAME_RE = /[a-z0-9_]+([a-z0-9_\.-]+[a-z0-9_]+)?/i - MENTION_RE = /(?<=^|[^\/[:word:]])@((#{USERNAME_RE})(?:@[[:word:]\.\-]+[a-z0-9]+)?)/i + USERNAME_RE = /[a-z0-9_]+([a-z0-9_\.-]+[a-z0-9_]+)?/i + MENTION_RE = /(?<=^|[^\/[:word:]])@((#{USERNAME_RE})(?:@[[:word:]\.\-]+[[:word:]]+)?)/i + URL_PREFIX_RE = /\Ahttp(s?):\/\/[^\/]+/ + include Attachmentable include AccountAssociations include AccountAvatar include AccountFinderConcern include AccountHeader include AccountInteractions - include Attachmentable include Paginable include AccountCounters include DomainNormalizable @@ -126,8 +127,9 @@ class Account < ApplicationRecord delegate :email, :unconfirmed_email, - :current_sign_in_ip, :current_sign_in_at, + :created_at, + :sign_up_ip, :confirmed?, :approved?, :pending?, @@ -146,7 +148,7 @@ class Account < ApplicationRecord delegate :chosen_languages, to: :user, prefix: false, allow_nil: true - update_index('accounts#account', :self) + update_index('accounts', :self) def local? domain.nil? @@ -236,11 +238,11 @@ class Account < ApplicationRecord suspended? && deletion_request.present? end - def suspend!(date: Time.now.utc, origin: :local) + def suspend!(date: Time.now.utc, origin: :local, block_email: true) transaction do create_deletion_request! update!(suspended_at: date, suspension_origin: origin) - create_canonical_email_block! + create_canonical_email_block! if block_email end end @@ -299,7 +301,11 @@ class Account < ApplicationRecord end def fields - (self[:fields] || []).map { |f| Field.new(self, f) } + (self[:fields] || []).map do |f| + Field.new(self, f) + rescue + nil + end.compact end def fields_attributes=(attributes) @@ -377,7 +383,7 @@ class Account < ApplicationRecord def synchronization_uri_prefix return 'local' if local? - @synchronization_uri_prefix ||= uri[/http(s?):\/\/[^\/]+\//] + @synchronization_uri_prefix ||= "#{uri[URL_PREFIX_RE]}/" end class Field < ActiveModelSerializers::Model @@ -423,6 +429,9 @@ class Account < ApplicationRecord end class << self + DISALLOWED_TSQUERY_CHARACTERS = /['?\\:‘’]/.freeze + TEXTSEARCH = "(setweight(to_tsvector('simple', accounts.display_name), 'A') || setweight(to_tsvector('simple', accounts.username), 'B') || setweight(to_tsvector('simple', coalesce(accounts.domain, '')), 'C'))" + def readonly_attributes super - %w(statuses_count following_count followers_count) end @@ -433,97 +442,99 @@ class Account < ApplicationRecord end def search_for(terms, limit = 10, offset = 0) - textsearch, query = generate_query_for_search(terms) + tsquery = generate_query_for_search(terms) sql = <<-SQL.squish SELECT accounts.*, - ts_rank_cd(#{textsearch}, #{query}, 32) AS rank + ts_rank_cd(#{TEXTSEARCH}, to_tsquery('simple', :tsquery), 32) AS rank FROM accounts - WHERE #{query} @@ #{textsearch} + WHERE to_tsquery('simple', :tsquery) @@ #{TEXTSEARCH} AND accounts.suspended_at IS NULL AND accounts.moved_to_account_id IS NULL ORDER BY rank DESC - LIMIT ? OFFSET ? + LIMIT :limit OFFSET :offset SQL - records = find_by_sql([sql, limit, offset]) + records = find_by_sql([sql, limit: limit, offset: offset, tsquery: tsquery]) ActiveRecord::Associations::Preloader.new.preload(records, :account_stat) records end def advanced_search_for(terms, account, limit = 10, following = false, offset = 0) - textsearch, query = generate_query_for_search(terms) + tsquery = generate_query_for_search(terms) + sql = advanced_search_for_sql_template(following) + records = find_by_sql([sql, id: account.id, limit: limit, offset: offset, tsquery: tsquery]) + ActiveRecord::Associations::Preloader.new.preload(records, :account_stat) + records + end + + def from_text(text) + return [] if text.blank? + + text.scan(MENTION_RE).map { |match| match.first.split('@', 2) }.uniq.filter_map do |(username, domain)| + domain = begin + if TagManager.instance.local_domain?(domain) + nil + else + TagManager.instance.normalize_domain(domain) + end + end + EntityCache.instance.mention(username, domain) + end + end + + private + + def generate_query_for_search(unsanitized_terms) + terms = unsanitized_terms.gsub(DISALLOWED_TSQUERY_CHARACTERS, ' ') + + # The final ":*" is for prefix search. + # The trailing space does not seem to fit any purpose, but `to_tsquery` + # behaves differently with and without a leading space if the terms start + # with `./`, `../`, or `.. `. I don't understand why, so, in doubt, keep + # the same query. + "' #{terms} ':*" + end + def advanced_search_for_sql_template(following) if following - sql = <<-SQL.squish + <<-SQL.squish WITH first_degree AS ( SELECT target_account_id FROM follows - WHERE account_id = ? + WHERE account_id = :id UNION ALL - SELECT ? + SELECT :id ) SELECT accounts.*, - (count(f.id) + 1) * ts_rank_cd(#{textsearch}, #{query}, 32) AS rank + (count(f.id) + 1) * ts_rank_cd(#{TEXTSEARCH}, to_tsquery('simple', :tsquery), 32) AS rank FROM accounts - LEFT OUTER JOIN follows AS f ON (accounts.id = f.account_id AND f.target_account_id = ?) + LEFT OUTER JOIN follows AS f ON (accounts.id = f.account_id AND f.target_account_id = :id) WHERE accounts.id IN (SELECT * FROM first_degree) - AND #{query} @@ #{textsearch} + AND to_tsquery('simple', :tsquery) @@ #{TEXTSEARCH} AND accounts.suspended_at IS NULL AND accounts.moved_to_account_id IS NULL GROUP BY accounts.id ORDER BY rank DESC - LIMIT ? OFFSET ? + LIMIT :limit OFFSET :offset SQL - - records = find_by_sql([sql, account.id, account.id, account.id, limit, offset]) else - sql = <<-SQL.squish + <<-SQL.squish SELECT accounts.*, - (count(f.id) + 1) * ts_rank_cd(#{textsearch}, #{query}, 32) AS rank + (count(f.id) + 1) * ts_rank_cd(#{TEXTSEARCH}, to_tsquery('simple', :tsquery), 32) AS rank FROM accounts - LEFT OUTER JOIN follows AS f ON (accounts.id = f.account_id AND f.target_account_id = ?) OR (accounts.id = f.target_account_id AND f.account_id = ?) - WHERE #{query} @@ #{textsearch} + LEFT OUTER JOIN follows AS f ON (accounts.id = f.account_id AND f.target_account_id = :id) OR (accounts.id = f.target_account_id AND f.account_id = :id) + WHERE to_tsquery('simple', :tsquery) @@ #{TEXTSEARCH} AND accounts.suspended_at IS NULL AND accounts.moved_to_account_id IS NULL GROUP BY accounts.id ORDER BY rank DESC - LIMIT ? OFFSET ? + LIMIT :limit OFFSET :offset SQL - - records = find_by_sql([sql, account.id, account.id, limit, offset]) end - - ActiveRecord::Associations::Preloader.new.preload(records, :account_stat) - records - end - - def from_text(text) - return [] if text.blank? - - text.scan(MENTION_RE).map { |match| match.first.split('@', 2) }.uniq.filter_map do |(username, domain)| - domain = begin - if TagManager.instance.local_domain?(domain) - nil - else - TagManager.instance.normalize_domain(domain) - end - end - EntityCache.instance.mention(username, domain) - end - end - - private - - def generate_query_for_search(terms) - terms = Arel.sql(connection.quote(terms.gsub(/['?\\:]/, ' '))) - textsearch = "(setweight(to_tsvector('simple', accounts.display_name), 'A') || setweight(to_tsvector('simple', accounts.username), 'B') || setweight(to_tsvector('simple', coalesce(accounts.domain, '')), 'C'))" - query = "to_tsquery('simple', ''' ' || #{terms} || ' ''' || ':*')" - - [textsearch, query] end end diff --git a/app/models/account_filter.rb b/app/models/account_filter.rb index 2b001385f..dcb174122 100644 --- a/app/models/account_filter.rb +++ b/app/models/account_filter.rb @@ -2,18 +2,15 @@ class AccountFilter KEYS = %i( - local - remote - by_domain - active - pending - silenced - suspended + origin + status + permissions username + by_domain display_name email ip - staff + invited_by order ).freeze @@ -21,11 +18,10 @@ class AccountFilter def initialize(params) @params = params - set_defaults! end def results - scope = Account.includes(:user).reorder(nil) + scope = Account.includes(:account_stat, user: [:ips, :invite_request]).without_instance_actor.reorder(nil) params.each do |key, value| scope.merge!(scope_for(key, value.to_s.strip)) if value.present? @@ -36,30 +32,16 @@ class AccountFilter private - def set_defaults! - params['local'] = '1' if params['remote'].blank? - params['active'] = '1' if params['suspended'].blank? && params['silenced'].blank? && params['pending'].blank? - params['order'] = 'recent' if params['order'].blank? - end - def scope_for(key, value) case key.to_s - when 'local' - Account.local.without_instance_actor - when 'remote' - Account.remote + when 'origin' + origin_scope(value) + when 'permissions' + permissions_scope(value) + when 'status' + status_scope(value) when 'by_domain' Account.where(domain: value) - when 'active' - Account.without_suspended - when 'pending' - accounts_with_users.merge(User.pending) - when 'disabled' - accounts_with_users.merge(User.disabled) - when 'silenced' - Account.silenced - when 'suspended' - Account.suspended when 'username' Account.matches_username(value) when 'display_name' @@ -68,8 +50,8 @@ class AccountFilter accounts_with_users.merge(User.matches_email(value)) when 'ip' valid_ip?(value) ? accounts_with_users.merge(User.matches_ip(value)) : Account.none - when 'staff' - accounts_with_users.merge(User.staff) + when 'invited_by' + invited_by_scope(value) when 'order' order_scope(value) else @@ -77,21 +59,56 @@ class AccountFilter end end + def origin_scope(value) + case value.to_s + when 'local' + Account.local + when 'remote' + Account.remote + else + raise "Unknown origin: #{value}" + end + end + + def status_scope(value) + case value.to_s + when 'active' + Account.without_suspended + when 'pending' + accounts_with_users.merge(User.pending) + when 'suspended' + Account.suspended + else + raise "Unknown status: #{value}" + end + end + def order_scope(value) - case value + case value.to_s when 'active' - params['remote'] ? Account.joins(:account_stat).by_recent_status : Account.joins(:user).by_recent_sign_in + accounts_with_users.left_joins(:account_stat).order(Arel.sql('coalesce(users.current_sign_in_at, account_stats.last_status_at, to_timestamp(0)) desc, accounts.id desc')) when 'recent' Account.recent - when 'alphabetic' - Account.alphabetic else raise "Unknown order: #{value}" end end + def invited_by_scope(value) + Account.left_joins(user: :invite).merge(Invite.where(user_id: value.to_s)) + end + + def permissions_scope(value) + case value.to_s + when 'staff' + accounts_with_users.merge(User.staff) + else + raise "Unknown permissions: #{value}" + end + end + def accounts_with_users - Account.joins(:user) + Account.left_joins(:user) end def valid_ip?(value) diff --git a/app/models/account_identity_proof.rb b/app/models/account_identity_proof.rb deleted file mode 100644 index 10b66cccf..000000000 --- a/app/models/account_identity_proof.rb +++ /dev/null @@ -1,46 +0,0 @@ -# frozen_string_literal: true -# == Schema Information -# -# Table name: account_identity_proofs -# -# id :bigint(8) not null, primary key -# account_id :bigint(8) -# provider :string default(""), not null -# provider_username :string default(""), not null -# token :text default(""), not null -# verified :boolean default(FALSE), not null -# live :boolean default(FALSE), not null -# created_at :datetime not null -# updated_at :datetime not null -# - -class AccountIdentityProof < ApplicationRecord - belongs_to :account - - validates :provider, inclusion: { in: ProofProvider::SUPPORTED_PROVIDERS } - validates :provider_username, format: { with: /\A[a-z0-9_]+\z/i }, length: { minimum: 2, maximum: 30 } - validates :provider_username, uniqueness: { scope: [:account_id, :provider] } - validates :token, format: { with: /\A[a-f0-9]+\z/ }, length: { maximum: 66 } - - validate :validate_with_provider, if: :token_changed? - - scope :active, -> { where(verified: true, live: true) } - - after_commit :queue_worker, if: :saved_change_to_token? - - delegate :refresh!, :on_success_path, :badge, to: :provider_instance - - def provider_instance - @provider_instance ||= ProofProvider.find(provider, self) - end - - private - - def queue_worker - provider_instance.worker_class.perform_async(id) - end - - def validate_with_provider - provider_instance.validate! - end -end diff --git a/app/models/account_note.rb b/app/models/account_note.rb index bf61df923..b338bc92f 100644 --- a/app/models/account_note.rb +++ b/app/models/account_note.rb @@ -17,4 +17,5 @@ class AccountNote < ApplicationRecord belongs_to :target_account, class_name: 'Account' validates :account_id, uniqueness: { scope: :target_account_id } + validates :comment, length: { maximum: 2_000 } end diff --git a/app/models/account_stat.rb b/app/models/account_stat.rb index 44da4f0d0..b49827267 100644 --- a/app/models/account_stat.rb +++ b/app/models/account_stat.rb @@ -15,8 +15,9 @@ class AccountStat < ApplicationRecord self.locking_column = nil + self.ignored_columns = %w(lock_version) belongs_to :account, inverse_of: :account_stat - update_index('accounts#account', :account) + update_index('accounts', :account) end diff --git a/app/models/account_statuses_cleanup_policy.rb b/app/models/account_statuses_cleanup_policy.rb new file mode 100644 index 000000000..0f78c1a54 --- /dev/null +++ b/app/models/account_statuses_cleanup_policy.rb @@ -0,0 +1,171 @@ +# frozen_string_literal: true + +# == Schema Information +# +# Table name: account_statuses_cleanup_policies +# +# id :bigint(8) not null, primary key +# account_id :bigint(8) not null +# enabled :boolean default(TRUE), not null +# min_status_age :integer default(1209600), not null +# keep_direct :boolean default(TRUE), not null +# keep_pinned :boolean default(TRUE), not null +# keep_polls :boolean default(FALSE), not null +# keep_media :boolean default(FALSE), not null +# keep_self_fav :boolean default(TRUE), not null +# keep_self_bookmark :boolean default(TRUE), not null +# min_favs :integer +# min_reblogs :integer +# created_at :datetime not null +# updated_at :datetime not null +# +class AccountStatusesCleanupPolicy < ApplicationRecord + include Redisable + + ALLOWED_MIN_STATUS_AGE = [ + 2.weeks.seconds, + 1.month.seconds, + 2.months.seconds, + 3.months.seconds, + 6.months.seconds, + 1.year.seconds, + 2.years.seconds, + ].freeze + + EXCEPTION_BOOLS = %w(keep_direct keep_pinned keep_polls keep_media keep_self_fav keep_self_bookmark).freeze + EXCEPTION_THRESHOLDS = %w(min_favs min_reblogs).freeze + + # Depending on the cleanup policy, the query to discover the next + # statuses to delete my get expensive if the account has a lot of old + # statuses otherwise excluded from deletion by the other exceptions. + # + # Therefore, `EARLY_SEARCH_CUTOFF` is meant to be the maximum number of + # old statuses to be considered for deletion prior to checking exceptions. + # + # This is used in `compute_cutoff_id` to provide a `max_id` to + # `statuses_to_delete`. + EARLY_SEARCH_CUTOFF = 5_000 + + belongs_to :account + + validates :min_status_age, inclusion: { in: ALLOWED_MIN_STATUS_AGE } + validates :min_favs, numericality: { greater_than_or_equal_to: 1, allow_nil: true } + validates :min_reblogs, numericality: { greater_than_or_equal_to: 1, allow_nil: true } + validate :validate_local_account + + before_save :update_last_inspected + + def statuses_to_delete(limit = 50, max_id = nil, min_id = nil) + scope = account.statuses + scope.merge!(old_enough_scope(max_id)) + scope = scope.where(Status.arel_table[:id].gteq(min_id)) if min_id.present? + scope.merge!(without_popular_scope) unless min_favs.nil? && min_reblogs.nil? + scope.merge!(without_direct_scope) if keep_direct? + scope.merge!(without_pinned_scope) if keep_pinned? + scope.merge!(without_poll_scope) if keep_polls? + scope.merge!(without_media_scope) if keep_media? + scope.merge!(without_self_fav_scope) if keep_self_fav? + scope.merge!(without_self_bookmark_scope) if keep_self_bookmark? + + scope.reorder(id: :asc).limit(limit) + end + + # This computes a toot id such that: + # - the toot would be old enough to be candidate for deletion + # - there are at most EARLY_SEARCH_CUTOFF toots between the last inspected toot and this one + # + # The idea is to limit expensive SQL queries when an account has lots of toots excluded from + # deletion, while not starting anew on each run. + def compute_cutoff_id + min_id = last_inspected || 0 + max_id = Mastodon::Snowflake.id_at(min_status_age.seconds.ago, with_random: false) + subquery = account.statuses.where(Status.arel_table[:id].gteq(min_id)).where(Status.arel_table[:id].lteq(max_id)) + subquery = subquery.select(:id).reorder(id: :asc).limit(EARLY_SEARCH_CUTOFF) + + # We're textually interpolating a subquery here as ActiveRecord seem to not provide + # a way to apply the limit to the subquery + Status.connection.execute("SELECT MAX(id) FROM (#{subquery.to_sql}) t").values.first.first + end + + # The most important thing about `last_inspected` is that any toot older than it is guaranteed + # not to be kept by the policy regardless of its age. + def record_last_inspected(last_id) + redis.set("account_cleanup:#{account.id}", last_id, ex: 1.week.seconds) + end + + def last_inspected + redis.get("account_cleanup:#{account.id}")&.to_i + end + + def invalidate_last_inspected(status, action) + last_value = last_inspected + return if last_value.nil? || status.id > last_value || status.account_id != account_id + + case action + when :unbookmark + return unless keep_self_bookmark? + when :unfav + return unless keep_self_fav? + when :unpin + return unless keep_pinned? + end + + record_last_inspected(status.id) + end + + private + + def update_last_inspected + if EXCEPTION_BOOLS.map { |name| attribute_change_to_be_saved(name) }.compact.include?([true, false]) + # Policy has been widened in such a way that any previously-inspected status + # may need to be deleted, so we'll have to start again. + redis.del("account_cleanup:#{account.id}") + end + if EXCEPTION_THRESHOLDS.map { |name| attribute_change_to_be_saved(name) }.compact.any? { |old, new| old.present? && (new.nil? || new > old) } + redis.del("account_cleanup:#{account.id}") + end + end + + def validate_local_account + errors.add(:account, :invalid) unless account&.local? + end + + def without_direct_scope + Status.where.not(visibility: :direct) + end + + def old_enough_scope(max_id = nil) + # Filtering on `id` rather than `min_status_age` ago will treat + # non-snowflake statuses as older than they really are, but Mastodon + # has switched to snowflake IDs significantly over 2 years ago anyway. + max_id = [max_id, Mastodon::Snowflake.id_at(min_status_age.seconds.ago, with_random: false)].compact.min + Status.where(Status.arel_table[:id].lteq(max_id)) + end + + def without_self_fav_scope + Status.where('NOT EXISTS (SELECT * FROM favourites fav WHERE fav.account_id = statuses.account_id AND fav.status_id = statuses.id)') + end + + def without_self_bookmark_scope + Status.where('NOT EXISTS (SELECT * FROM bookmarks bookmark WHERE bookmark.account_id = statuses.account_id AND bookmark.status_id = statuses.id)') + end + + def without_pinned_scope + Status.where('NOT EXISTS (SELECT * FROM status_pins pin WHERE pin.account_id = statuses.account_id AND pin.status_id = statuses.id)') + end + + def without_media_scope + Status.where('NOT EXISTS (SELECT * FROM media_attachments media WHERE media.status_id = statuses.id)') + end + + def without_poll_scope + Status.where(poll_id: nil) + end + + def without_popular_scope + scope = Status.left_joins(:status_stat) + scope = scope.where('COALESCE(status_stats.reblogs_count, 0) < ?', min_reblogs) unless min_reblogs.nil? + scope = scope.where('COALESCE(status_stats.favourites_count, 0) < ?', min_favs) unless min_favs.nil? + scope + end +end diff --git a/app/models/account_warning.rb b/app/models/account_warning.rb index 5efc924d5..fc0d988fd 100644 --- a/app/models/account_warning.rb +++ b/app/models/account_warning.rb @@ -10,14 +10,30 @@ # text :text default(""), not null # created_at :datetime not null # updated_at :datetime not null +# report_id :bigint(8) +# status_ids :string is an Array # class AccountWarning < ApplicationRecord - enum action: %i(none disable sensitive silence suspend), _suffix: :action + enum action: { + none: 0, + disable: 1_000, + delete_statuses: 1_500, + sensitive: 2_000, + silence: 3_000, + suspend: 4_000, + }, _suffix: :action belongs_to :account, inverse_of: :account_warnings - belongs_to :target_account, class_name: 'Account', inverse_of: :targeted_account_warnings + belongs_to :target_account, class_name: 'Account', inverse_of: :strikes + belongs_to :report, optional: true - scope :latest, -> { order(created_at: :desc) } + has_one :appeal, dependent: :destroy + + scope :latest, -> { order(id: :desc) } scope :custom, -> { where.not(text: '') } + + def statuses + Status.with_discarded.where(id: status_ids || []) + end end diff --git a/app/models/admin/account_action.rb b/app/models/admin/account_action.rb index bf222391f..d3be4be3f 100644 --- a/app/models/admin/account_action.rb +++ b/app/models/admin/account_action.rb @@ -33,7 +33,7 @@ class Admin::AccountAction def save! ApplicationRecord.transaction do process_action! - process_warning! + process_strike! end process_email! @@ -74,20 +74,14 @@ class Admin::AccountAction end end - def process_warning! - return unless warnable? - - authorize(target_account, :warn?) - - @warning = AccountWarning.create!(target_account: target_account, - account: current_account, - action: type, - text: text_for_warning) - - # A log entry is only interesting if the warning contains - # custom text from someone. Otherwise it's just noise. - - log_action(:create, warning) if warning.text.present? + def process_strike! + @warning = target_account.strikes.create!( + account: current_account, + report: report, + action: type, + text: text_for_warning, + status_ids: status_ids + ) end def process_reports! @@ -143,7 +137,7 @@ class Admin::AccountAction end def process_email! - UserMailer.warning(target_account.user, warning, status_ids).deliver_later! if warnable? + UserMailer.warning(target_account.user, warning).deliver_later! if warnable? end def warnable? @@ -151,7 +145,7 @@ class Admin::AccountAction end def status_ids - report.status_ids if report && include_statuses + report.status_ids if with_report? && include_statuses end def reports diff --git a/app/models/admin/action_log.rb b/app/models/admin/action_log.rb index 1d1db1b7a..852bff713 100644 --- a/app/models/admin/action_log.rb +++ b/app/models/admin/action_log.rb @@ -17,7 +17,7 @@ class Admin::ActionLog < ApplicationRecord serialize :recorded_changes belongs_to :account - belongs_to :target, polymorphic: true + belongs_to :target, polymorphic: true, optional: true default_scope -> { order('id desc') } diff --git a/app/models/admin/action_log_filter.rb b/app/models/admin/action_log_filter.rb index a1c156a8b..12136223b 100644 --- a/app/models/admin/action_log_filter.rb +++ b/app/models/admin/action_log_filter.rb @@ -11,6 +11,8 @@ class Admin::ActionLogFilter assigned_to_self_report: { target_type: 'Report', action: 'assigned_to_self' }.freeze, change_email_user: { target_type: 'User', action: 'change_email' }.freeze, confirm_user: { target_type: 'User', action: 'confirm' }.freeze, + approve_user: { target_type: 'User', action: 'approve' }.freeze, + reject_user: { target_type: 'User', action: 'reject' }.freeze, create_account_warning: { target_type: 'AccountWarning', action: 'create' }.freeze, create_announcement: { target_type: 'Announcement', action: 'create' }.freeze, create_custom_emoji: { target_type: 'CustomEmoji', action: 'create' }.freeze, @@ -24,6 +26,7 @@ class Admin::ActionLogFilter destroy_domain_allow: { target_type: 'DomainAllow', action: 'destroy' }.freeze, destroy_domain_block: { target_type: 'DomainBlock', action: 'destroy' }.freeze, destroy_email_domain_block: { target_type: 'EmailDomainBlock', action: 'destroy' }.freeze, + destroy_instance: { target_type: 'Instance', action: 'destroy' }.freeze, destroy_unavailable_domain: { target_type: 'UnavailableDomain', action: 'destroy' }.freeze, destroy_status: { target_type: 'Status', action: 'destroy' }.freeze, disable_2fa_user: { target_type: 'User', action: 'disable' }.freeze, @@ -47,6 +50,7 @@ class Admin::ActionLogFilter update_announcement: { target_type: 'Announcement', action: 'update' }.freeze, update_custom_emoji: { target_type: 'CustomEmoji', action: 'update' }.freeze, update_status: { target_type: 'Status', action: 'update' }.freeze, + unblock_email_account: { target_type: 'Account', action: 'unblock_email' }.freeze, }.freeze attr_reader :params @@ -76,7 +80,7 @@ class Admin::ActionLogFilter when 'account_id' Admin::ActionLog.where(account_id: value) when 'target_account_id' - account = Account.find(value) + account = Account.find_or_initialize_by(id: value) Admin::ActionLog.where(target: [account, account.user].compact) else raise "Unknown filter: #{key}" diff --git a/app/models/admin/status_batch_action.rb b/app/models/admin/status_batch_action.rb new file mode 100644 index 000000000..85822214b --- /dev/null +++ b/app/models/admin/status_batch_action.rb @@ -0,0 +1,92 @@ +# frozen_string_literal: true + +class Admin::StatusBatchAction + include ActiveModel::Model + include AccountableConcern + include Authorization + + attr_accessor :current_account, :type, + :status_ids, :report_id + + def save! + process_action! + end + + private + + def statuses + Status.with_discarded.where(id: status_ids) + end + + def process_action! + return if status_ids.empty? + + case type + when 'delete' + handle_delete! + when 'report' + handle_report! + when 'remove_from_report' + handle_remove_from_report! + end + end + + def handle_delete! + statuses.each { |status| authorize(status, :destroy?) } + + ApplicationRecord.transaction do + statuses.each do |status| + status.discard + log_action(:destroy, status) + end + + if with_report? + report.resolve!(current_account) + log_action(:resolve, report) + end + + @warning = target_account.strikes.create!( + action: :delete_statuses, + account: current_account, + report: report, + status_ids: status_ids + ) + + statuses.each { |status| Tombstone.find_or_create_by(uri: status.uri, account: status.account, by_moderator: true) } unless target_account.local? + end + + UserMailer.warning(target_account.user, @warning).deliver_later! if target_account.local? + RemovalWorker.push_bulk(status_ids) { |status_id| [status_id, { 'preserve' => target_account.local?, 'immediate' => !target_account.local? }] } + end + + def handle_report! + @report = Report.new(report_params) unless with_report? + @report.status_ids = (@report.status_ids + status_ids.map(&:to_i)).uniq + @report.save! + + @report_id = @report.id + end + + def handle_remove_from_report! + return unless with_report? + + report.status_ids -= status_ids.map(&:to_i) + report.save! + end + + def report + @report ||= Report.find(report_id) if report_id.present? + end + + def with_report? + !report.nil? + end + + def target_account + @target_account ||= statuses.first.account + end + + def report_params + { account: current_account, target_account: target_account } + end +end diff --git a/app/models/admin/status_filter.rb b/app/models/admin/status_filter.rb new file mode 100644 index 000000000..ce5bb5f46 --- /dev/null +++ b/app/models/admin/status_filter.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +class Admin::StatusFilter + KEYS = %i( + media + id + report_id + ).freeze + + attr_reader :params + + def initialize(account, params) + @account = account + @params = params + end + + def results + scope = @account.statuses.where(visibility: [:public, :unlisted]) + + params.each do |key, value| + next if %w(page report_id).include?(key.to_s) + + scope.merge!(scope_for(key, value.to_s.strip)) if value.present? + end + + scope + end + + private + + def scope_for(key, value) + case key.to_s + when 'media' + Status.joins(:media_attachments).merge(@account.media_attachments.reorder(nil)).group(:id) + when 'id' + Status.where(id: value) + else + raise "Unknown filter: #{key}" + end + end +end diff --git a/app/models/bookmark.rb b/app/models/bookmark.rb index 916261a17..6334ef0df 100644 --- a/app/models/bookmark.rb +++ b/app/models/bookmark.rb @@ -13,7 +13,7 @@ class Bookmark < ApplicationRecord include Paginable - update_index('statuses#status', :status) if Chewy.enabled? + update_index('statuses', :status) if Chewy.enabled? belongs_to :account, inverse_of: :bookmarks belongs_to :status, inverse_of: :bookmarks @@ -23,4 +23,12 @@ class Bookmark < ApplicationRecord before_validation do self.status = status.reblog if status&.reblog? end + + after_destroy :invalidate_cleanup_info + + def invalidate_cleanup_info + return unless status&.account_id == account_id && account.local? + + account.statuses_cleanup_policy&.invalidate_last_inspected(status, :unbookmark) + end end diff --git a/app/models/canonical_email_block.rb b/app/models/canonical_email_block.rb index a8546d65a..94781386c 100644 --- a/app/models/canonical_email_block.rb +++ b/app/models/canonical_email_block.rb @@ -15,7 +15,7 @@ class CanonicalEmailBlock < ApplicationRecord belongs_to :reference_account, class_name: 'Account' - validates :canonical_email_hash, presence: true + validates :canonical_email_hash, presence: true, uniqueness: true def email=(email) self.canonical_email_hash = email_to_canonical_email_hash(email) @@ -24,4 +24,8 @@ class CanonicalEmailBlock < ApplicationRecord def self.block?(email) where(canonical_email_hash: email_to_canonical_email_hash(email)).exists? end + + def self.find_blocks(email) + where(canonical_email_hash: email_to_canonical_email_hash(email)) + end end diff --git a/app/models/concerns/account_associations.rb b/app/models/concerns/account_associations.rb index aaf371ebd..bbe269e8f 100644 --- a/app/models/concerns/account_associations.rb +++ b/app/models/concerns/account_associations.rb @@ -7,8 +7,7 @@ module AccountAssociations # Local users has_one :user, inverse_of: :account, dependent: :destroy - # Identity proofs - has_many :identity_proofs, class_name: 'AccountIdentityProof', dependent: :destroy, inverse_of: :account + # E2EE has_many :devices, dependent: :destroy, inverse_of: :account # Timelines @@ -43,7 +42,7 @@ module AccountAssociations has_many :account_moderation_notes, dependent: :destroy, inverse_of: :account has_many :targeted_moderation_notes, class_name: 'AccountModerationNote', foreign_key: :target_account_id, dependent: :destroy, inverse_of: :target_account has_many :account_warnings, dependent: :destroy, inverse_of: :account - has_many :targeted_account_warnings, class_name: 'AccountWarning', foreign_key: :target_account_id, dependent: :destroy, inverse_of: :target_account + has_many :strikes, class_name: 'AccountWarning', foreign_key: :target_account_id, dependent: :destroy, inverse_of: :target_account # Lists (that the account is on, not owned by the account) has_many :list_accounts, inverse_of: :account, dependent: :destroy @@ -66,5 +65,8 @@ module AccountAssociations # Follow recommendations has_one :follow_recommendation_suppression, inverse_of: :account, dependent: :destroy + + # Account statuses cleanup policy + has_one :statuses_cleanup_policy, class_name: 'AccountStatusesCleanupPolicy', inverse_of: :account, dependent: :destroy end end diff --git a/app/models/concerns/account_interactions.rb b/app/models/concerns/account_interactions.rb index 958f6c78e..ad1665dc4 100644 --- a/app/models/concerns/account_interactions.rb +++ b/app/models/concerns/account_interactions.rb @@ -81,6 +81,9 @@ module AccountInteractions has_many :following, -> { order('follows.id desc') }, through: :active_relationships, source: :target_account has_many :followers, -> { order('follows.id desc') }, through: :passive_relationships, source: :account + # Account notes + has_many :account_notes, dependent: :destroy + # Block relationships has_many :block_relationships, class_name: 'Block', foreign_key: 'account_id', dependent: :destroy has_many :blocking, -> { order('blocks.id desc') }, through: :block_relationships, source: :target_account @@ -192,6 +195,10 @@ module AccountInteractions !following_anyone? end + def followed_by?(other_account) + other_account.following?(self) + end + def blocking?(other_account) block_relationships.where(target_account: other_account).exists? end @@ -251,10 +258,13 @@ module AccountInteractions .where('users.current_sign_in_at > ?', User::ACTIVE_DURATION.ago) end - def remote_followers_hash(url_prefix) - Rails.cache.fetch("followers_hash:#{id}:#{url_prefix}") do + def remote_followers_hash(url) + url_prefix = url[Account::URL_PREFIX_RE] + return if url_prefix.blank? + + Rails.cache.fetch("followers_hash:#{id}:#{url_prefix}/") do digest = "\x00" * 32 - followers.where(Account.arel_table[:uri].matches(url_prefix + '%', false, true)).pluck_each(:uri) do |uri| + followers.where(Account.arel_table[:uri].matches("#{Account.sanitize_sql_like(url_prefix)}/%", false, true)).or(followers.where(uri: url_prefix)).pluck_each(:uri) do |uri| Xorcist.xor!(digest, Digest::SHA256.digest(uri)) end digest.unpack('H*')[0] diff --git a/app/models/concerns/account_merging.rb b/app/models/concerns/account_merging.rb index 8d37c6e56..119773e6b 100644 --- a/app/models/concerns/account_merging.rb +++ b/app/models/concerns/account_merging.rb @@ -13,7 +13,7 @@ module AccountMerging owned_classes = [ Status, StatusPin, MediaAttachment, Poll, Report, Tombstone, Favourite, - Follow, FollowRequest, Block, Mute, AccountIdentityProof, + Follow, FollowRequest, Block, Mute, AccountModerationNote, AccountPin, AccountStat, ListAccount, PollVote, Mention, AccountDeletionRequest, AccountNote, FollowRecommendationSuppression ] diff --git a/app/models/concerns/attachmentable.rb b/app/models/concerns/attachmentable.rb index c5febb828..01fae4236 100644 --- a/app/models/concerns/attachmentable.rb +++ b/app/models/concerns/attachmentable.rb @@ -15,50 +15,47 @@ module Attachmentable # those files, it is necessary to use the output of the # `file` utility instead INCORRECT_CONTENT_TYPES = %w( + audio/vorbis video/ogg video/webm ).freeze included do - before_post_process :obfuscate_file_name - before_post_process :set_file_extensions - before_post_process :check_image_dimensions - before_post_process :set_file_content_type + def self.has_attached_file(name, options = {}) # rubocop:disable Naming/PredicateName + options = { validate_media_type: false }.merge(options) + super(name, options) + send(:"before_#{name}_post_process") do + attachment = send(name) + check_image_dimension(attachment) + set_file_content_type(attachment) + obfuscate_file_name(attachment) + set_file_extension(attachment) + Paperclip::Validators::MediaTypeSpoofDetectionValidator.new(attributes: [name]).validate(self) + end + end end private - def set_file_content_type - self.class.attachment_definitions.each_key do |attachment_name| - attachment = send(attachment_name) - - next if attachment.blank? || attachment.queued_for_write[:original].blank? || !INCORRECT_CONTENT_TYPES.include?(attachment.instance_read(:content_type)) + def set_file_content_type(attachment) # rubocop:disable Naming/AccessorMethodName + return if attachment.blank? || attachment.queued_for_write[:original].blank? || !INCORRECT_CONTENT_TYPES.include?(attachment.instance_read(:content_type)) - attachment.instance_write :content_type, calculated_content_type(attachment) - end + attachment.instance_write :content_type, calculated_content_type(attachment) end - def set_file_extensions - self.class.attachment_definitions.each_key do |attachment_name| - attachment = send(attachment_name) + def set_file_extension(attachment) # rubocop:disable Naming/AccessorMethodName + return if attachment.blank? - next if attachment.blank? - - attachment.instance_write :file_name, [Paperclip::Interpolations.basename(attachment, :original), appropriate_extension(attachment)].delete_if(&:blank?).join('.') - end + attachment.instance_write :file_name, [Paperclip::Interpolations.basename(attachment, :original), appropriate_extension(attachment)].delete_if(&:blank?).join('.') end - def check_image_dimensions - self.class.attachment_definitions.each_key do |attachment_name| - attachment = send(attachment_name) + def check_image_dimension(attachment) + return if attachment.blank? || !/image.*/.match?(attachment.content_type) || attachment.queued_for_write[:original].blank? - next if attachment.blank? || !/image.*/.match?(attachment.content_type) || attachment.queued_for_write[:original].blank? + width, height = FastImage.size(attachment.queued_for_write[:original].path) + matrix_limit = attachment.content_type == 'image/gif' ? GIF_MATRIX_LIMIT : MAX_MATRIX_LIMIT - width, height = FastImage.size(attachment.queued_for_write[:original].path) - matrix_limit = attachment.content_type == 'image/gif' ? GIF_MATRIX_LIMIT : MAX_MATRIX_LIMIT - - raise Mastodon::DimensionsValidationError, "#{width}x#{height} images are not supported" if width.present? && height.present? && (width * height > matrix_limit) - end + raise Mastodon::DimensionsValidationError, "#{width}x#{height} images are not supported" if width.present? && height.present? && (width * height > matrix_limit) end def appropriate_extension(attachment) @@ -79,13 +76,9 @@ module Attachmentable '' end - def obfuscate_file_name - self.class.attachment_definitions.each_key do |attachment_name| - attachment = send(attachment_name) + def obfuscate_file_name(attachment) + return if attachment.blank? || attachment.queued_for_write[:original].blank? || attachment.options[:preserve_files] - next if attachment.blank? || attachment.queued_for_write[:original].blank? || attachment.options[:preserve_files] - - attachment.instance_write :file_name, SecureRandom.hex(8) + File.extname(attachment.instance_read(:file_name)) - end + attachment.instance_write :file_name, SecureRandom.hex(8) + File.extname(attachment.instance_read(:file_name)) end end diff --git a/app/models/custom_emoji.rb b/app/models/custom_emoji.rb index f14357932..d7349dc6a 100644 --- a/app/models/custom_emoji.rb +++ b/app/models/custom_emoji.rb @@ -21,6 +21,8 @@ # class CustomEmoji < ApplicationRecord + include Attachmentable + LOCAL_LIMIT = (ENV['MAX_EMOJI_SIZE'] || 50.kilobytes).to_i LIMIT = [LOCAL_LIMIT, (ENV['MAX_REMOTE_EMOJI_SIZE'] || 200.kilobytes).to_i].max @@ -35,7 +37,7 @@ class CustomEmoji < ApplicationRecord belongs_to :category, class_name: 'CustomEmojiCategory', optional: true has_one :local_counterpart, -> { where(domain: nil) }, class_name: 'CustomEmoji', primary_key: :shortcode, foreign_key: :shortcode - has_attached_file :image, styles: { static: { format: 'png', convert_options: '-coalesce -strip' } } + has_attached_file :image, styles: { static: { format: 'png', convert_options: '-coalesce -strip' } }, validate_media_type: false before_validation :downcase_domain @@ -52,8 +54,6 @@ class CustomEmoji < ApplicationRecord remotable_attachment :image, LIMIT - include Attachmentable - after_commit :remove_entity_cache def local? diff --git a/app/models/favourite.rb b/app/models/favourite.rb index 35028b7dd..2f355739a 100644 --- a/app/models/favourite.rb +++ b/app/models/favourite.rb @@ -13,7 +13,7 @@ class Favourite < ApplicationRecord include Paginable - update_index('statuses#status', :status) + update_index('statuses', :status) belongs_to :account, inverse_of: :favourites belongs_to :status, inverse_of: :favourites @@ -28,6 +28,7 @@ class Favourite < ApplicationRecord after_create :increment_cache_counters after_destroy :decrement_cache_counters + after_destroy :invalidate_cleanup_info private @@ -39,4 +40,10 @@ class Favourite < ApplicationRecord return if association(:status).loaded? && status.marked_for_destruction? status&.decrement_count!(:favourites_count) end + + def invalidate_cleanup_info + return unless status&.account_id == account_id && account.local? + + account.statuses_cleanup_policy&.invalidate_last_inspected(status, :unfav) + end end diff --git a/app/models/form/account_batch.rb b/app/models/form/account_batch.rb index 698933c9f..dcf155840 100644 --- a/app/models/form/account_batch.rb +++ b/app/models/form/account_batch.rb @@ -3,6 +3,7 @@ class Form::AccountBatch include ActiveModel::Model include Authorization + include AccountableConcern include Payloadable attr_accessor :account_ids, :action, :current_account @@ -25,27 +26,27 @@ class Form::AccountBatch suppress_follow_recommendation! when 'unsuppress_follow_recommendation' unsuppress_follow_recommendation! + when 'suspend' + suspend! end end private def follow! - accounts.find_each do |target_account| + accounts.each do |target_account| FollowService.new.call(current_account, target_account) end end def unfollow! - accounts.find_each do |target_account| + accounts.each do |target_account| UnfollowService.new.call(current_account, target_account) end end def remove_from_followers! - current_account.passive_relationships.where(account_id: account_ids).find_each do |follow| - reject_follow!(follow) - end + RemoveFromFollowersService.new.call(current_account, account_ids) end def block_domains! @@ -62,32 +63,32 @@ class Form::AccountBatch Account.where(id: account_ids) end - def reject_follow!(follow) - follow.destroy - - return unless follow.account.activitypub? - - ActivityPub::DeliveryWorker.perform_async(Oj.dump(serialize_payload(follow, ActivityPub::RejectFollowSerializer)), current_account.id, follow.account.inbox_url) - end - def approve! - users = accounts.includes(:user).map(&:user) - - users.each { |user| authorize(user, :approve?) } - .each(&:approve!) + accounts.includes(:user).find_each do |account| + approve_account(account) + end end def reject! - records = accounts.includes(:user) + accounts.includes(:user).find_each do |account| + reject_account(account) + end + end - records.each { |account| authorize(account.user, :reject?) } - .each { |account| DeleteAccountService.new.call(account, reserve_email: false, reserve_username: false) } + def suspend! + accounts.find_each do |account| + if account.user_pending? + reject_account(account) + else + suspend_account(account) + end + end end def suppress_follow_recommendation! authorize(:follow_recommendation, :suppress?) - accounts.each do |account| + accounts.find_each do |account| FollowRecommendationSuppression.create(account: account) end end @@ -97,4 +98,24 @@ class Form::AccountBatch FollowRecommendationSuppression.where(account_id: account_ids).destroy_all end + + def reject_account(account) + authorize(account.user, :reject?) + log_action(:reject, account.user, username: account.username) + account.suspend!(origin: :local) + AccountDeletionWorker.perform_async(account.id, { 'reserve_username' => false }) + end + + def suspend_account(account) + authorize(account, :suspend?) + log_action(:suspend, account) + account.suspend!(origin: :local) + Admin::SuspensionWorker.perform_async(account.id) + end + + def approve_account(account) + authorize(account.user, :approve?) + log_action(:approve, account.user) + account.user.approve! + end end diff --git a/app/models/form/admin_settings.rb b/app/models/form/admin_settings.rb index 0276ec058..34f14e312 100644 --- a/app/models/form/admin_settings.rb +++ b/app/models/form/admin_settings.rb @@ -27,7 +27,6 @@ class Form::AdminSettings custom_css profile_directory hide_followers_count - enable_keybase flavour_and_skin thumbnail hero @@ -41,6 +40,7 @@ class Form::AdminSettings noindex outgoing_spoilers require_invite_text + captcha_enabled ).freeze BOOLEAN_KEYS = %i( @@ -53,13 +53,13 @@ class Form::AdminSettings preview_sensitive_media profile_directory hide_followers_count - enable_keybase show_reblogs_in_public_timelines show_replies_in_public_timelines trends trendable_by_default noindex require_invite_text + captcha_enabled ).freeze UPLOAD_KEYS = %i( diff --git a/app/models/form/preview_card_batch.rb b/app/models/form/preview_card_batch.rb new file mode 100644 index 000000000..5f6e6522a --- /dev/null +++ b/app/models/form/preview_card_batch.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +class Form::PreviewCardBatch + include ActiveModel::Model + include Authorization + + attr_accessor :preview_card_ids, :action, :current_account, :precision + + def save + case action + when 'approve' + approve! + when 'approve_all' + approve_all! + when 'reject' + reject! + when 'reject_all' + reject_all! + end + end + + private + + def preview_cards + @preview_cards ||= PreviewCard.where(id: preview_card_ids) + end + + def preview_card_providers + @preview_card_providers ||= preview_cards.map(&:domain).uniq.map { |domain| PreviewCardProvider.matching_domain(domain) || PreviewCardProvider.new(domain: domain) } + end + + def approve! + preview_cards.each { |preview_card| authorize(preview_card, :update?) } + preview_cards.update_all(trendable: true) + end + + def approve_all! + preview_card_providers.each do |provider| + authorize(provider, :update?) + provider.update(trendable: true, reviewed_at: action_time) + end + + # Reset any individual overrides + preview_cards.update_all(trendable: nil) + end + + def reject! + preview_cards.each { |preview_card| authorize(preview_card, :update?) } + preview_cards.update_all(trendable: false) + end + + def reject_all! + preview_card_providers.each do |provider| + authorize(provider, :update?) + provider.update(trendable: false, reviewed_at: action_time) + end + + # Reset any individual overrides + preview_cards.update_all(trendable: nil) + end + + def action_time + @action_time ||= Time.now.utc + end +end diff --git a/app/models/form/preview_card_provider_batch.rb b/app/models/form/preview_card_provider_batch.rb new file mode 100644 index 000000000..e6ab3d8fa --- /dev/null +++ b/app/models/form/preview_card_provider_batch.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +class Form::PreviewCardProviderBatch + include ActiveModel::Model + include Authorization + + attr_accessor :preview_card_provider_ids, :action, :current_account + + def save + case action + when 'approve' + approve! + when 'reject' + reject! + end + end + + private + + def preview_card_providers + PreviewCardProvider.where(id: preview_card_provider_ids) + end + + def approve! + preview_card_providers.each { |provider| authorize(provider, :update?) } + preview_card_providers.update_all(trendable: true, reviewed_at: Time.now.utc) + end + + def reject! + preview_card_providers.each { |provider| authorize(provider, :update?) } + preview_card_providers.update_all(trendable: false, reviewed_at: Time.now.utc) + end +end diff --git a/app/models/form/status_batch.rb b/app/models/form/status_batch.rb deleted file mode 100644 index c4943a7ea..000000000 --- a/app/models/form/status_batch.rb +++ /dev/null @@ -1,45 +0,0 @@ -# frozen_string_literal: true - -class Form::StatusBatch - include ActiveModel::Model - include AccountableConcern - - attr_accessor :status_ids, :action, :current_account - - def save - case action - when 'nsfw_on', 'nsfw_off' - change_sensitive(action == 'nsfw_on') - when 'delete' - delete_statuses - end - end - - private - - def change_sensitive(sensitive) - media_attached_status_ids = MediaAttachment.where(status_id: status_ids).pluck(:status_id) - - ApplicationRecord.transaction do - Status.where(id: media_attached_status_ids).reorder(nil).find_each do |status| - status.update!(sensitive: sensitive) - log_action :update, status - end - end - - true - rescue ActiveRecord::RecordInvalid - false - end - - def delete_statuses - Status.where(id: status_ids).reorder(nil).find_each do |status| - status.discard - RemovalWorker.perform_async(status.id, immediate: true) - Tombstone.find_or_create_by(uri: status.uri, account: status.account, by_moderator: true) - log_action :destroy, status - end - - true - end -end diff --git a/app/models/form/tag_batch.rb b/app/models/form/tag_batch.rb index fd517a1a6..b9330745f 100644 --- a/app/models/form/tag_batch.rb +++ b/app/models/form/tag_batch.rb @@ -23,11 +23,15 @@ class Form::TagBatch def approve! tags.each { |tag| authorize(tag, :update?) } - tags.update_all(trendable: true, reviewed_at: Time.now.utc) + tags.update_all(trendable: true, reviewed_at: action_time) end def reject! tags.each { |tag| authorize(tag, :update?) } - tags.update_all(trendable: false, reviewed_at: Time.now.utc) + tags.update_all(trendable: false, reviewed_at: action_time) + end + + def action_time + @action_time ||= Time.now.utc end end diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb index a6ab22f61..14e6cabae 100644 --- a/app/models/media_attachment.rb +++ b/app/models/media_attachment.rb @@ -31,6 +31,8 @@ class MediaAttachment < ApplicationRecord self.inheritance_column = nil + include Attachmentable + enum type: [:image, :gifv, :video, :unknown, :audio] enum processing: [:queued, :in_progress, :complete, :failed], _prefix: true @@ -50,7 +52,7 @@ class MediaAttachment < ApplicationRecord IMAGE_MIME_TYPES = %w(image/jpeg image/png image/gif).freeze VIDEO_MIME_TYPES = %w(video/webm video/mp4 video/quicktime video/ogg).freeze VIDEO_CONVERTIBLE_MIME_TYPES = %w(video/webm video/quicktime).freeze - AUDIO_MIME_TYPES = %w(audio/wave audio/wav audio/x-wav audio/x-pn-wave audio/ogg audio/mpeg audio/mp3 audio/webm audio/flac audio/aac audio/m4a audio/x-m4a audio/mp4 audio/3gpp video/x-ms-asf).freeze + AUDIO_MIME_TYPES = %w(audio/wave audio/wav audio/x-wav audio/x-pn-wave audio/ogg audio/vorbis audio/mpeg audio/mp3 audio/webm audio/flac audio/aac audio/m4a audio/x-m4a audio/mp4 audio/3gpp video/x-ms-asf).freeze BLURHASH_OPTIONS = { x_comp: 4, @@ -165,12 +167,11 @@ class MediaAttachment < ApplicationRecord processors: ->(f) { file_processors f }, convert_options: GLOBAL_CONVERT_OPTIONS - before_file_post_process :set_type_and_extension - before_file_post_process :check_video_dimensions + before_file_validate :set_type_and_extension + before_file_validate :check_video_dimensions validates_attachment_content_type :file, content_type: IMAGE_MIME_TYPES + VIDEO_MIME_TYPES + AUDIO_MIME_TYPES - validates_attachment_size :file, less_than: IMAGE_LIMIT, unless: :larger_media_format? - validates_attachment_size :file, less_than: VIDEO_LIMIT, if: :larger_media_format? + validates_attachment_size :file, less_than: ->(m) { m.larger_media_format? ? VIDEO_LIMIT : IMAGE_LIMIT } remotable_attachment :file, VIDEO_LIMIT, suppress_errors: false, download_on_assign: false, attribute_name: :remote_url has_attached_file :thumbnail, @@ -182,8 +183,6 @@ class MediaAttachment < ApplicationRecord validates_attachment_size :thumbnail, less_than: IMAGE_LIMIT remotable_attachment :thumbnail, IMAGE_LIMIT, suppress_errors: true, download_on_assign: false - include Attachmentable - validates :account, presence: true validates :description, length: { maximum: MAX_DESCRIPTION_LENGTH }, if: :local? validates :file, presence: true, if: :local? @@ -218,7 +217,7 @@ class MediaAttachment < ApplicationRecord end def to_param - shortcode + shortcode.presence || id&.to_s end def focus=(point) @@ -255,7 +254,7 @@ class MediaAttachment < ApplicationRecord after_commit :reset_parent_cache, on: :update before_create :prepare_description, unless: :local? - before_create :set_shortcode + before_create :set_unknown_type before_create :set_processing after_post_process :set_meta @@ -298,15 +297,8 @@ class MediaAttachment < ApplicationRecord private - def set_shortcode + def set_unknown_type self.type = :unknown if file.blank? && !type_changed? - - return unless local? - - loop do - self.shortcode = SecureRandom.urlsafe_base64(14) - break if MediaAttachment.find_by(shortcode: shortcode).nil? - end end def prepare_description diff --git a/app/models/poll.rb b/app/models/poll.rb index d2a17277b..71b5e191f 100644 --- a/app/models/poll.rb +++ b/app/models/poll.rb @@ -26,6 +26,7 @@ class Poll < ApplicationRecord belongs_to :status has_many :votes, class_name: 'PollVote', inverse_of: :poll, dependent: :delete_all + has_many :voters, -> { group('accounts.id') }, through: :votes, class_name: 'Account', source: :account has_many :notifications, as: :activity, dependent: :destroy diff --git a/app/models/preview_card.rb b/app/models/preview_card.rb index a6ec839f8..0f9e23fa1 100644 --- a/app/models/preview_card.rb +++ b/app/models/preview_card.rb @@ -24,9 +24,16 @@ # embed_url :string default(""), not null # image_storage_schema_version :integer # blurhash :string +# language :string +# max_score :float +# max_score_at :datetime +# trendable :boolean +# link_type :integer # class PreviewCard < ApplicationRecord + include Attachmentable + IMAGE_MIME_TYPES = ['image/jpeg', 'image/png', 'image/gif'].freeze LIMIT = 1.megabytes @@ -38,12 +45,11 @@ class PreviewCard < ApplicationRecord self.inheritance_column = false enum type: [:link, :photo, :video, :rich] + enum link_type: [:unknown, :article] has_and_belongs_to_many :statuses - has_attached_file :image, processors: [:thumbnail, :blurhash_transcoder], styles: ->(f) { image_styles(f) }, convert_options: { all: '-quality 80 -strip' } - - include Attachmentable + has_attached_file :image, processors: [:thumbnail, :blurhash_transcoder], styles: ->(f) { image_styles(f) }, convert_options: { all: '-quality 80 -strip' }, validate_media_type: false validates :url, presence: true, uniqueness: true validates_attachment_content_type :image, content_type: IMAGE_MIME_TYPES @@ -54,6 +60,36 @@ class PreviewCard < ApplicationRecord before_save :extract_dimensions, if: :link? + def appropriate_for_trends? + link? && article? && title.present? && description.present? && image.present? && provider_name.present? + end + + def domain + @domain ||= Addressable::URI.parse(url).normalized_host + end + + def provider + @provider ||= PreviewCardProvider.matching_domain(domain) + end + + def trendable? + if attributes['trendable'].nil? + provider&.trendable? + else + attributes['trendable'] + end + end + + def requires_review_notification? + attributes['trendable'].nil? && (provider.nil? || provider.requires_review_notification?) + end + + def decaying? + max_score_at && max_score_at >= Trends.links.options[:max_score_cooldown].ago && max_score_at < 1.day.ago + end + + attr_writer :provider + def local? false end @@ -69,11 +105,14 @@ class PreviewCard < ApplicationRecord save! end + def history + @history ||= Trends::History.new('links', id) + end + class << self private - # rubocop:disable Naming/MethodParameterName - def image_styles(f) + def image_styles(file) styles = { original: { geometry: '400x400>', @@ -83,10 +122,9 @@ class PreviewCard < ApplicationRecord }, } - styles[:original][:format] = 'jpg' if f.instance.image_content_type == 'image/gif' + styles[:original][:format] = 'jpg' if file.instance.image_content_type == 'image/gif' styles end - # rubocop:enable Naming/MethodParameterName end private diff --git a/app/models/preview_card_filter.rb b/app/models/preview_card_filter.rb new file mode 100644 index 000000000..8dda9989c --- /dev/null +++ b/app/models/preview_card_filter.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +class PreviewCardFilter + KEYS = %i( + trending + ).freeze + + attr_reader :params + + def initialize(params) + @params = params + end + + def results + scope = PreviewCard.unscoped + + params.each do |key, value| + next if key.to_s == 'page' + + scope.merge!(scope_for(key, value.to_s.strip)) if value.present? + end + + scope + end + + private + + def scope_for(key, value) + case key.to_s + when 'trending' + trending_scope(value) + else + raise "Unknown filter: #{key}" + end + end + + def trending_scope(value) + ids = begin + case value.to_s + when 'allowed' + Trends.links.currently_trending_ids(true, -1) + else + Trends.links.currently_trending_ids(false, -1) + end + end + + if ids.empty? + PreviewCard.none + else + PreviewCard.joins("join unnest(array[#{ids.map(&:to_i).join(',')}]::integer[]) with ordinality as x (id, ordering) on preview_cards.id = x.id").order('x.ordering') + end + end +end diff --git a/app/models/preview_card_provider.rb b/app/models/preview_card_provider.rb new file mode 100644 index 000000000..15b24e2bd --- /dev/null +++ b/app/models/preview_card_provider.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true +# == Schema Information +# +# Table name: preview_card_providers +# +# id :bigint(8) not null, primary key +# domain :string default(""), not null +# icon_file_name :string +# icon_content_type :string +# icon_file_size :bigint(8) +# icon_updated_at :datetime +# trendable :boolean +# reviewed_at :datetime +# requested_review_at :datetime +# created_at :datetime not null +# updated_at :datetime not null +# + +class PreviewCardProvider < ApplicationRecord + include DomainNormalizable + include Attachmentable + + ICON_MIME_TYPES = %w(image/x-icon image/vnd.microsoft.icon image/png).freeze + LIMIT = 1.megabyte + + validates :domain, presence: true, uniqueness: true, domain: true + + has_attached_file :icon, styles: { static: { format: 'png', convert_options: '-coalesce -strip' } }, validate_media_type: false + validates_attachment :icon, content_type: { content_type: ICON_MIME_TYPES }, size: { less_than: LIMIT } + remotable_attachment :icon, LIMIT + + scope :trendable, -> { where(trendable: true) } + scope :not_trendable, -> { where(trendable: false) } + scope :reviewed, -> { where.not(reviewed_at: nil) } + scope :pending_review, -> { where(reviewed_at: nil) } + + def requires_review? + reviewed_at.nil? + end + + def reviewed? + reviewed_at.present? + end + + def requested_review? + requested_review_at.present? + end + + def requires_review_notification? + requires_review? && !requested_review? + end + + def self.matching_domain(domain) + segments = domain.split('.') + where(domain: segments.map.with_index { |_, i| segments[i..-1].join('.') }).order(Arel.sql('char_length(domain) desc')).first + end +end diff --git a/app/models/preview_card_provider_filter.rb b/app/models/preview_card_provider_filter.rb new file mode 100644 index 000000000..1e90d3c9d --- /dev/null +++ b/app/models/preview_card_provider_filter.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +class PreviewCardProviderFilter + KEYS = %i( + status + ).freeze + + attr_reader :params + + def initialize(params) + @params = params + end + + def results + scope = PreviewCardProvider.unscoped + + params.each do |key, value| + next if key.to_s == 'page' + + scope.merge!(scope_for(key, value.to_s.strip)) if value.present? + end + + scope.order(domain: :asc) + end + + private + + def scope_for(key, value) + case key.to_s + when 'status' + status_scope(value) + else + raise "Unknown filter: #{key}" + end + end + + def status_scope(value) + case value.to_s + when 'approved' + PreviewCardProvider.trendable + when 'rejected' + PreviewCardProvider.not_trendable + when 'pending_review' + PreviewCardProvider.pending_review + else + raise "Unknown status: #{value}" + end + end +end diff --git a/app/models/report.rb b/app/models/report.rb index ef41547d9..ceb15133b 100644 --- a/app/models/report.rb +++ b/app/models/report.rb @@ -6,7 +6,6 @@ # id :bigint(8) not null, primary key # status_ids :bigint(8) default([]), not null, is an Array # comment :text default(""), not null -# action_taken :boolean default(FALSE), not null # created_at :datetime not null # updated_at :datetime not null # account_id :bigint(8) not null @@ -15,9 +14,14 @@ # assigned_account_id :bigint(8) # uri :string # forwarded :boolean +# category :integer default("other"), not null +# action_taken_at :datetime +# rule_ids :bigint(8) is an Array # class Report < ApplicationRecord + self.ignored_columns = %w(action_taken) + include Paginable include RateLimitable @@ -30,11 +34,17 @@ class Report < ApplicationRecord has_many :notes, class_name: 'ReportNote', foreign_key: :report_id, inverse_of: :report, dependent: :destroy - scope :unresolved, -> { where(action_taken: false) } - scope :resolved, -> { where(action_taken: true) } + scope :unresolved, -> { where(action_taken_at: nil) } + scope :resolved, -> { where.not(action_taken_at: nil) } scope :with_accounts, -> { includes([:account, :target_account, :action_taken_by_account, :assigned_account].index_with({ user: [:invite_request, :invite] })) } - validates :comment, length: { maximum: 1000 } + validates :comment, length: { maximum: 1_000 } + + enum category: { + other: 0, + spam: 1_000, + violation: 2_000, + } def local? false # Force uri_for to use uri attribute @@ -47,13 +57,17 @@ class Report < ApplicationRecord end def statuses - Status.with_discarded.where(id: status_ids).includes(:account, :media_attachments, :mentions) + Status.with_discarded.where(id: status_ids) end def media_attachments MediaAttachment.where(status_id: status_ids) end + def rules + Rule.with_discarded.where(id: rule_ids) + end + def assign_to_self!(current_account) update!(assigned_account_id: current_account.id) end @@ -63,22 +77,19 @@ class Report < ApplicationRecord end def resolve!(acting_account) - if account_id == -99 && target_account.trust_level == Account::TRUST_LEVELS[:untrusted] - # This is an automated report and it is being dismissed, so it's - # a false positive, in which case update the account's trust level - # to prevent further spam checks - - target_account.update(trust_level: Account::TRUST_LEVELS[:trusted]) - end - - RemovalWorker.push_bulk(Status.with_discarded.discarded.where(id: status_ids).pluck(:id)) { |status_id| [status_id, { immediate: true }] } - update!(action_taken: true, action_taken_by_account_id: acting_account.id) + update!(action_taken_at: Time.now.utc, action_taken_by_account_id: acting_account.id) end def unresolve! - update!(action_taken: false, action_taken_by_account_id: nil) + update!(action_taken_at: nil, action_taken_by_account_id: nil) + end + + def action_taken? + action_taken_at.present? end + alias action_taken action_taken? + def unresolved? !action_taken? end @@ -88,29 +99,24 @@ class Report < ApplicationRecord end def history - time_range = created_at..updated_at - - sql = [ + subquery = [ Admin::ActionLog.where( target_type: 'Report', - target_id: id, - created_at: time_range - ).unscope(:order), + target_id: id + ).unscope(:order).arel, Admin::ActionLog.where( target_type: 'Account', - target_id: target_account_id, - created_at: time_range - ).unscope(:order), + target_id: target_account_id + ).unscope(:order).arel, Admin::ActionLog.where( target_type: 'Status', - target_id: status_ids, - created_at: time_range - ).unscope(:order), - ].map { |query| "(#{query.to_sql})" }.join(' UNION ALL ') + target_id: status_ids + ).unscope(:order).arel, + ].reduce { |union, query| Arel::Nodes::UnionAll.new(union, query) } - Admin::ActionLog.from("(#{sql}) AS admin_action_logs") + Admin::ActionLog.from(Arel::Nodes::As.new(subquery, Admin::ActionLog.arel_table)) end def set_uri diff --git a/app/models/report_filter.rb b/app/models/report_filter.rb index c32d4359e..dc444a552 100644 --- a/app/models/report_filter.rb +++ b/app/models/report_filter.rb @@ -6,6 +6,7 @@ class ReportFilter account_id target_account_id by_target_domain + target_origin ).freeze attr_reader :params @@ -18,7 +19,7 @@ class ReportFilter scope = Report.unresolved params.each do |key, value| - scope = scope.merge scope_for(key, value) + scope = scope.merge scope_for(key, value), rewhere: true end scope @@ -34,8 +35,21 @@ class ReportFilter Report.where(account_id: value) when :target_account_id Report.where(target_account_id: value) + when :target_origin + target_origin_scope(value) else raise "Unknown filter: #{key}" end end + + def target_origin_scope(value) + case value.to_sym + when :local + Report.where(target_account: Account.local) + when :remote + Report.where(target_account: Account.remote) + else + raise "Unknown value: #{value}" + end + end end diff --git a/app/models/status.rb b/app/models/status.rb index 9f673ee53..9bb2b3746 100644 --- a/app/models/status.rb +++ b/app/models/status.rb @@ -26,6 +26,7 @@ # poll_id :bigint(8) # content_type :string # deleted_at :datetime +# edited_at :datetime # class Status < ApplicationRecord @@ -45,7 +46,7 @@ class Status < ApplicationRecord # will be based on current time instead of `created_at` attr_accessor :override_timestamps - update_index('statuses#status', :proper) + update_index('statuses', :proper) enum visibility: [:public, :unlisted, :private, :direct, :limited], _suffix: :visibility @@ -59,6 +60,8 @@ class Status < ApplicationRecord belongs_to :thread, foreign_key: 'in_reply_to_id', class_name: 'Status', inverse_of: :replies, optional: true belongs_to :reblog, foreign_key: 'reblog_of_id', class_name: 'Status', inverse_of: :reblogs, optional: true + has_many :edits, class_name: 'StatusEdit', inverse_of: :status, dependent: :destroy + has_many :favourites, inverse_of: :status, dependent: :destroy has_many :bookmarks, inverse_of: :status, dependent: :destroy has_many :reblogs, foreign_key: 'reblog_of_id', class_name: 'Status', inverse_of: :reblog, dependent: :destroy @@ -100,15 +103,12 @@ class Status < ApplicationRecord scope :not_excluded_by_account, ->(account) { where.not(account_id: account.excluded_from_timeline_account_ids) } scope :not_domain_blocked_by_account, ->(account) { account.excluded_from_timeline_domains.blank? ? left_outer_joins(:account) : left_outer_joins(:account).where('accounts.domain IS NULL OR accounts.domain NOT IN (?)', account.excluded_from_timeline_domains) } scope :tagged_with_all, ->(tag_ids) { - Array(tag_ids).reduce(self) do |result, id| + Array(tag_ids).map(&:to_i).reduce(self) do |result, id| result.joins("INNER JOIN statuses_tags t#{id} ON t#{id}.status_id = statuses.id AND t#{id}.tag_id = #{id}") end } scope :tagged_with_none, ->(tag_ids) { - Array(tag_ids).reduce(self) do |result, id| - result.joins("LEFT OUTER JOIN statuses_tags t#{id} ON t#{id}.status_id = statuses.id AND t#{id}.tag_id = #{id}") - .where("t#{id}.tag_id IS NULL") - end + where('NOT EXISTS (SELECT * FROM statuses_tags forbidden WHERE forbidden.status_id = statuses.id AND forbidden.tag_id IN (?))', tag_ids) } scope :not_local_only, -> { where(local_only: [false, nil]) } @@ -215,6 +215,10 @@ class Status < ApplicationRecord public_visibility? || unlisted_visibility? end + def edited? + edited_at.present? + end + alias sign? distributable? def with_media? @@ -391,7 +395,7 @@ class Status < ApplicationRecord def from_text(text) return [] if text.blank? - text.scan(FetchLinkCardService::URL_PATTERN).map(&:first).uniq.filter_map do |url| + text.scan(FetchLinkCardService::URL_PATTERN).map(&:second).uniq.filter_map do |url| status = begin if TagManager.instance.local_url?(url) ActivityPub::TagManager.instance.uri_to_resource(url, Status) @@ -494,7 +498,7 @@ class Status < ApplicationRecord end def decrement_counter_caches - return if direct_visibility? + return if direct_visibility? || new_record? account&.decrement_count!(:statuses_count) reblog&.decrement_count!(:reblogs_count) if reblog? diff --git a/app/models/status_edit.rb b/app/models/status_edit.rb new file mode 100644 index 000000000..a89df86c5 --- /dev/null +++ b/app/models/status_edit.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true +# == Schema Information +# +# Table name: status_edits +# +# id :bigint(8) not null, primary key +# status_id :bigint(8) not null +# account_id :bigint(8) +# text :text default(""), not null +# spoiler_text :text default(""), not null +# media_attachments_changed :boolean default(FALSE), not null +# created_at :datetime not null +# updated_at :datetime not null +# + +class StatusEdit < ApplicationRecord + belongs_to :status + belongs_to :account, optional: true + + default_scope { order(id: :asc) } + + delegate :local?, to: :status +end diff --git a/app/models/status_pin.rb b/app/models/status_pin.rb index afc76bded..93a0ea1c0 100644 --- a/app/models/status_pin.rb +++ b/app/models/status_pin.rb @@ -15,4 +15,12 @@ class StatusPin < ApplicationRecord belongs_to :status validates_with StatusPinValidator + + after_destroy :invalidate_cleanup_info + + def invalidate_cleanup_info + return unless status&.account_id == account_id && account.local? + + account.statuses_cleanup_policy&.invalidate_last_inspected(status, :unpin) + end end diff --git a/app/models/tag.rb b/app/models/tag.rb index 735c30608..a64042614 100644 --- a/app/models/tag.rb +++ b/app/models/tag.rb @@ -36,10 +36,11 @@ class Tag < ApplicationRecord scope :usable, -> { where(usable: [true, nil]) } scope :listable, -> { where(listable: [true, nil]) } scope :trendable, -> { Setting.trendable_by_default ? where(trendable: [true, nil]) : where(trendable: true) } + scope :not_trendable, -> { where(trendable: false) } scope :recently_used, ->(account) { joins(:statuses).where(statuses: { id: account.statuses.select(:id).limit(1000) }).group(:id).order(Arel.sql('count(*) desc')) } scope :matches_name, ->(term) { where(arel_table[:name].lower.matches(arel_table.lower("#{sanitize_sql_like(Tag.normalize(term))}%"), nil, true)) } # Search with case-sensitive to use B-tree index - update_index('tags#tag', :self) + update_index('tags', :self) def to_param name @@ -75,28 +76,16 @@ class Tag < ApplicationRecord requested_review_at.present? end - def use!(account, status: nil, at_time: Time.now.utc) - TrendingTags.record_use!(self, account, status: status, at_time: at_time) + def requires_review_notification? + requires_review? && !requested_review? end - def trending? - TrendingTags.trending?(self) + def decaying? + max_score_at && max_score_at >= Trends.tags.options[:max_score_cooldown].ago && max_score_at < 1.day.ago end def history - days = [] - - 7.times do |i| - day = i.days.ago.beginning_of_day.to_i - - days << { - day: day.to_s, - uses: Redis.current.get("activity:tags:#{id}:#{day}") || '0', - accounts: Redis.current.pfcount("activity:tags:#{id}:#{day}:accounts").to_s, - } - end - - days + @history ||= Trends::History.new('tags', id) end class << self diff --git a/app/models/tag_filter.rb b/app/models/tag_filter.rb index 85bfcbea5..ecdb52503 100644 --- a/app/models/tag_filter.rb +++ b/app/models/tag_filter.rb @@ -2,13 +2,8 @@ class TagFilter KEYS = %i( - directory - reviewed - unreviewed - pending_review - popular - active - name + trending + status ).freeze attr_reader :params @@ -18,7 +13,13 @@ class TagFilter end def results - scope = Tag.unscoped + scope = begin + if params[:status] == 'pending_review' + Tag.unscoped + else + trending_scope + end + end params.each do |key, value| next if key.to_s == 'page' @@ -26,27 +27,40 @@ class TagFilter scope.merge!(scope_for(key, value.to_s.strip)) if value.present? end - scope.order(id: :desc) + scope end private def scope_for(key, value) case key.to_s - when 'reviewed' - Tag.reviewed.order(reviewed_at: :desc) - when 'unreviewed' - Tag.unreviewed - when 'pending_review' - Tag.pending_review.order(requested_review_at: :desc) - when 'popular' - Tag.order('max_score DESC NULLS LAST') - when 'active' - Tag.order('last_status_at DESC NULLS LAST') - when 'name' - Tag.matches_name(value) + when 'status' + status_scope(value) else raise "Unknown filter: #{key}" end end + + def trending_scope + ids = Trends.tags.currently_trending_ids(false, -1) + + if ids.empty? + Tag.none + else + Tag.joins("join unnest(array[#{ids.map(&:to_i).join(',')}]::integer[]) with ordinality as x (id, ordering) on tags.id = x.id").order('x.ordering') + end + end + + def status_scope(value) + case value.to_s + when 'approved' + Tag.trendable + when 'rejected' + Tag.not_trendable + when 'pending_review' + Tag.pending_review + else + raise "Unknown status: #{value}" + end + end end diff --git a/app/models/trending_tags.rb b/app/models/trending_tags.rb deleted file mode 100644 index 31890b082..000000000 --- a/app/models/trending_tags.rb +++ /dev/null @@ -1,128 +0,0 @@ -# frozen_string_literal: true - -class TrendingTags - KEY = 'trending_tags' - EXPIRE_HISTORY_AFTER = 7.days.seconds - EXPIRE_TRENDS_AFTER = 1.day.seconds - THRESHOLD = 5 - LIMIT = 10 - REVIEW_THRESHOLD = 3 - MAX_SCORE_COOLDOWN = 2.days.freeze - MAX_SCORE_HALFLIFE = 2.hours.freeze - - class << self - include Redisable - - def record_use!(tag, account, status: nil, at_time: Time.now.utc) - return unless tag.usable? && !account.silenced? - - # Even if a tag is not allowed to trend, we still need to - # record the stats since they can be displayed in other places - increment_historical_use!(tag.id, at_time) - increment_unique_use!(tag.id, account.id, at_time) - increment_use!(tag.id, at_time) - - # Only update when the tag was last used once every 12 hours - # and only if a status is given (lets use ignore reblogs) - tag.update(last_status_at: at_time) if status.present? && (tag.last_status_at.nil? || (tag.last_status_at < at_time && tag.last_status_at < 12.hours.ago)) - end - - def update!(at_time = Time.now.utc) - tag_ids = redis.smembers("#{KEY}:used:#{at_time.beginning_of_day.to_i}") + redis.zrange(KEY, 0, -1) - tags = Tag.trendable.where(id: tag_ids.uniq) - - # First pass to calculate scores and update the set - - tags.each do |tag| - expected = redis.pfcount("activity:tags:#{tag.id}:#{(at_time - 1.day).beginning_of_day.to_i}:accounts").to_f - expected = 1.0 if expected.zero? - observed = redis.pfcount("activity:tags:#{tag.id}:#{at_time.beginning_of_day.to_i}:accounts").to_f - max_time = tag.max_score_at - max_score = tag.max_score - max_score = 0 if max_time.nil? || max_time < (at_time - MAX_SCORE_COOLDOWN) - - score = begin - if expected > observed || observed < THRESHOLD - 0 - else - ((observed - expected)**2) / expected - end - end - - if score > max_score - max_score = score - max_time = at_time - - # Not interested in triggering any callbacks for this - tag.update_columns(max_score: max_score, max_score_at: max_time) - end - - decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / MAX_SCORE_HALFLIFE.to_f)) - - if decaying_score.zero? - redis.zrem(KEY, tag.id) - else - redis.zadd(KEY, decaying_score, tag.id) - end - end - - users_for_review = User.staff.includes(:account).to_a.select(&:allows_trending_tag_emails?) - - # Second pass to notify about previously unreviewed trends - - tags.each do |tag| - current_rank = redis.zrevrank(KEY, tag.id) - needs_review_notification = tag.requires_review? && !tag.requested_review? - rank_passes_threshold = current_rank.present? && current_rank <= REVIEW_THRESHOLD - - next unless !tag.trendable? && rank_passes_threshold && needs_review_notification - - tag.touch(:requested_review_at) - - users_for_review.each do |user| - AdminMailer.new_trending_tag(user.account, tag).deliver_later! - end - end - - # Trim older items - - redis.zremrangebyrank(KEY, 0, -(LIMIT + 1)) - redis.zremrangebyscore(KEY, '(0.3', '-inf') - end - - def get(limit, filtered: true) - tag_ids = redis.zrevrange(KEY, 0, LIMIT - 1).map(&:to_i) - - tags = Tag.where(id: tag_ids) - tags = tags.trendable if filtered - tags = tags.index_by(&:id) - - tag_ids.map { |tag_id| tags[tag_id] }.compact.take(limit) - end - - def trending?(tag) - rank = redis.zrevrank(KEY, tag.id) - rank.present? && rank < LIMIT - end - - private - - def increment_historical_use!(tag_id, at_time) - key = "activity:tags:#{tag_id}:#{at_time.beginning_of_day.to_i}" - redis.incrby(key, 1) - redis.expire(key, EXPIRE_HISTORY_AFTER) - end - - def increment_unique_use!(tag_id, account_id, at_time) - key = "activity:tags:#{tag_id}:#{at_time.beginning_of_day.to_i}:accounts" - redis.pfadd(key, account_id) - redis.expire(key, EXPIRE_HISTORY_AFTER) - end - - def increment_use!(tag_id, at_time) - key = "#{KEY}:used:#{at_time.beginning_of_day.to_i}" - redis.sadd(key, tag_id) - redis.expire(key, EXPIRE_HISTORY_AFTER) - end - end -end diff --git a/app/models/trends.rb b/app/models/trends.rb new file mode 100644 index 000000000..8f8cb0261 --- /dev/null +++ b/app/models/trends.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Trends + def self.table_name_prefix + 'trends_' + end + + def self.links + @links ||= Trends::Links.new + end + + def self.tags + @tags ||= Trends::Tags.new + end + + def self.refresh! + [links, tags].each(&:refresh) + end + + def self.request_review! + [tags].each(&:request_review) if enabled? + end + + def self.enabled? + Setting.trends + end +end diff --git a/app/models/trends/base.rb b/app/models/trends/base.rb new file mode 100644 index 000000000..b767dcb1a --- /dev/null +++ b/app/models/trends/base.rb @@ -0,0 +1,80 @@ +# frozen_string_literal: true + +class Trends::Base + include Redisable + + class_attribute :default_options + + attr_reader :options + + # @param [Hash] options + # @option options [Integer] :threshold Minimum amount of uses by unique accounts to begin calculating the score + # @option options [Integer] :review_threshold Minimum rank (lower = better) before requesting a review + # @option options [ActiveSupport::Duration] :max_score_cooldown For this amount of time, the peak score (if bigger than current score) is decayed-from + # @option options [ActiveSupport::Duration] :max_score_halflife How quickly a peak score decays + def initialize(options = {}) + @options = self.class.default_options.merge(options) + end + + def register(_status) + raise NotImplementedError + end + + def add(*) + raise NotImplementedError + end + + def refresh(*) + raise NotImplementedError + end + + def request_review + raise NotImplementedError + end + + def get(*) + raise NotImplementedError + end + + def score(id) + redis.zscore("#{key_prefix}:all", id) || 0 + end + + def rank(id) + redis.zrevrank("#{key_prefix}:allowed", id) + end + + def currently_trending_ids(allowed, limit) + redis.zrevrange(allowed ? "#{key_prefix}:allowed" : "#{key_prefix}:all", 0, limit.positive? ? limit - 1 : limit).map(&:to_i) + end + + protected + + def key_prefix + raise NotImplementedError + end + + def recently_used_ids(at_time = Time.now.utc) + redis.smembers(used_key(at_time)).map(&:to_i) + end + + def record_used_id(id, at_time = Time.now.utc) + redis.sadd(used_key(at_time), id) + redis.expire(used_key(at_time), 1.day.seconds) + end + + def trim_older_items + redis.zremrangebyscore("#{key_prefix}:all", '-inf', '(1') + redis.zremrangebyscore("#{key_prefix}:allowed", '-inf', '(1') + end + + def score_at_rank(rank) + redis.zrevrange("#{key_prefix}:allowed", 0, rank, with_scores: true).last&.last || 0 + end + + private + + def used_key(at_time) + "#{key_prefix}:used:#{at_time.beginning_of_day.to_i}" + end +end diff --git a/app/models/trends/history.rb b/app/models/trends/history.rb new file mode 100644 index 000000000..608e33792 --- /dev/null +++ b/app/models/trends/history.rb @@ -0,0 +1,98 @@ +# frozen_string_literal: true + +class Trends::History + include Enumerable + + class Aggregate + include Redisable + + def initialize(prefix, id, date_range) + @days = date_range.map { |date| Day.new(prefix, id, date.to_time(:utc)) } + end + + def uses + redis.mget(*@days.map { |day| day.key_for(:uses) }).map(&:to_i).sum + end + + def accounts + redis.pfcount(*@days.map { |day| day.key_for(:accounts) }) + end + end + + class Day + include Redisable + + EXPIRE_AFTER = 14.days.seconds + + def initialize(prefix, id, day) + @prefix = prefix + @id = id + @day = day.beginning_of_day + end + + attr_reader :day + + def accounts + redis.pfcount(key_for(:accounts)) + end + + def uses + redis.get(key_for(:uses))&.to_i || 0 + end + + def add(account_id) + redis.pipelined do + redis.incrby(key_for(:uses), 1) + redis.pfadd(key_for(:accounts), account_id) + redis.expire(key_for(:uses), EXPIRE_AFTER) + redis.expire(key_for(:accounts), EXPIRE_AFTER) + end + end + + def as_json + { day: day.to_i.to_s, accounts: accounts.to_s, uses: uses.to_s } + end + + def key_for(suffix) + case suffix + when :accounts + "#{key_prefix}:#{suffix}" + when :uses + key_prefix + end + end + + def key_prefix + "activity:#{@prefix}:#{@id}:#{day.to_i}" + end + end + + def initialize(prefix, id) + @prefix = prefix + @id = id + end + + def get(date) + Day.new(@prefix, @id, date) + end + + def add(account_id, at_time = Time.now.utc) + Day.new(@prefix, @id, at_time).add(account_id) + end + + def aggregate(date_range) + Aggregate.new(@prefix, @id, date_range) + end + + def each(&block) + if block_given? + (0...7).map { |i| block.call(get(i.days.ago)) } + else + to_enum(:each) + end + end + + def as_json(*) + map(&:as_json) + end +end diff --git a/app/models/trends/links.rb b/app/models/trends/links.rb new file mode 100644 index 000000000..a0d65138b --- /dev/null +++ b/app/models/trends/links.rb @@ -0,0 +1,117 @@ +# frozen_string_literal: true + +class Trends::Links < Trends::Base + PREFIX = 'trending_links' + + self.default_options = { + threshold: 15, + review_threshold: 10, + max_score_cooldown: 2.days.freeze, + max_score_halflife: 8.hours.freeze, + } + + def register(status, at_time = Time.now.utc) + original_status = status.reblog? ? status.reblog : status + + return unless original_status.public_visibility? && status.public_visibility? && + !original_status.account.silenced? && !status.account.silenced? && + !original_status.spoiler_text? + + original_status.preview_cards.each do |preview_card| + add(preview_card, status.account_id, at_time) if preview_card.appropriate_for_trends? + end + end + + def add(preview_card, account_id, at_time = Time.now.utc) + preview_card.history.add(account_id, at_time) + record_used_id(preview_card.id, at_time) + end + + def get(allowed, limit) + preview_card_ids = currently_trending_ids(allowed, limit) + preview_cards = PreviewCard.where(id: preview_card_ids).index_by(&:id) + preview_card_ids.map { |id| preview_cards[id] }.compact + end + + def refresh(at_time = Time.now.utc) + preview_cards = PreviewCard.where(id: (recently_used_ids(at_time) + currently_trending_ids(false, -1)).uniq) + calculate_scores(preview_cards, at_time) + trim_older_items + end + + def request_review + preview_cards = PreviewCard.where(id: currently_trending_ids(false, -1)) + + preview_cards_requiring_review = preview_cards.filter_map do |preview_card| + next unless would_be_trending?(preview_card.id) && !preview_card.trendable? && preview_card.requires_review_notification? + + if preview_card.provider.nil? + preview_card.provider = PreviewCardProvider.create(domain: preview_card.domain, requested_review_at: Time.now.utc) + else + preview_card.provider.touch(:requested_review_at) + end + + preview_card + end + + return if preview_cards_requiring_review.empty? + + User.staff.includes(:account).find_each do |user| + AdminMailer.new_trending_links(user.account, preview_cards_requiring_review).deliver_later! if user.allows_trending_tag_emails? + end + end + + protected + + def key_prefix + PREFIX + end + + private + + def calculate_scores(preview_cards, at_time) + preview_cards.each do |preview_card| + expected = preview_card.history.get(at_time - 1.day).accounts.to_f + expected = 1.0 if expected.zero? + observed = preview_card.history.get(at_time).accounts.to_f + max_time = preview_card.max_score_at + max_score = preview_card.max_score + max_score = 0 if max_time.nil? || max_time < (at_time - options[:max_score_cooldown]) + + score = begin + if expected > observed || observed < options[:threshold] + 0 + else + ((observed - expected)**2) / expected + end + end + + if score > max_score + max_score = score + max_time = at_time + + # Not interested in triggering any callbacks for this + preview_card.update_columns(max_score: max_score, max_score_at: max_time) + end + + decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / options[:max_score_halflife].to_f)) + + if decaying_score.zero? + redis.zrem("#{PREFIX}:all", preview_card.id) + redis.zrem("#{PREFIX}:allowed", preview_card.id) + else + redis.zadd("#{PREFIX}:all", decaying_score, preview_card.id) + + if preview_card.trendable? + redis.zadd("#{PREFIX}:allowed", decaying_score, preview_card.id) + else + redis.zrem("#{PREFIX}:allowed", preview_card.id) + end + end + end + end + + def would_be_trending?(id) + score(id) > score_at_rank(options[:review_threshold] - 1) + end +end diff --git a/app/models/trends/tags.rb b/app/models/trends/tags.rb new file mode 100644 index 000000000..a425fd207 --- /dev/null +++ b/app/models/trends/tags.rb @@ -0,0 +1,111 @@ +# frozen_string_literal: true + +class Trends::Tags < Trends::Base + PREFIX = 'trending_tags' + + self.default_options = { + threshold: 5, + review_threshold: 10, + max_score_cooldown: 2.days.freeze, + max_score_halflife: 4.hours.freeze, + } + + def register(status, at_time = Time.now.utc) + original_status = status.reblog? ? status.reblog : status + + return unless original_status.public_visibility? && status.public_visibility? && + !original_status.account.silenced? && !status.account.silenced? + + original_status.tags.each do |tag| + add(tag, status.account_id, at_time) if tag.usable? + end + end + + def add(tag, account_id, at_time = Time.now.utc) + tag.history.add(account_id, at_time) + record_used_id(tag.id, at_time) + end + + def refresh(at_time = Time.now.utc) + tags = Tag.where(id: (recently_used_ids(at_time) + currently_trending_ids(false, -1)).uniq) + calculate_scores(tags, at_time) + trim_older_items + end + + def get(allowed, limit) + tag_ids = currently_trending_ids(allowed, limit) + tags = Tag.where(id: tag_ids).index_by(&:id) + tag_ids.map { |id| tags[id] }.compact + end + + def request_review + tags = Tag.where(id: currently_trending_ids(false, -1)) + + tags_requiring_review = tags.filter_map do |tag| + next unless would_be_trending?(tag.id) && !tag.trendable? && tag.requires_review_notification? + + tag.touch(:requested_review_at) + tag + end + + return if tags_requiring_review.empty? + + User.staff.includes(:account).find_each do |user| + AdminMailer.new_trending_tags(user.account, tags_requiring_review).deliver_later! if user.allows_trending_tag_emails? + end + end + + protected + + def key_prefix + PREFIX + end + + private + + def calculate_scores(tags, at_time) + tags.each do |tag| + expected = tag.history.get(at_time - 1.day).accounts.to_f + expected = 1.0 if expected.zero? + observed = tag.history.get(at_time).accounts.to_f + max_time = tag.max_score_at + max_score = tag.max_score + max_score = 0 if max_time.nil? || max_time < (at_time - options[:max_score_cooldown]) + + score = begin + if expected > observed || observed < options[:threshold] + 0 + else + ((observed - expected)**2) / expected + end + end + + if score > max_score + max_score = score + max_time = at_time + + # Not interested in triggering any callbacks for this + tag.update_columns(max_score: max_score, max_score_at: max_time) + end + + decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / options[:max_score_halflife].to_f)) + + if decaying_score.zero? + redis.zrem("#{PREFIX}:all", tag.id) + redis.zrem("#{PREFIX}:allowed", tag.id) + else + redis.zadd("#{PREFIX}:all", decaying_score, tag.id) + + if tag.trendable? + redis.zadd("#{PREFIX}:allowed", decaying_score, tag.id) + else + redis.zrem("#{PREFIX}:allowed", tag.id) + end + end + end + end + + def would_be_trending?(id) + score(id) > score_at_rank(options[:review_threshold] - 1) + end +end diff --git a/app/models/user.rb b/app/models/user.rb index 5c5e926e6..e47b5f135 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -10,12 +10,9 @@ # encrypted_password :string default(""), not null # reset_password_token :string # reset_password_sent_at :datetime -# remember_created_at :datetime # sign_in_count :integer default(0), not null # current_sign_in_at :datetime # last_sign_in_at :datetime -# current_sign_in_ip :inet -# last_sign_in_ip :inet # admin :boolean default(FALSE), not null # confirmation_token :string # confirmed_at :datetime @@ -34,7 +31,6 @@ # disabled :boolean default(FALSE), not null # moderator :boolean default(FALSE), not null # invite_id :bigint(8) -# remember_token :string # chosen_languages :string is an Array # created_by_application_id :bigint(8) # approved :boolean default(TRUE), not null @@ -42,9 +38,15 @@ # sign_in_token_sent_at :datetime # webauthn_id :string # sign_up_ip :inet +# skip_sign_in_token :boolean # class User < ApplicationRecord + self.ignored_columns = %w( + remember_created_at + remember_token + ) + include Settings::Extend include UserRoles @@ -63,7 +65,7 @@ class User < ApplicationRecord devise :two_factor_backupable, otp_number_of_backup_codes: 10 - devise :registerable, :recoverable, :rememberable, :validatable, + devise :registerable, :recoverable, :validatable, :confirmable include Omniauthable @@ -80,6 +82,7 @@ class User < ApplicationRecord has_many :invites, inverse_of: :user has_many :markers, inverse_of: :user, dependent: :destroy has_many :webauthn_credentials, dependent: :destroy + has_many :ips, class_name: 'UserIp', inverse_of: :user has_one :invite_request, class_name: 'UserInviteRequest', inverse_of: :user, dependent: :destroy accepts_nested_attributes_for :invite_request, reject_if: ->(attributes) { attributes['text'].blank? && !Setting.require_invite_text } @@ -106,7 +109,7 @@ class User < ApplicationRecord scope :inactive, -> { where(arel_table[:current_sign_in_at].lt(ACTIVE_DURATION.ago)) } scope :active, -> { confirmed.where(arel_table[:current_sign_in_at].gteq(ACTIVE_DURATION.ago)).joins(:account).where(accounts: { suspended_at: nil }) } scope :matches_email, ->(value) { where(arel_table[:email].matches("#{value}%")) } - scope :matches_ip, ->(value) { left_joins(:session_activations).where('users.current_sign_in_ip <<= ?', value).or(left_joins(:session_activations).where('users.sign_up_ip <<= ?', value)).or(left_joins(:session_activations).where('users.last_sign_in_ip <<= ?', value)).or(left_joins(:session_activations).where('session_activations.ip <<= ?', value)) } + scope :matches_ip, ->(value) { left_joins(:ips).where('user_ips.ip <<= ?', value) } scope :emailable, -> { confirmed.enabled.joins(:account).merge(Account.searchable) } before_validation :sanitize_languages @@ -173,15 +176,11 @@ class User < ApplicationRecord prepare_new_user! if new_user && approved? end - def update_sign_in!(request, new_sign_in: false) + def update_sign_in!(new_sign_in: false) old_current, new_current = current_sign_in_at, Time.now.utc self.last_sign_in_at = old_current || new_current self.current_sign_in_at = new_current - old_current, new_current = current_sign_in_ip, request.remote_ip - self.last_sign_in_ip = old_current || new_current - self.current_sign_in_ip = new_current - if new_sign_in self.sign_in_count ||= 0 self.sign_in_count += 1 @@ -200,7 +199,7 @@ class User < ApplicationRecord end def suspicious_sign_in?(ip) - !otp_required_for_login? && current_sign_in_at.present? && current_sign_in_at < 2.weeks.ago && !recent_ip?(ip) + !otp_required_for_login? && !skip_sign_in_token? && current_sign_in_at.present? && !ips.where(ip: ip).exists? end def functional? @@ -276,31 +275,28 @@ class User < ApplicationRecord @shows_application ||= settings.show_application end - # rubocop:disable Naming/MethodParameterName - def token_for_app(a) - return nil if a.nil? || a.owner != self - Doorkeeper::AccessToken.find_or_create_by(application_id: a.id, resource_owner_id: id) do |t| - t.scopes = a.scopes - t.expires_in = Doorkeeper.configuration.access_token_expires_in + def token_for_app(app) + return nil if app.nil? || app.owner != self + + Doorkeeper::AccessToken.find_or_create_by(application_id: app.id, resource_owner_id: id) do |t| + t.scopes = app.scopes + t.expires_in = Doorkeeper.configuration.access_token_expires_in t.use_refresh_token = Doorkeeper.configuration.refresh_token_enabled? end end - # rubocop:enable Naming/MethodParameterName def activate_session(request) - session_activations.activate(session_id: SecureRandom.hex, - user_agent: request.user_agent, - ip: request.remote_ip).session_id + session_activations.activate( + session_id: SecureRandom.hex, + user_agent: request.user_agent, + ip: request.remote_ip + ).session_id end def clear_other_sessions(id) session_activations.exclusive(id) end - def session_active?(id) - session_activations.active? id - end - def web_push_subscription(session) session.web_push_subscription.nil? ? nil : session.web_push_subscription end @@ -329,12 +325,31 @@ class User < ApplicationRecord super end - def reset_password!(new_password, new_password_confirmation) + def reset_password(new_password, new_password_confirmation) return false if encrypted_password.blank? super end + def reset_password! + # First, change password to something random and deactivate all sessions + transaction do + update(password: SecureRandom.hex) + session_activations.destroy_all + end + + # Then, remove all authorized applications and connected push subscriptions + Doorkeeper::AccessGrant.by_resource_owner(self).in_batches.update_all(revoked_at: Time.now.utc) + + Doorkeeper::AccessToken.by_resource_owner(self).in_batches do |batch| + batch.update_all(revoked_at: Time.now.utc) + Web::PushSubscription.where(access_token_id: batch).delete_all + end + + # Finally, send a reset password prompt to the user + send_reset_password_instructions + end + def show_all_media? setting_display_media == 'show_all' end @@ -343,22 +358,6 @@ class User < ApplicationRecord setting_display_media == 'hide_all' end - def recent_ips - @recent_ips ||= begin - arr = [] - - session_activations.each do |session_activation| - arr << [session_activation.updated_at, session_activation.ip] - end - - arr << [current_sign_in_at, current_sign_in_ip] if current_sign_in_ip.present? - arr << [last_sign_in_at, last_sign_in_ip] if last_sign_in_ip.present? - arr << [created_at, sign_up_ip] if sign_up_ip.present? - - arr.sort_by { |pair| pair.first || Time.now.utc }.uniq(&:last).reverse! - end - end - def sign_in_token_expired? sign_in_token_sent_at.nil? || sign_in_token_sent_at < 5.minutes.ago end @@ -389,10 +388,6 @@ class User < ApplicationRecord private - def recent_ip?(ip) - recent_ips.any? { |(_, recent_ip)| recent_ip == ip } - end - def send_pending_devise_notifications pending_devise_notifications.each do |notification, args, kwargs| render_and_send_devise_message(notification, *args, **kwargs) diff --git a/app/models/user_ip.rb b/app/models/user_ip.rb new file mode 100644 index 000000000..a8e802e13 --- /dev/null +++ b/app/models/user_ip.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true +# == Schema Information +# +# Table name: user_ips +# +# user_id :bigint(8) primary key +# ip :inet +# used_at :datetime +# + +class UserIp < ApplicationRecord + self.primary_key = :user_id + + belongs_to :user, foreign_key: :user_id + + def readonly? + true + end +end |