diff options
Diffstat (limited to 'app/models')
52 files changed, 742 insertions, 265 deletions
diff --git a/app/models/account.rb b/app/models/account.rb index 9627cc608..7059c555f 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -64,6 +64,7 @@ class Account < ApplicationRecord USERNAME_RE = /[a-z0-9_]+([a-z0-9_\.-]+[a-z0-9_]+)?/i MENTION_RE = /(?<=^|[^\/[:word:]])@((#{USERNAME_RE})(?:@[[:word:]\.\-]+[[:word:]]+)?)/i URL_PREFIX_RE = /\Ahttp(s?):\/\/[^\/]+/ + USERNAME_ONLY_RE = /\A#{USERNAME_RE}\z/i include Attachmentable include AccountAssociations @@ -88,7 +89,7 @@ class Account < ApplicationRecord validates_with UniqueUsernameValidator, if: -> { will_save_change_to_username? } # Remote user validations - validates :username, format: { with: /\A#{USERNAME_RE}\z/i }, if: -> { !local? && will_save_change_to_username? } + validates :username, format: { with: USERNAME_ONLY_RE }, if: -> { !local? && will_save_change_to_username? } # Local user validations validates :username, format: { with: /\A[a-z0-9_]+\z/i }, length: { maximum: 30 }, if: -> { local? && will_save_change_to_username? && actor_type != 'Application' } @@ -138,6 +139,7 @@ class Account < ApplicationRecord :role, :locale, :shows_application?, + :prefers_noindex?, to: :user, prefix: true, allow_nil: true @@ -194,10 +196,6 @@ class Account < ApplicationRecord "acct:#{local_username_and_domain}" end - def searchable? - !(suspended? || moved?) && (!local? || (approved? && confirmed?)) - end - def possibly_stale? last_webfingered_at.nil? || last_webfingered_at <= 1.day.ago end @@ -298,7 +296,7 @@ class Account < ApplicationRecord def fields (self[:fields] || []).map do |f| - Field.new(self, f) + Account::Field.new(self, f) rescue nil end.compact @@ -364,6 +362,10 @@ class Account < ApplicationRecord username end + def to_log_human_identifier + acct + end + def excluded_from_timeline_account_ids Rails.cache.fetch("exclude_account_ids_for:#{id}") { block_relationships.pluck(:target_account_id) + blocked_by_relationships.pluck(:account_id) + mute_relationships.pluck(:target_account_id) } end @@ -398,52 +400,15 @@ class Account < ApplicationRecord requires_review? && !requested_review? end - class Field < ActiveModelSerializers::Model - attributes :name, :value, :verified_at, :account - - def initialize(account, attributes) - @original_field = attributes - string_limit = account.local? ? 255 : 2047 - super( - account: account, - name: attributes['name'].strip[0, string_limit], - value: attributes['value'].strip[0, string_limit], - verified_at: attributes['verified_at']&.to_datetime, - ) - end - - def verified? - verified_at.present? - end - - def value_for_verification - @value_for_verification ||= begin - if account.local? - value - else - ActionController::Base.helpers.strip_tags(value) - end - end - end - - def verifiable? - value_for_verification.present? && value_for_verification.start_with?('http://', 'https://') - end - - def mark_verified! - self.verified_at = Time.now.utc - @original_field['verified_at'] = verified_at - end - - def to_h - { name: name, value: value, verified_at: verified_at } - end - end - class << self DISALLOWED_TSQUERY_CHARACTERS = /['?\\:‘’]/.freeze TEXTSEARCH = "(setweight(to_tsvector('simple', accounts.display_name), 'A') || setweight(to_tsvector('simple', accounts.username), 'B') || setweight(to_tsvector('simple', coalesce(accounts.domain, '')), 'C'))" + REPUTATION_SCORE_FUNCTION = '(greatest(0, coalesce(s.followers_count, 0)) / (greatest(0, coalesce(s.following_count, 0)) + 1.0))' + FOLLOWERS_SCORE_FUNCTION = 'log(greatest(0, coalesce(s.followers_count, 0)) + 2)' + TIME_DISTANCE_FUNCTION = '(case when s.last_status_at is null then 0 else exp(-1.0 * ((greatest(0, abs(extract(DAY FROM age(s.last_status_at))) - 30.0)^2) / (2.0 * ((-1.0 * 30^2) / (2.0 * ln(0.3)))))) end)' + BOOST = "((#{REPUTATION_SCORE_FUNCTION} + #{FOLLOWERS_SCORE_FUNCTION} + #{TIME_DISTANCE_FUNCTION}) / 3.0)" + def readonly_attributes super - %w(statuses_count following_count followers_count) end @@ -459,9 +424,10 @@ class Account < ApplicationRecord sql = <<-SQL.squish SELECT accounts.*, - ts_rank_cd(#{TEXTSEARCH}, to_tsquery('simple', :tsquery), 32) AS rank + #{BOOST} * ts_rank_cd(#{TEXTSEARCH}, to_tsquery('simple', :tsquery), 32) AS rank FROM accounts LEFT JOIN users ON accounts.id = users.account_id + LEFT JOIN account_stats AS s ON accounts.id = s.account_id WHERE to_tsquery('simple', :tsquery) @@ #{TEXTSEARCH} AND accounts.suspended_at IS NULL AND accounts.moved_to_account_id IS NULL @@ -523,14 +489,15 @@ class Account < ApplicationRecord ) SELECT accounts.*, - (count(f.id) + 1) * ts_rank_cd(#{TEXTSEARCH}, to_tsquery('simple', :tsquery), 32) AS rank + (count(f.id) + 1) * #{BOOST} * ts_rank_cd(#{TEXTSEARCH}, to_tsquery('simple', :tsquery), 32) AS rank FROM accounts LEFT OUTER JOIN follows AS f ON (accounts.id = f.account_id AND f.target_account_id = :id) + LEFT JOIN account_stats AS s ON accounts.id = s.account_id WHERE accounts.id IN (SELECT * FROM first_degree) AND to_tsquery('simple', :tsquery) @@ #{TEXTSEARCH} AND accounts.suspended_at IS NULL AND accounts.moved_to_account_id IS NULL - GROUP BY accounts.id + GROUP BY accounts.id, s.id ORDER BY rank DESC LIMIT :limit OFFSET :offset SQL @@ -538,15 +505,16 @@ class Account < ApplicationRecord <<-SQL.squish SELECT accounts.*, - (count(f.id) + 1) * ts_rank_cd(#{TEXTSEARCH}, to_tsquery('simple', :tsquery), 32) AS rank + (count(f.id) + 1) * #{BOOST} * ts_rank_cd(#{TEXTSEARCH}, to_tsquery('simple', :tsquery), 32) AS rank FROM accounts LEFT OUTER JOIN follows AS f ON (accounts.id = f.account_id AND f.target_account_id = :id) OR (accounts.id = f.target_account_id AND f.account_id = :id) LEFT JOIN users ON accounts.id = users.account_id + LEFT JOIN account_stats AS s ON accounts.id = s.account_id WHERE to_tsquery('simple', :tsquery) @@ #{TEXTSEARCH} AND accounts.suspended_at IS NULL AND accounts.moved_to_account_id IS NULL AND (accounts.domain IS NOT NULL OR (users.approved = TRUE AND users.confirmed_at IS NOT NULL)) - GROUP BY accounts.id + GROUP BY accounts.id, s.id ORDER BY rank DESC LIMIT :limit OFFSET :offset SQL diff --git a/app/models/account/field.rb b/app/models/account/field.rb new file mode 100644 index 000000000..d74f90b2b --- /dev/null +++ b/app/models/account/field.rb @@ -0,0 +1,87 @@ +# frozen_string_literal: true + +class Account::Field < ActiveModelSerializers::Model + MAX_CHARACTERS_LOCAL = 255 + MAX_CHARACTERS_COMPAT = 2_047 + ACCEPTED_SCHEMES = %w(http https).freeze + + attributes :name, :value, :verified_at, :account + + def initialize(account, attributes) + # Keeping this as reference allows us to update the field on the account + # from methods in this class, so that changes can be saved. + @original_field = attributes + @account = account + + super( + name: sanitize(attributes['name']), + value: sanitize(attributes['value']), + verified_at: attributes['verified_at']&.to_datetime, + ) + end + + def verified? + verified_at.present? + end + + def value_for_verification + @value_for_verification ||= begin + if account.local? + value + else + extract_url_from_html + end + end + end + + def verifiable? + return false if value_for_verification.blank? + + # This is slower than checking through a regular expression, but we + # need to confirm that it's not an IDN domain. + + parsed_url = Addressable::URI.parse(value_for_verification) + + ACCEPTED_SCHEMES.include?(parsed_url.scheme) && + parsed_url.user.nil? && + parsed_url.password.nil? && + parsed_url.host.present? && + parsed_url.normalized_host == parsed_url.host + rescue Addressable::URI::InvalidURIError, IDN::Idna::IdnaError + false + end + + def requires_verification? + !verified? && verifiable? + end + + def mark_verified! + @original_field['verified_at'] = self.verified_at = Time.now.utc + end + + def to_h + { name: name, value: value, verified_at: verified_at } + end + + private + + def sanitize(str) + str.strip[0, character_limit] + end + + def character_limit + account.local? ? MAX_CHARACTERS_LOCAL : MAX_CHARACTERS_COMPAT + end + + def extract_url_from_html + doc = Nokogiri::HTML(value).at_xpath('//body') + + return if doc.children.size > 1 + + element = doc.children.first + + return if element.name != 'a' || element['href'] != element.text + + element['href'] + end +end diff --git a/app/models/account_alias.rb b/app/models/account_alias.rb index b421c66e2..b7267d632 100644 --- a/app/models/account_alias.rb +++ b/app/models/account_alias.rb @@ -29,7 +29,7 @@ class AccountAlias < ApplicationRecord end def pretty_acct - username, domain = acct.split('@') + username, domain = acct.split('@', 2) domain.nil? ? username : "#{username}@#{Addressable::IDNA.to_unicode(domain)}" end diff --git a/app/models/account_migration.rb b/app/models/account_migration.rb index 06291c9f3..16276158d 100644 --- a/app/models/account_migration.rb +++ b/app/models/account_migration.rb @@ -58,7 +58,7 @@ class AccountMigration < ApplicationRecord private def set_target_account - self.target_account = ResolveAccountService.new.call(acct) + self.target_account = ResolveAccountService.new.call(acct, skip_cache: true) rescue Webfinger::Error, HTTP::Error, OpenSSL::SSL::SSLError, Mastodon::Error # Validation will take care of it end diff --git a/app/models/account_statuses_cleanup_policy.rb b/app/models/account_statuses_cleanup_policy.rb index 365123653..49adc6ad0 100644 --- a/app/models/account_statuses_cleanup_policy.rb +++ b/app/models/account_statuses_cleanup_policy.rb @@ -139,7 +139,12 @@ class AccountStatusesCleanupPolicy < ApplicationRecord # Filtering on `id` rather than `min_status_age` ago will treat # non-snowflake statuses as older than they really are, but Mastodon # has switched to snowflake IDs significantly over 2 years ago anyway. - max_id = [max_id, Mastodon::Snowflake.id_at(min_status_age.seconds.ago, with_random: false)].compact.min + snowflake_id = Mastodon::Snowflake.id_at(min_status_age.seconds.ago, with_random: false) + + if max_id.nil? || snowflake_id < max_id + max_id = snowflake_id + end + Status.where(Status.arel_table[:id].lteq(max_id)) end diff --git a/app/models/account_warning.rb b/app/models/account_warning.rb index 6067b54b7..961a078b9 100644 --- a/app/models/account_warning.rb +++ b/app/models/account_warning.rb @@ -43,4 +43,8 @@ class AccountWarning < ApplicationRecord def overruled? overruled_at.present? end + + def to_log_human_identifier + target_account.acct + end end diff --git a/app/models/admin/account_action.rb b/app/models/admin/account_action.rb index aed3bc0c7..bce0d6e17 100644 --- a/app/models/admin/account_action.rb +++ b/app/models/admin/account_action.rb @@ -25,6 +25,8 @@ class Admin::AccountAction alias send_email_notification? send_email_notification alias include_statuses? include_statuses + validates :type, :target_account, :current_account, presence: true + def initialize(attributes = {}) @send_email_notification = true @include_statuses = true @@ -41,13 +43,15 @@ class Admin::AccountAction end def save! + raise ActiveRecord::RecordInvalid, self unless valid? + ApplicationRecord.transaction do process_action! process_strike! + process_reports! end process_email! - process_reports! process_queue! end @@ -106,9 +110,8 @@ class Admin::AccountAction # Otherwise, we will mark all unresolved reports about # the account as resolved. - reports.each { |report| authorize(report, :update?) } - reports.each do |report| + authorize(report, :update?) log_action(:resolve, report) report.resolve!(current_account) end diff --git a/app/models/admin/action_log.rb b/app/models/admin/action_log.rb index 401bfd9ac..4fa8008f5 100644 --- a/app/models/admin/action_log.rb +++ b/app/models/admin/action_log.rb @@ -9,38 +9,42 @@ # action :string default(""), not null # target_type :string # target_id :bigint(8) -# recorded_changes :text default(""), not null # created_at :datetime not null # updated_at :datetime not null +# human_identifier :string +# route_param :string +# permalink :string # class Admin::ActionLog < ApplicationRecord - serialize :recorded_changes + self.ignored_columns = %w( + recorded_changes + ) belongs_to :account belongs_to :target, polymorphic: true, optional: true default_scope -> { order('id desc') } + before_validation :set_human_identifier + before_validation :set_route_param + before_validation :set_permalink + def action super.to_sym end - before_validation :set_changes - private - def set_changes - case action - when :destroy, :create - self.recorded_changes = target.attributes - when :update, :promote, :demote - self.recorded_changes = target.previous_changes - when :change_email - self.recorded_changes = ActiveSupport::HashWithIndifferentAccess.new( - email: [target.email, nil], - unconfirmed_email: [nil, target.unconfirmed_email] - ) - end + def set_human_identifier + self.human_identifier = target.to_log_human_identifier if target.respond_to?(:to_log_human_identifier) + end + + def set_route_param + self.route_param = target.to_log_route_param if target.respond_to?(:to_log_route_param) + end + + def set_permalink + self.permalink = target.to_log_permalink if target.respond_to?(:to_log_permalink) end end diff --git a/app/models/admin/action_log_filter.rb b/app/models/admin/action_log_filter.rb index 0f2f712a2..edb391e2e 100644 --- a/app/models/admin/action_log_filter.rb +++ b/app/models/admin/action_log_filter.rb @@ -12,6 +12,7 @@ class Admin::ActionLogFilter reject_appeal: { target_type: 'Appeal', action: 'reject' }.freeze, assigned_to_self_report: { target_type: 'Report', action: 'assigned_to_self' }.freeze, change_email_user: { target_type: 'User', action: 'change_email' }.freeze, + change_role_user: { target_type: 'User', action: 'change_role' }.freeze, confirm_user: { target_type: 'User', action: 'confirm' }.freeze, approve_user: { target_type: 'User', action: 'approve' }.freeze, reject_user: { target_type: 'User', action: 'reject' }.freeze, @@ -21,16 +22,22 @@ class Admin::ActionLogFilter create_domain_allow: { target_type: 'DomainAllow', action: 'create' }.freeze, create_domain_block: { target_type: 'DomainBlock', action: 'create' }.freeze, create_email_domain_block: { target_type: 'EmailDomainBlock', action: 'create' }.freeze, + create_ip_block: { target_type: 'IpBlock', action: 'create' }.freeze, create_unavailable_domain: { target_type: 'UnavailableDomain', action: 'create' }.freeze, + create_user_role: { target_type: 'UserRole', action: 'create' }.freeze, + create_canonical_email_block: { target_type: 'CanonicalEmailBlock', action: 'create' }.freeze, demote_user: { target_type: 'User', action: 'demote' }.freeze, destroy_announcement: { target_type: 'Announcement', action: 'destroy' }.freeze, destroy_custom_emoji: { target_type: 'CustomEmoji', action: 'destroy' }.freeze, destroy_domain_allow: { target_type: 'DomainAllow', action: 'destroy' }.freeze, destroy_domain_block: { target_type: 'DomainBlock', action: 'destroy' }.freeze, + destroy_ip_block: { target_type: 'IpBlock', action: 'destroy' }.freeze, destroy_email_domain_block: { target_type: 'EmailDomainBlock', action: 'destroy' }.freeze, destroy_instance: { target_type: 'Instance', action: 'destroy' }.freeze, destroy_unavailable_domain: { target_type: 'UnavailableDomain', action: 'destroy' }.freeze, destroy_status: { target_type: 'Status', action: 'destroy' }.freeze, + destroy_user_role: { target_type: 'UserRole', action: 'destroy' }.freeze, + destroy_canonical_email_block: { target_type: 'CanonicalEmailBlock', action: 'destroy' }.freeze, disable_2fa_user: { target_type: 'User', action: 'disable' }.freeze, disable_custom_emoji: { target_type: 'CustomEmoji', action: 'disable' }.freeze, disable_user: { target_type: 'User', action: 'disable' }.freeze, @@ -40,6 +47,7 @@ class Admin::ActionLogFilter promote_user: { target_type: 'User', action: 'promote' }.freeze, remove_avatar_user: { target_type: 'User', action: 'remove_avatar' }.freeze, reopen_report: { target_type: 'Report', action: 'reopen' }.freeze, + resend_user: { target_type: 'User', action: 'resend' }.freeze, reset_password_user: { target_type: 'User', action: 'reset_password' }.freeze, resolve_report: { target_type: 'Report', action: 'resolve' }.freeze, sensitive_account: { target_type: 'Account', action: 'sensitive' }.freeze, @@ -52,6 +60,8 @@ class Admin::ActionLogFilter update_announcement: { target_type: 'Announcement', action: 'update' }.freeze, update_custom_emoji: { target_type: 'CustomEmoji', action: 'update' }.freeze, update_status: { target_type: 'Status', action: 'update' }.freeze, + update_user_role: { target_type: 'UserRole', action: 'update' }.freeze, + update_ip_block: { target_type: 'IpBlock', action: 'update' }.freeze, unblock_email_account: { target_type: 'Account', action: 'unblock_email' }.freeze, }.freeze diff --git a/app/models/admin/status_batch_action.rb b/app/models/admin/status_batch_action.rb index 7bf6fa6da..0f019b854 100644 --- a/app/models/admin/status_batch_action.rb +++ b/app/models/admin/status_batch_action.rb @@ -40,11 +40,11 @@ class Admin::StatusBatchAction end def handle_delete! - statuses.each { |status| authorize(status, :destroy?) } + statuses.each { |status| authorize([:admin, status], :destroy?) } ApplicationRecord.transaction do statuses.each do |status| - status.discard + status.discard_with_reblogs log_action(:destroy, status) end @@ -75,7 +75,7 @@ class Admin::StatusBatchAction statuses.includes(:media_attachments, :preview_cards).find_each do |status| next unless status.with_media? || status.with_preview_card? - authorize(status, :update?) + authorize([:admin, status], :update?) if target_account.local? UpdateStatusService.new.call(status, representative_account.id, sensitive: true) diff --git a/app/models/admin/status_filter.rb b/app/models/admin/status_filter.rb index 4fba612a6..d7a16f760 100644 --- a/app/models/admin/status_filter.rb +++ b/app/models/admin/status_filter.rb @@ -3,7 +3,6 @@ class Admin::StatusFilter KEYS = %i( media - id report_id ).freeze @@ -28,12 +27,10 @@ class Admin::StatusFilter private - def scope_for(key, value) + def scope_for(key, _value) case key.to_s when 'media' Status.joins(:media_attachments).merge(@account.media_attachments.reorder(nil)).group(:id).reorder('statuses.id desc') - when 'id' - Status.where(id: value) else raise "Unknown filter: #{key}" end diff --git a/app/models/announcement.rb b/app/models/announcement.rb index f8183aabc..4b2cb4c6d 100644 --- a/app/models/announcement.rb +++ b/app/models/announcement.rb @@ -31,9 +31,12 @@ class Announcement < ApplicationRecord validates :starts_at, presence: true, if: -> { ends_at.present? } validates :ends_at, presence: true, if: -> { starts_at.present? } - before_validation :set_all_day before_validation :set_published, on: :create + def to_log_human_identifier + text + end + def publish! update!(published: true, published_at: Time.now.utc, scheduled_at: nil) end @@ -85,10 +88,6 @@ class Announcement < ApplicationRecord private - def set_all_day - self.all_day = false if starts_at.blank? || ends_at.blank? - end - def set_published return unless scheduled_at.blank? || scheduled_at.past? diff --git a/app/models/appeal.rb b/app/models/appeal.rb index 1f32cfa8b..6fbf60b39 100644 --- a/app/models/appeal.rb +++ b/app/models/appeal.rb @@ -52,6 +52,14 @@ class Appeal < ApplicationRecord update!(rejected_at: Time.now.utc, rejected_by_account: current_account) end + def to_log_human_identifier + account.acct + end + + def to_log_route_param + account_warning_id + end + private def validate_time_frame diff --git a/app/models/canonical_email_block.rb b/app/models/canonical_email_block.rb index 94781386c..1eb69ac67 100644 --- a/app/models/canonical_email_block.rb +++ b/app/models/canonical_email_block.rb @@ -5,27 +5,30 @@ # # id :bigint(8) not null, primary key # canonical_email_hash :string default(""), not null -# reference_account_id :bigint(8) not null +# reference_account_id :bigint(8) # created_at :datetime not null # updated_at :datetime not null # class CanonicalEmailBlock < ApplicationRecord include EmailHelper + include Paginable - belongs_to :reference_account, class_name: 'Account' + belongs_to :reference_account, class_name: 'Account', optional: true validates :canonical_email_hash, presence: true, uniqueness: true + scope :matching_email, ->(email) { where(canonical_email_hash: email_to_canonical_email_hash(email)) } + + def to_log_human_identifier + canonical_email_hash + end + def email=(email) self.canonical_email_hash = email_to_canonical_email_hash(email) end def self.block?(email) - where(canonical_email_hash: email_to_canonical_email_hash(email)).exists? - end - - def self.find_blocks(email) - where(canonical_email_hash: email_to_canonical_email_hash(email)) + matching_email(email).exists? end end diff --git a/app/models/concerns/account_interactions.rb b/app/models/concerns/account_interactions.rb index 9b358d338..15c49f2fe 100644 --- a/app/models/concerns/account_interactions.rb +++ b/app/models/concerns/account_interactions.rb @@ -9,6 +9,7 @@ module AccountInteractions mapping[follow.target_account_id] = { reblogs: follow.show_reblogs?, notify: follow.notify?, + languages: follow.languages, } end end @@ -38,6 +39,7 @@ module AccountInteractions mapping[follow_request.target_account_id] = { reblogs: follow_request.show_reblogs?, notify: follow_request.notify?, + languages: follow_request.languages, } end end @@ -100,12 +102,13 @@ module AccountInteractions has_many :announcement_mutes, dependent: :destroy end - def follow!(other_account, reblogs: nil, notify: nil, uri: nil, rate_limit: false, bypass_limit: false) - rel = active_relationships.create_with(show_reblogs: reblogs.nil? ? true : reblogs, notify: notify.nil? ? false : notify, uri: uri, rate_limit: rate_limit, bypass_follow_limit: bypass_limit) + def follow!(other_account, reblogs: nil, notify: nil, languages: nil, uri: nil, rate_limit: false, bypass_limit: false) + rel = active_relationships.create_with(show_reblogs: reblogs.nil? ? true : reblogs, notify: notify.nil? ? false : notify, languages: languages, uri: uri, rate_limit: rate_limit, bypass_follow_limit: bypass_limit) .find_or_create_by!(target_account: other_account) - rel.show_reblogs = reblogs unless reblogs.nil? - rel.notify = notify unless notify.nil? + rel.show_reblogs = reblogs unless reblogs.nil? + rel.notify = notify unless notify.nil? + rel.languages = languages unless languages.nil? rel.save! if rel.changed? @@ -114,12 +117,13 @@ module AccountInteractions rel end - def request_follow!(other_account, reblogs: nil, notify: nil, uri: nil, rate_limit: false, bypass_limit: false) - rel = follow_requests.create_with(show_reblogs: reblogs.nil? ? true : reblogs, notify: notify.nil? ? false : notify, uri: uri, rate_limit: rate_limit, bypass_follow_limit: bypass_limit) + def request_follow!(other_account, reblogs: nil, notify: nil, languages: nil, uri: nil, rate_limit: false, bypass_limit: false) + rel = follow_requests.create_with(show_reblogs: reblogs.nil? ? true : reblogs, notify: notify.nil? ? false : notify, uri: uri, languages: languages, rate_limit: rate_limit, bypass_follow_limit: bypass_limit) .find_or_create_by!(target_account: other_account) - rel.show_reblogs = reblogs unless reblogs.nil? - rel.notify = notify unless notify.nil? + rel.show_reblogs = reblogs unless reblogs.nil? + rel.notify = notify unless notify.nil? + rel.languages = languages unless languages.nil? rel.save! if rel.changed? @@ -288,8 +292,7 @@ module AccountInteractions private - def remove_potential_friendship(other_account, mutual = false) + def remove_potential_friendship(other_account) PotentialFriendshipTracker.remove(id, other_account.id) - PotentialFriendshipTracker.remove(other_account.id, id) if mutual end end diff --git a/app/models/content_retention_policy.rb b/app/models/content_retention_policy.rb new file mode 100644 index 000000000..b5e922c8c --- /dev/null +++ b/app/models/content_retention_policy.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +class ContentRetentionPolicy + def self.current + new + end + + def media_cache_retention_period + retention_period Setting.media_cache_retention_period + end + + def content_cache_retention_period + retention_period Setting.content_cache_retention_period + end + + def backups_retention_period + retention_period Setting.backups_retention_period + end + + private + + def retention_period(value) + value.days if value.is_a?(Integer) && value.positive? + end +end diff --git a/app/models/custom_emoji.rb b/app/models/custom_emoji.rb index c89bf0586..5af6aead8 100644 --- a/app/models/custom_emoji.rb +++ b/app/models/custom_emoji.rb @@ -31,6 +31,7 @@ class CustomEmoji < ApplicationRecord SCAN_RE = /(?<=[^[:alnum:]:]|\n|^) :(#{SHORTCODE_RE_FRAGMENT}): (?=[^[:alnum:]:]|$)/x + SHORTCODE_ONLY_RE = /\A#{SHORTCODE_RE_FRAGMENT}\z/ IMAGE_MIME_TYPES = %w(image/png image/gif image/webp).freeze @@ -44,12 +45,12 @@ class CustomEmoji < ApplicationRecord validates_attachment :image, content_type: { content_type: IMAGE_MIME_TYPES }, presence: true validates_attachment_size :image, less_than: LIMIT, unless: :local? validates_attachment_size :image, less_than: LOCAL_LIMIT, if: :local? - validates :shortcode, uniqueness: { scope: :domain }, format: { with: /\A#{SHORTCODE_RE_FRAGMENT}\z/ }, length: { minimum: 2 } + validates :shortcode, uniqueness: { scope: :domain }, format: { with: SHORTCODE_ONLY_RE }, length: { minimum: 2 } scope :local, -> { where(domain: nil) } scope :remote, -> { where.not(domain: nil) } scope :alphabetic, -> { order(domain: :asc, shortcode: :asc) } - scope :by_domain_and_subdomains, ->(domain) { where(domain: domain).or(where(arel_table[:domain].matches('%.' + domain))) } + scope :by_domain_and_subdomains, ->(domain) { where(domain: domain).or(where(arel_table[:domain].matches("%.#{domain}"))) } scope :listed, -> { local.where(disabled: false).where(visible_in_picker: true) } remotable_attachment :image, LIMIT @@ -70,6 +71,10 @@ class CustomEmoji < ApplicationRecord copy.tap(&:save!) end + def to_log_human_identifier + shortcode + end + class << self def from_text(text, domain = nil) return [] if text.blank? diff --git a/app/models/custom_filter_status.rb b/app/models/custom_filter_status.rb index b6bea1394..e748d6963 100644 --- a/app/models/custom_filter_status.rb +++ b/app/models/custom_filter_status.rb @@ -5,7 +5,7 @@ # # id :bigint(8) not null, primary key # custom_filter_id :bigint(8) not null -# status_id :bigint(8) default(""), not null +# status_id :bigint(8) not null # created_at :datetime not null # updated_at :datetime not null # diff --git a/app/models/domain_allow.rb b/app/models/domain_allow.rb index 7a0acbe32..9e746b915 100644 --- a/app/models/domain_allow.rb +++ b/app/models/domain_allow.rb @@ -19,6 +19,10 @@ class DomainAllow < ApplicationRecord scope :matches_domain, ->(value) { where(arel_table[:domain].matches("%#{value}%")) } + def to_log_human_identifier + domain + end + class << self def allowed?(domain) !rule_for(domain).nil? diff --git a/app/models/domain_block.rb b/app/models/domain_block.rb index a15206b5e..8e298ac9d 100644 --- a/app/models/domain_block.rb +++ b/app/models/domain_block.rb @@ -28,8 +28,13 @@ class DomainBlock < ApplicationRecord delegate :count, to: :accounts, prefix: true scope :matches_domain, ->(value) { where(arel_table[:domain].matches("%#{value}%")) } - scope :with_user_facing_limitations, -> { where(severity: [:silence, :suspend]).or(where(reject_media: true)) } - scope :by_severity, -> { order(Arel.sql('(CASE severity WHEN 0 THEN 1 WHEN 1 THEN 2 WHEN 2 THEN 0 END), reject_media, domain')) } + scope :with_user_facing_limitations, -> { where(severity: [:silence, :suspend]) } + scope :with_limitations, -> { where(severity: [:silence, :suspend]).or(where(reject_media: true)) } + scope :by_severity, -> { order(Arel.sql('(CASE severity WHEN 0 THEN 1 WHEN 1 THEN 2 WHEN 2 THEN 0 END), domain')) } + + def to_log_human_identifier + domain + end def policies if suspend? diff --git a/app/models/email_domain_block.rb b/app/models/email_domain_block.rb index f9d74332b..10a0e5102 100644 --- a/app/models/email_domain_block.rb +++ b/app/models/email_domain_block.rb @@ -17,6 +17,7 @@ class EmailDomainBlock < ApplicationRecord ) include DomainNormalizable + include Paginable belongs_to :parent, class_name: 'EmailDomainBlock', optional: true has_many :children, class_name: 'EmailDomainBlock', foreign_key: :parent_id, inverse_of: :parent, dependent: :destroy @@ -26,6 +27,10 @@ class EmailDomainBlock < ApplicationRecord # Used for adding multiple blocks at once attr_accessor :other_domains + def to_log_human_identifier + domain + end + def history @history ||= Trends::History.new('email_domain_blocks', id) end diff --git a/app/models/export.rb b/app/models/export.rb index 5216eed5e..2457dcc15 100644 --- a/app/models/export.rb +++ b/app/models/export.rb @@ -30,9 +30,9 @@ class Export end def to_following_accounts_csv - CSV.generate(headers: ['Account address', 'Show boosts'], write_headers: true) do |csv| + CSV.generate(headers: ['Account address', 'Show boosts', 'Notify on new posts', 'Languages'], write_headers: true) do |csv| account.active_relationships.includes(:target_account).reorder(id: :desc).each do |follow| - csv << [acct(follow.target_account), follow.show_reblogs] + csv << [acct(follow.target_account), follow.show_reblogs, follow.notify, follow.languages&.join(', ')] end end end diff --git a/app/models/extended_description.rb b/app/models/extended_description.rb new file mode 100644 index 000000000..6e5c0d1b6 --- /dev/null +++ b/app/models/extended_description.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class ExtendedDescription < ActiveModelSerializers::Model + attributes :updated_at, :text + + def self.current + custom = Setting.find_by(var: 'site_extended_description') + + if custom&.value.present? + new(text: custom.value, updated_at: custom.updated_at) + else + new + end + end +end diff --git a/app/models/featured_tag.rb b/app/models/featured_tag.rb index 201ce75f5..70f949b6a 100644 --- a/app/models/featured_tag.rb +++ b/app/models/featured_tag.rb @@ -10,24 +10,33 @@ # last_status_at :datetime # created_at :datetime not null # updated_at :datetime not null +# name :string # class FeaturedTag < ApplicationRecord belongs_to :account, inverse_of: :featured_tags belongs_to :tag, inverse_of: :featured_tags, optional: true # Set after validation - validate :validate_tag_name, on: :create + validates :name, presence: true, format: { with: Tag::HASHTAG_NAME_RE }, on: :create + + validate :validate_tag_uniqueness, on: :create validate :validate_featured_tags_limit, on: :create + before_validation :strip_name + before_create :set_tag before_create :reset_data - delegate :display_name, to: :tag + scope :by_name, ->(name) { joins(:tag).where(tag: { name: HashtagNormalizer.new.normalize(name) }) } - attr_writer :name + LIMIT = 10 + + def sign? + true + end - def name - tag_id.present? ? tag.name : @name + def display_name + attributes['name'] || tag.display_name end def increment(timestamp) @@ -40,8 +49,12 @@ class FeaturedTag < ApplicationRecord private + def strip_name + self.name = name&.strip&.gsub(/\A#/, '') + end + def set_tag - self.tag = Tag.find_or_create_by_names(@name)&.first + self.tag = Tag.find_or_create_by_names(name)&.first end def reset_data @@ -50,11 +63,12 @@ class FeaturedTag < ApplicationRecord end def validate_featured_tags_limit - errors.add(:base, I18n.t('featured_tags.errors.limit')) if account.featured_tags.count >= 10 + return unless account.local? + + errors.add(:base, I18n.t('featured_tags.errors.limit')) if account.featured_tags.count >= LIMIT end - def validate_tag_name - errors.add(:name, :blank) if @name.blank? - errors.add(:name, :invalid) unless @name.match?(/\A(#{Tag::HASHTAG_NAME_RE})\z/i) + def validate_tag_uniqueness + errors.add(:name, :taken) if FeaturedTag.by_name(name).where(account_id: account_id).exists? end end diff --git a/app/models/follow.rb b/app/models/follow.rb index a5e3fe809..e5cecbbc1 100644 --- a/app/models/follow.rb +++ b/app/models/follow.rb @@ -11,6 +11,7 @@ # show_reblogs :boolean default(TRUE), not null # uri :string # notify :boolean default(FALSE), not null +# languages :string is an Array # class Follow < ApplicationRecord @@ -27,6 +28,7 @@ class Follow < ApplicationRecord has_one :notification, as: :activity, dependent: :destroy validates :account_id, uniqueness: { scope: :target_account_id } + validates :languages, language: true scope :recent, -> { reorder(id: :desc) } @@ -35,7 +37,7 @@ class Follow < ApplicationRecord end def revoke_request! - FollowRequest.create!(account: account, target_account: target_account, show_reblogs: show_reblogs, notify: notify, uri: uri) + FollowRequest.create!(account: account, target_account: target_account, show_reblogs: show_reblogs, notify: notify, languages: languages, uri: uri) destroy! end diff --git a/app/models/follow_request.rb b/app/models/follow_request.rb index 0b6f7629a..9034250c0 100644 --- a/app/models/follow_request.rb +++ b/app/models/follow_request.rb @@ -11,6 +11,7 @@ # show_reblogs :boolean default(TRUE), not null # uri :string # notify :boolean default(FALSE), not null +# languages :string is an Array # class FollowRequest < ApplicationRecord @@ -27,9 +28,10 @@ class FollowRequest < ApplicationRecord has_one :notification, as: :activity, dependent: :destroy validates :account_id, uniqueness: { scope: :target_account_id } + validates :languages, language: true def authorize! - account.follow!(target_account, reblogs: show_reblogs, notify: notify, uri: uri, bypass_limit: true) + account.follow!(target_account, reblogs: show_reblogs, notify: notify, languages: languages, uri: uri, bypass_limit: true) MergeWorker.perform_async(target_account.id, account.id) if account.local? destroy! end diff --git a/app/models/form/account_batch.rb b/app/models/form/account_batch.rb index dcf155840..5cfcf7205 100644 --- a/app/models/form/account_batch.rb +++ b/app/models/form/account_batch.rb @@ -6,7 +6,8 @@ class Form::AccountBatch include AccountableConcern include Payloadable - attr_accessor :account_ids, :action, :current_account + attr_accessor :account_ids, :action, :current_account, + :select_all_matching, :query def save case action @@ -60,7 +61,11 @@ class Form::AccountBatch end def accounts - Account.where(id: account_ids) + if select_all_matching? + query + else + Account.where(id: account_ids) + end end def approve! @@ -101,7 +106,7 @@ class Form::AccountBatch def reject_account(account) authorize(account.user, :reject?) - log_action(:reject, account.user, username: account.username) + log_action(:reject, account.user) account.suspend!(origin: :local) AccountDeletionWorker.perform_async(account.id, { 'reserve_username' => false }) end @@ -118,4 +123,8 @@ class Form::AccountBatch log_action(:approve, account.user) account.user.approve! end + + def select_all_matching? + select_all_matching == '1' + end end diff --git a/app/models/form/admin_settings.rb b/app/models/form/admin_settings.rb index 4c100ba6b..b53a82db2 100644 --- a/app/models/form/admin_settings.rb +++ b/app/models/form/admin_settings.rb @@ -8,26 +8,22 @@ class Form::AdminSettings site_contact_email site_title site_short_description - site_description site_extended_description site_terms registrations_mode closed_registrations_message - open_deletion timeline_preview bootstrap_timeline_accounts flavour skin activity_api_enabled peers_api_enabled - show_known_fediverse_at_about_page preview_sensitive_media custom_css profile_directory hide_followers_count flavour_and_skin thumbnail - hero mascot show_reblogs_in_public_timelines show_replies_in_public_timelines @@ -40,14 +36,21 @@ class Form::AdminSettings outgoing_spoilers require_invite_text captcha_enabled + media_cache_retention_period + content_cache_retention_period + backups_retention_period + ).freeze + + INTEGER_KEYS = %i( + media_cache_retention_period + content_cache_retention_period + backups_retention_period ).freeze BOOLEAN_KEYS = %i( - open_deletion timeline_preview activity_api_enabled peers_api_enabled - show_known_fediverse_at_about_page preview_sensitive_media profile_directory hide_followers_count @@ -63,7 +66,6 @@ class Form::AdminSettings UPLOAD_KEYS = %i( thumbnail - hero mascot ).freeze @@ -73,33 +75,49 @@ class Form::AdminSettings attr_accessor(*KEYS) - validates :site_short_description, :site_description, html: { wrap_with: :p } - validates :site_extended_description, :site_terms, :closed_registrations_message, html: true - validates :registrations_mode, inclusion: { in: %w(open approved none) } - validates :site_contact_email, :site_contact_username, presence: true - validates :site_contact_username, existing_username: true - validates :bootstrap_timeline_accounts, existing_username: { multiple: true } - validates :show_domain_blocks, inclusion: { in: %w(disabled users all) } - validates :show_domain_blocks_rationale, inclusion: { in: %w(disabled users all) } - - def initialize(_attributes = {}) - super - initialize_attributes + validates :registrations_mode, inclusion: { in: %w(open approved none) }, if: -> { defined?(@registrations_mode) } + validates :site_contact_email, :site_contact_username, presence: true, if: -> { defined?(@site_contact_username) || defined?(@site_contact_email) } + validates :site_contact_username, existing_username: true, if: -> { defined?(@site_contact_username) } + validates :bootstrap_timeline_accounts, existing_username: { multiple: true }, if: -> { defined?(@bootstrap_timeline_accounts) } + validates :show_domain_blocks, inclusion: { in: %w(disabled users all) }, if: -> { defined?(@show_domain_blocks) } + validates :show_domain_blocks_rationale, inclusion: { in: %w(disabled users all) }, if: -> { defined?(@show_domain_blocks_rationale) } + validates :media_cache_retention_period, :content_cache_retention_period, :backups_retention_period, numericality: { only_integer: true }, allow_blank: true, if: -> { defined?(@media_cache_retention_period) || defined?(@content_cache_retention_period) || defined?(@backups_retention_period) } + validates :site_short_description, length: { maximum: 200 }, if: -> { defined?(@site_short_description) } + + KEYS.each do |key| + define_method(key) do + return instance_variable_get("@#{key}") if instance_variable_defined?("@#{key}") + + stored_value = begin + if UPLOAD_KEYS.include?(key) + SiteUpload.where(var: key).first_or_initialize(var: key) + else + Setting.public_send(key) + end + end + + instance_variable_set("@#{key}", stored_value) + end + end + + UPLOAD_KEYS.each do |key| + define_method("#{key}=") do |file| + value = public_send(key) + value.file = file + end end def save return false unless valid? KEYS.each do |key| - next if PSEUDO_KEYS.include?(key) - value = instance_variable_get("@#{key}") + next if PSEUDO_KEYS.include?(key) || !instance_variable_defined?("@#{key}") - if UPLOAD_KEYS.include?(key) && !value.nil? - upload = SiteUpload.where(var: key).first_or_initialize(var: key) - upload.update(file: value) + if UPLOAD_KEYS.include?(key) + public_send(key).save else setting = Setting.where(var: key).first_or_initialize(var: key) - setting.update(value: typecast_value(key, value)) + setting.update(value: typecast_value(key, instance_variable_get("@#{key}"))) end end end @@ -114,16 +132,11 @@ class Form::AdminSettings private - def initialize_attributes - KEYS.each do |key| - next if PSEUDO_KEYS.include?(key) - instance_variable_set("@#{key}", Setting.public_send(key)) if instance_variable_get("@#{key}").nil? - end - end - def typecast_value(key, value) if BOOLEAN_KEYS.include?(key) value == '1' + elsif INTEGER_KEYS.include?(key) + value.blank? ? value : Integer(value) else value end diff --git a/app/models/form/redirect.rb b/app/models/form/redirect.rb index 19ee9faed..795fdd057 100644 --- a/app/models/form/redirect.rb +++ b/app/models/form/redirect.rb @@ -31,7 +31,7 @@ class Form::Redirect private def set_target_account - @target_account = ResolveAccountService.new.call(acct) + @target_account = ResolveAccountService.new.call(acct, skip_cache: true) rescue Webfinger::Error, HTTP::Error, OpenSSL::SSL::SSLError, Mastodon::Error # Validation will take care of it end diff --git a/app/models/instance.rb b/app/models/instance.rb index 36110ee40..edbf02a6d 100644 --- a/app/models/instance.rb +++ b/app/models/instance.rb @@ -48,6 +48,8 @@ class Instance < ApplicationRecord domain end + alias to_log_human_identifier to_param + delegate :exhausted_deliveries_days, to: :delivery_failure_tracker def availability_over_days(num_days, end_date = Time.now.utc.to_date) diff --git a/app/models/ip_block.rb b/app/models/ip_block.rb index e1ab59806..31343f0e1 100644 --- a/app/models/ip_block.rb +++ b/app/models/ip_block.rb @@ -16,6 +16,7 @@ class IpBlock < ApplicationRecord CACHE_KEY = 'blocked_ips' include Expireable + include Paginable enum severity: { sign_up_requires_approval: 5000, @@ -24,9 +25,14 @@ class IpBlock < ApplicationRecord } validates :ip, :severity, presence: true + validates :ip, uniqueness: true after_commit :reset_cache + def to_log_human_identifier + "#{ip}/#{ip.prefix}" + end + class << self def blocked?(remote_ip) blocked_ips_map = Rails.cache.fetch(CACHE_KEY) { FastIpMap.new(IpBlock.where(severity: :no_access).pluck(:ip)) } diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb index 69feffbf0..e29be5d97 100644 --- a/app/models/media_attachment.rb +++ b/app/models/media_attachment.rb @@ -44,7 +44,7 @@ class MediaAttachment < ApplicationRecord MAX_VIDEO_MATRIX_LIMIT = 2_304_000 # 1920x1200px MAX_VIDEO_FRAME_RATE = 60 - IMAGE_FILE_EXTENSIONS = %w(.jpg .jpeg .png .gif .webp).freeze + IMAGE_FILE_EXTENSIONS = %w(.jpg .jpeg .png .gif .webp .heic .heif .avif).freeze VIDEO_FILE_EXTENSIONS = %w(.webm .mp4 .m4v .mov).freeze AUDIO_FILE_EXTENSIONS = %w(.ogg .oga .mp3 .wav .flac .opus .aac .m4a .3gp .wma).freeze @@ -55,7 +55,8 @@ class MediaAttachment < ApplicationRecord small ).freeze - IMAGE_MIME_TYPES = %w(image/jpeg image/png image/gif image/webp).freeze + IMAGE_MIME_TYPES = %w(image/jpeg image/png image/gif image/heic image/heif image/webp image/avif).freeze + IMAGE_CONVERTIBLE_MIME_TYPES = %w(image/heic image/heif).freeze VIDEO_MIME_TYPES = %w(video/webm video/mp4 video/quicktime video/ogg).freeze VIDEO_CONVERTIBLE_MIME_TYPES = %w(video/webm video/quicktime).freeze AUDIO_MIME_TYPES = %w(audio/wave audio/wav audio/x-wav audio/x-pn-wave audio/vnd.wave audio/ogg audio/vorbis audio/mpeg audio/mp3 audio/webm audio/flac audio/aac audio/m4a audio/x-m4a audio/mp4 audio/3gpp video/x-ms-asf).freeze @@ -72,12 +73,22 @@ class MediaAttachment < ApplicationRecord }.freeze, small: { - pixels: 160_000, # 400x400px + pixels: 230_400, # 640x360px file_geometry_parser: FastGeometryParser, blurhash: BLURHASH_OPTIONS, }.freeze, }.freeze + IMAGE_CONVERTED_STYLES = { + original: { + format: 'jpeg', + }.merge(IMAGE_STYLES[:original]).freeze, + + small: { + format: 'jpeg', + }.merge(IMAGE_STYLES[:small]).freeze, + }.freeze + VIDEO_FORMAT = { format: 'mp4', content_type: 'video/mp4', @@ -248,11 +259,11 @@ class MediaAttachment < ApplicationRecord attr_writer :delay_processing def delay_processing? - @delay_processing + @delay_processing && larger_media_format? end def delay_processing_for_attachment?(attachment_name) - @delay_processing && attachment_name == :file + delay_processing? && attachment_name == :file end after_commit :enqueue_processing, on: :create @@ -277,6 +288,8 @@ class MediaAttachment < ApplicationRecord def file_styles(attachment) if attachment.instance.file_content_type == 'image/gif' || VIDEO_CONVERTIBLE_MIME_TYPES.include?(attachment.instance.file_content_type) VIDEO_CONVERTED_STYLES + elsif IMAGE_CONVERTIBLE_MIME_TYPES.include?(attachment.instance.file_content_type) + IMAGE_CONVERTED_STYLES elsif IMAGE_MIME_TYPES.include?(attachment.instance.file_content_type) IMAGE_STYLES elsif VIDEO_MIME_TYPES.include?(attachment.instance.file_content_type) diff --git a/app/models/preview_card.rb b/app/models/preview_card.rb index c49c51a1b..b5d3f9c8f 100644 --- a/app/models/preview_card.rb +++ b/app/models/preview_card.rb @@ -48,6 +48,7 @@ class PreviewCard < ApplicationRecord enum link_type: [:unknown, :article] has_and_belongs_to_many :statuses + has_one :trend, class_name: 'PreviewCardTrend', inverse_of: :preview_card, dependent: :destroy has_attached_file :image, processors: [:thumbnail, :blurhash_transcoder], styles: ->(f) { image_styles(f) }, convert_options: { all: '-quality 80 -strip' }, validate_media_type: false diff --git a/app/models/preview_card_trend.rb b/app/models/preview_card_trend.rb new file mode 100644 index 000000000..018400dfa --- /dev/null +++ b/app/models/preview_card_trend.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +# == Schema Information +# +# Table name: preview_card_trends +# +# id :bigint(8) not null, primary key +# preview_card_id :bigint(8) not null +# score :float default(0.0), not null +# rank :integer default(0), not null +# allowed :boolean default(FALSE), not null +# language :string +# +class PreviewCardTrend < ApplicationRecord + belongs_to :preview_card + scope :allowed, -> { where(allowed: true) } +end diff --git a/app/models/privacy_policy.rb b/app/models/privacy_policy.rb new file mode 100644 index 000000000..36cbf1882 --- /dev/null +++ b/app/models/privacy_policy.rb @@ -0,0 +1,77 @@ +# frozen_string_literal: true + +class PrivacyPolicy < ActiveModelSerializers::Model + DEFAULT_PRIVACY_POLICY = <<~TXT + This privacy policy describes how %{domain} ("%{domain}", "we", "us") collects, protects and uses the personally identifiable information you may provide through the %{domain} website or its API. The policy also describes the choices available to you regarding our use of your personal information and how you can access and update this information. This policy does not apply to the practices of companies that %{domain} does not own or control, or to individuals that %{domain} does not employ or manage. + + # What information do we collect? + + - **Basic account information**: If you register on this server, you may be asked to enter a username, an e-mail address and a password. You may also enter additional profile information such as a display name and biography, and upload a profile picture and header image. The username, display name, biography, profile picture and header image are always listed publicly. + - **Posts, following and other public information**: The list of people you follow is listed publicly, the same is true for your followers. When you submit a message, the date and time is stored as well as the application you submitted the message from. Messages may contain media attachments, such as pictures and videos. Public and unlisted posts are available publicly. When you feature a post on your profile, that is also publicly available information. Your posts are delivered to your followers, in some cases it means they are delivered to different servers and copies are stored there. When you delete posts, this is likewise delivered to your followers. The action of reblogging or favouriting another post is always public. + - **Direct and followers-only posts**: All posts are stored and processed on the server. Followers-only posts are delivered to your followers and users who are mentioned in them, and direct posts are delivered only to users mentioned in them. In some cases it means they are delivered to different servers and copies are stored there. We make a good faith effort to limit the access to those posts only to authorized persons, but other servers may fail to do so. Therefore it's important to review servers your followers belong to. You may toggle an option to approve and reject new followers manually in the settings. **Please keep in mind that the operators of the server and any receiving server may view such messages**, and that recipients may screenshot, copy or otherwise re-share them. **Do not share any sensitive information over Mastodon.** + - **IPs and other metadata**: When you log in, we record the IP address you log in from, as well as the name of your browser application. All the logged in sessions are available for your review and revocation in the settings. The latest IP address used is stored for up to 12 months. We also may retain server logs which include the IP address of every request to our server. + + # What do we use your information for? + + Any of the information we collect from you may be used in the following ways: + + - To provide the core functionality of Mastodon. You can only interact with other people's content and post your own content when you are logged in. For example, you may follow other people to view their combined posts in your own personalized home timeline. + - To aid moderation of the community, for example comparing your IP address with other known ones to determine ban evasion or other violations. + - The email address you provide may be used to send you information, notifications about other people interacting with your content or sending you messages, and to respond to inquiries, and/or other requests or questions. + + # How do we protect your information? + + We implement a variety of security measures to maintain the safety of your personal information when you enter, submit, or access your personal information. Among other things, your browser session, as well as the traffic between your applications and the API, are secured with SSL, and your password is hashed using a strong one-way algorithm. You may enable two-factor authentication to further secure access to your account. + + # What is our data retention policy? + + We will make a good faith effort to: + + - Retain server logs containing the IP address of all requests to this server, in so far as such logs are kept, no more than 90 days. + - Retain the IP addresses associated with registered users no more than 12 months. + + You can request and download an archive of your content, including your posts, media attachments, profile picture, and header image. + + You may irreversibly delete your account at any time. + + # Do we use cookies? + + Yes. Cookies are small files that a site or its service provider transfers to your computer's hard drive through your Web browser (if you allow). These cookies enable the site to recognize your browser and, if you have a registered account, associate it with your registered account. + + We use cookies to understand and save your preferences for future visits. + + # Do we disclose any information to outside parties? + + We do not sell, trade, or otherwise transfer to outside parties your personally identifiable information. This does not include trusted third parties who assist us in operating our site, conducting our business, or servicing you, so long as those parties agree to keep this information confidential. We may also release your information when we believe release is appropriate to comply with the law, enforce our site policies, or protect ours or others rights, property, or safety. + + Your public content may be downloaded by other servers in the network. Your public and followers-only posts are delivered to the servers where your followers reside, and direct messages are delivered to the servers of the recipients, in so far as those followers or recipients reside on a different server than this. + + When you authorize an application to use your account, depending on the scope of permissions you approve, it may access your public profile information, your following list, your followers, your lists, all your posts, and your favourites. Applications can never access your e-mail address or password. + + # Site usage by children + + If this server is in the EU or the EEA: Our site, products and services are all directed to people who are at least 16 years old. If you are under the age of 16, per the requirements of the GDPR (General Data Protection Regulation) do not use this site. + + If this server is in the USA: Our site, products and services are all directed to people who are at least 13 years old. If you are under the age of 13, per the requirements of COPPA (Children's Online Privacy Protection Act) do not use this site. + + Law requirements can be different if this server is in another jurisdiction. + + ___ + + This document is CC-BY-SA. Originally adapted from the [Discourse privacy policy](https://github.com/discourse/discourse). + TXT + + DEFAULT_UPDATED_AT = DateTime.new(2022, 10, 7).freeze + + attributes :updated_at, :text + + def self.current + custom = Setting.find_by(var: 'site_terms') + + if custom&.value.present? + new(text: custom.value, updated_at: custom.updated_at) + else + new(text: DEFAULT_PRIVACY_POLICY, updated_at: DEFAULT_UPDATED_AT) + end + end +end diff --git a/app/models/public_feed.rb b/app/models/public_feed.rb index 2528ef1b6..a987bb72c 100644 --- a/app/models/public_feed.rb +++ b/app/models/public_feed.rb @@ -29,6 +29,7 @@ class PublicFeed scope.merge!(remote_only_scope) if remote_only? scope.merge!(account_filters_scope) if account? scope.merge!(media_only_scope) if media_only? + scope.merge!(language_scope) if account&.chosen_languages.present? scope.cache_ids.to_a_paginated_by_id(limit, max_id: max_id, since_id: since_id, min_id: min_id) end @@ -97,10 +98,13 @@ class PublicFeed Status.not_local_only end + def language_scope + Status.where(language: account.chosen_languages) + end + def account_filters_scope Status.not_excluded_by_account(account).tap do |scope| scope.merge!(Status.not_domain_blocked_by_account(account)) unless local_only? - scope.merge!(Status.in_chosen_languages(account)) if account.chosen_languages.present? end end end diff --git a/app/models/report.rb b/app/models/report.rb index 2efb6d4a7..525d22ad5 100644 --- a/app/models/report.rb +++ b/app/models/report.rb @@ -33,6 +33,7 @@ class Report < ApplicationRecord belongs_to :assigned_account, class_name: 'Account', optional: true has_many :notes, class_name: 'ReportNote', foreign_key: :report_id, inverse_of: :report, dependent: :destroy + has_many :notifications, as: :activity, dependent: :destroy scope :unresolved, -> { where(action_taken_at: nil) } scope :resolved, -> { where.not(action_taken_at: nil) } @@ -115,6 +116,10 @@ class Report < ApplicationRecord Report.where.not(id: id).where(target_account_id: target_account_id).unresolved.exists? end + def to_log_human_identifier + id + end + def history subquery = [ Admin::ActionLog.where( @@ -136,6 +141,8 @@ class Report < ApplicationRecord Admin::ActionLog.from(Arel::Nodes::As.new(subquery, Admin::ActionLog.arel_table)) end + private + def set_uri self.uri = ActivityPub::TagManager.instance.generate_uri_for(self) if uri.nil? && account.local? end diff --git a/app/models/site_upload.rb b/app/models/site_upload.rb index cf10b30fc..d3b81d4d5 100644 --- a/app/models/site_upload.rb +++ b/app/models/site_upload.rb @@ -12,10 +12,35 @@ # meta :json # created_at :datetime not null # updated_at :datetime not null +# blurhash :string # class SiteUpload < ApplicationRecord - has_attached_file :file + include Attachmentable + + STYLES = { + thumbnail: { + '@1x': { + format: 'png', + geometry: '1200x630#', + file_geometry_parser: FastGeometryParser, + blurhash: { + x_comp: 4, + y_comp: 4, + }.freeze, + }, + + '@2x': { + format: 'png', + geometry: '2400x1260#', + file_geometry_parser: FastGeometryParser, + }.freeze, + }.freeze, + + mascot: {}.freeze, + }.freeze + + has_attached_file :file, styles: ->(file) { STYLES[file.instance.var.to_sym] }, convert_options: { all: '-coalesce -strip' }, processors: [:lazy_thumbnail, :blurhash_transcoder, :type_corrector] validates_attachment_content_type :file, content_type: /\Aimage\/.*\z/ validates :file, presence: true diff --git a/app/models/status.rb b/app/models/status.rb index 3efa23ae2..044816be7 100644 --- a/app/models/status.rb +++ b/app/models/status.rb @@ -77,6 +77,7 @@ class Status < ApplicationRecord has_one :notification, as: :activity, dependent: :destroy has_one :status_stat, inverse_of: :status has_one :poll, inverse_of: :status, dependent: :destroy + has_one :trend, class_name: 'StatusTrend', inverse_of: :status validates :uri, uniqueness: true, presence: true, unless: :local? validates :text, presence: true, unless: -> { with_media? || reblog? } @@ -98,7 +99,6 @@ class Status < ApplicationRecord scope :without_reblogs, -> { where('statuses.reblog_of_id IS NULL') } scope :with_public_visibility, -> { where(visibility: :public) } scope :tagged_with, ->(tag_ids) { joins(:statuses_tags).where(statuses_tags: { tag_id: tag_ids }) } - scope :in_chosen_languages, ->(account) { where(language: nil).or where(language: account.chosen_languages) } scope :excluding_silenced_accounts, -> { left_outer_joins(:account).where(accounts: { silenced_at: nil }) } scope :including_silenced_accounts, -> { left_outer_joins(:account).where.not(accounts: { silenced_at: nil }) } scope :not_excluded_by_account, ->(account) { where.not(account_id: account.excluded_from_timeline_account_ids) } @@ -171,6 +171,14 @@ class Status < ApplicationRecord ].compact.join("\n\n") end + def to_log_human_identifier + account.acct + end + + def to_log_permalink + ActivityPub::TagManager.instance.uri_for(self) + end + def reply? !in_reply_to_id.nil? || attributes['reply'] end @@ -489,6 +497,12 @@ class Status < ApplicationRecord im end + def discard_with_reblogs + discard_time = Time.current + Status.unscoped.where(reblog_of_id: id, deleted_at: [nil, deleted_at]).in_batches.update_all(deleted_at: discard_time) unless reblog? + update_attribute(:deleted_at, discard_time) + end + private def update_status_stat!(attrs) diff --git a/app/models/status_edit.rb b/app/models/status_edit.rb index 33528eb0d..c2330c04f 100644 --- a/app/models/status_edit.rb +++ b/app/models/status_edit.rb @@ -31,7 +31,7 @@ class StatusEdit < ApplicationRecord :preview_remote_url, :text_url, :meta, :blurhash, :not_processed?, :needs_redownload?, :local?, :file, :thumbnail, :thumbnail_remote_url, - :shortcode, to: :media_attachment + :shortcode, :video?, :audio?, to: :media_attachment end rate_limit by: :account, family: :statuses @@ -41,7 +41,8 @@ class StatusEdit < ApplicationRecord default_scope { order(id: :asc) } - delegate :local?, to: :status + delegate :local?, :application, :edited?, :edited_at, + :discarded?, :visibility, to: :status def emojis return @emojis if defined?(@emojis) @@ -60,4 +61,12 @@ class StatusEdit < ApplicationRecord end end end + + def proper + self + end + + def reblog? + false + end end diff --git a/app/models/status_trend.rb b/app/models/status_trend.rb new file mode 100644 index 000000000..b0f1b6942 --- /dev/null +++ b/app/models/status_trend.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +# == Schema Information +# +# Table name: status_trends +# +# id :bigint(8) not null, primary key +# status_id :bigint(8) not null +# account_id :bigint(8) not null +# score :float default(0.0), not null +# rank :integer default(0), not null +# allowed :boolean default(FALSE), not null +# language :string +# + +class StatusTrend < ApplicationRecord + belongs_to :status + belongs_to :account + + scope :allowed, -> { joins('INNER JOIN (SELECT account_id, MAX(score) AS max_score FROM status_trends GROUP BY account_id) AS grouped_status_trends ON status_trends.account_id = grouped_status_trends.account_id AND status_trends.score = grouped_status_trends.max_score').where(allowed: true) } +end diff --git a/app/models/tag.rb b/app/models/tag.rb index 8929baf66..b66f85423 100644 --- a/app/models/tag.rb +++ b/app/models/tag.rb @@ -27,11 +27,14 @@ class Tag < ApplicationRecord has_many :followers, through: :passive_relationships, source: :account HASHTAG_SEPARATORS = "_\u00B7\u200c" - HASHTAG_NAME_RE = "([[:word:]_][[:word:]#{HASHTAG_SEPARATORS}]*[[:alpha:]#{HASHTAG_SEPARATORS}][[:word:]#{HASHTAG_SEPARATORS}]*[[:word:]_])|([[:word:]_]*[[:alpha:]][[:word:]_]*)" - HASHTAG_RE = /(?:^|[^\/\)\w])#(#{HASHTAG_NAME_RE})/i + HASHTAG_NAME_PAT = "([[:word:]_][[:word:]#{HASHTAG_SEPARATORS}]*[[:alpha:]#{HASHTAG_SEPARATORS}][[:word:]#{HASHTAG_SEPARATORS}]*[[:word:]_])|([[:word:]_]*[[:alpha:]][[:word:]_]*)" - validates :name, presence: true, format: { with: /\A(#{HASHTAG_NAME_RE})\z/i } - validates :display_name, format: { with: /\A(#{HASHTAG_NAME_RE})\z/i } + HASHTAG_RE = /(?:^|[^\/\)\w])#(#{HASHTAG_NAME_PAT})/i + HASHTAG_NAME_RE = /\A(#{HASHTAG_NAME_PAT})\z/i + HASHTAG_INVALID_CHARS_RE = /[^[:alnum:]#{HASHTAG_SEPARATORS}]/ + + validates :name, presence: true, format: { with: HASHTAG_NAME_RE } + validates :display_name, format: { with: HASHTAG_NAME_RE } validate :validate_name_change, if: -> { !new_record? && name_changed? } validate :validate_display_name_change, if: -> { !new_record? && display_name_changed? } @@ -102,7 +105,7 @@ class Tag < ApplicationRecord names = Array(name_or_names).map { |str| [normalize(str), str] }.uniq(&:first) names.map do |(normalized_name, display_name)| - tag = matching_name(normalized_name).first || create(name: normalized_name, display_name: display_name.gsub(/[^[:alnum:]#{HASHTAG_SEPARATORS}]/, '')) + tag = matching_name(normalized_name).first || create(name: normalized_name, display_name: display_name.gsub(HASHTAG_INVALID_CHARS_RE, '')) yield tag if block_given? diff --git a/app/models/trends.rb b/app/models/trends.rb index 5d5f2eb22..b09db940e 100644 --- a/app/models/trends.rb +++ b/app/models/trends.rb @@ -26,7 +26,7 @@ module Trends end def self.request_review! - return unless enabled? + return if skip_review? || !enabled? links_requiring_review = links.request_review tags_requiring_review = tags.request_review @@ -46,6 +46,10 @@ module Trends Setting.trends end + def self.skip_review? + Setting.trendable_by_default + end + def self.available_locales @available_locales ||= I18n.available_locales.map { |locale| locale.to_s.split(/[_-]/).first }.uniq end diff --git a/app/models/trends/base.rb b/app/models/trends/base.rb index 047111248..a189f11f2 100644 --- a/app/models/trends/base.rb +++ b/app/models/trends/base.rb @@ -98,4 +98,8 @@ class Trends::Base pipeline.rename(from_key, to_key) end end + + def skip_review? + Setting.trendable_by_default + end end diff --git a/app/models/trends/links.rb b/app/models/trends/links.rb index 604894cd6..8808b3ab6 100644 --- a/app/models/trends/links.rb +++ b/app/models/trends/links.rb @@ -11,6 +11,40 @@ class Trends::Links < Trends::Base decay_threshold: 1, } + class Query < Trends::Query + def filtered_for!(account) + @account = account + self + end + + def filtered_for(account) + clone.filtered_for!(account) + end + + def to_arel + scope = PreviewCard.joins(:trend).reorder(score: :desc) + scope = scope.reorder(language_order_clause.desc, score: :desc) if preferred_languages.present? + scope = scope.merge(PreviewCardTrend.allowed) if @allowed + scope = scope.offset(@offset) if @offset.present? + scope = scope.limit(@limit) if @limit.present? + scope + end + + private + + def language_order_clause + Arel::Nodes::Case.new.when(PreviewCardTrend.arel_table[:language].in(preferred_languages)).then(1).else(0) + end + + def preferred_languages + if @account&.chosen_languages.present? + @account.chosen_languages + else + @locale + end + end + end + def register(status, at_time = Time.now.utc) original_status = status.proper @@ -28,24 +62,33 @@ class Trends::Links < Trends::Base record_used_id(preview_card.id, at_time) end + def query + Query.new(key_prefix, klass) + end + def refresh(at_time = Time.now.utc) - preview_cards = PreviewCard.where(id: (recently_used_ids(at_time) + currently_trending_ids(false, -1)).uniq) + preview_cards = PreviewCard.where(id: (recently_used_ids(at_time) + PreviewCardTrend.pluck(:preview_card_id)).uniq) calculate_scores(preview_cards, at_time) end def request_review - preview_cards = PreviewCard.where(id: currently_trending_ids(false, -1)) + PreviewCardTrend.pluck('distinct language').flat_map do |language| + score_at_threshold = PreviewCardTrend.where(language: language, allowed: true).order(rank: :desc).where('rank <= ?', options[:review_threshold]).first&.score || 0 + preview_card_trends = PreviewCardTrend.where(language: language, allowed: false).joins(:preview_card) - preview_cards.filter_map do |preview_card| - next unless would_be_trending?(preview_card.id) && !preview_card.trendable? && preview_card.requires_review_notification? + preview_card_trends.filter_map do |trend| + preview_card = trend.preview_card - if preview_card.provider.nil? - preview_card.provider = PreviewCardProvider.create(domain: preview_card.domain, requested_review_at: Time.now.utc) - else - preview_card.provider.touch(:requested_review_at) - end + next unless trend.score > score_at_threshold && !preview_card.trendable? && preview_card.requires_review_notification? + + if preview_card.provider.nil? + preview_card.provider = PreviewCardProvider.create(domain: preview_card.domain, requested_review_at: Time.now.utc) + else + preview_card.provider.touch(:requested_review_at) + end - preview_card + preview_card + end end end @@ -62,10 +105,7 @@ class Trends::Links < Trends::Base private def calculate_scores(preview_cards, at_time) - global_items = [] - locale_items = Hash.new { |h, key| h[key] = [] } - - preview_cards.each do |preview_card| + items = preview_cards.map do |preview_card| expected = preview_card.history.get(at_time - 1.day).accounts.to_f expected = 1.0 if expected.zero? observed = preview_card.history.get(at_time).accounts.to_f @@ -89,26 +129,24 @@ class Trends::Links < Trends::Base preview_card.update_columns(max_score: max_score, max_score_at: max_time) end - decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / options[:max_score_halflife].to_f)) - - next unless decaying_score >= options[:decay_threshold] + decaying_score = begin + if max_score.zero? || !valid_locale?(preview_card.language) + 0 + else + max_score * (0.5**((at_time.to_f - max_time.to_f) / options[:max_score_halflife].to_f)) + end + end - global_items << { score: decaying_score, item: preview_card } - locale_items[preview_card.language] << { score: decaying_score, item: preview_card } if valid_locale?(preview_card.language) + [decaying_score, preview_card] end - replace_items('', global_items) + to_insert = items.filter { |(score, _)| score >= options[:decay_threshold] } + to_delete = items.filter { |(score, _)| score < options[:decay_threshold] } - Trends.available_locales.each do |locale| - replace_items(":#{locale}", locale_items[locale]) + PreviewCardTrend.transaction do + PreviewCardTrend.upsert_all(to_insert.map { |(score, preview_card)| { preview_card_id: preview_card.id, score: score, language: preview_card.language, allowed: preview_card.trendable? || false } }, unique_by: :preview_card_id) if to_insert.any? + PreviewCardTrend.where(preview_card_id: to_delete.map { |(_, preview_card)| preview_card.id }).delete_all if to_delete.any? + PreviewCardTrend.connection.exec_update('UPDATE preview_card_trends SET rank = t0.calculated_rank FROM (SELECT id, row_number() OVER w AS calculated_rank FROM preview_card_trends WINDOW w AS (PARTITION BY language ORDER BY score DESC)) t0 WHERE preview_card_trends.id = t0.id') end end - - def filter_for_allowed_items(items) - items.select { |item| item[:item].trendable? } - end - - def would_be_trending?(id) - score(id) > score_at_rank(options[:review_threshold] - 1) - end end diff --git a/app/models/trends/preview_card_filter.rb b/app/models/trends/preview_card_filter.rb index 25add58c8..0a81146d4 100644 --- a/app/models/trends/preview_card_filter.rb +++ b/app/models/trends/preview_card_filter.rb @@ -13,10 +13,10 @@ class Trends::PreviewCardFilter end def results - scope = PreviewCard.unscoped + scope = initial_scope params.each do |key, value| - next if %w(page locale).include?(key.to_s) + next if %w(page).include?(key.to_s) scope.merge!(scope_for(key, value.to_s.strip)) if value.present? end @@ -26,21 +26,30 @@ class Trends::PreviewCardFilter private + def initial_scope + PreviewCard.select(PreviewCard.arel_table[Arel.star]) + .joins(:trend) + .eager_load(:trend) + .reorder(score: :desc) + end + def scope_for(key, value) case key.to_s when 'trending' trending_scope(value) + when 'locale' + PreviewCardTrend.where(language: value) else raise "Unknown filter: #{key}" end end def trending_scope(value) - scope = Trends.links.query - - scope = scope.in_locale(@params[:locale].to_s) if @params[:locale].present? - scope = scope.allowed if value == 'allowed' - - scope.to_arel + case value + when 'allowed' + PreviewCardTrend.allowed + else + PreviewCardTrend.all + end end end diff --git a/app/models/trends/status_batch.rb b/app/models/trends/status_batch.rb index 78d93bed4..f9b97b224 100644 --- a/app/models/trends/status_batch.rb +++ b/app/models/trends/status_batch.rb @@ -30,7 +30,7 @@ class Trends::StatusBatch end def approve! - statuses.each { |status| authorize(status, :review?) } + statuses.each { |status| authorize([:admin, status], :review?) } statuses.update_all(trendable: true) end @@ -45,7 +45,7 @@ class Trends::StatusBatch end def reject! - statuses.each { |status| authorize(status, :review?) } + statuses.each { |status| authorize([:admin, status], :review?) } statuses.update_all(trendable: false) end diff --git a/app/models/trends/status_filter.rb b/app/models/trends/status_filter.rb index 7c453e339..cb0f75d67 100644 --- a/app/models/trends/status_filter.rb +++ b/app/models/trends/status_filter.rb @@ -13,10 +13,10 @@ class Trends::StatusFilter end def results - scope = Status.unscoped.kept + scope = initial_scope params.each do |key, value| - next if %w(page locale).include?(key.to_s) + next if %w(page).include?(key.to_s) scope.merge!(scope_for(key, value.to_s.strip)) if value.present? end @@ -26,21 +26,30 @@ class Trends::StatusFilter private + def initial_scope + Status.select(Status.arel_table[Arel.star]) + .joins(:trend) + .eager_load(:trend) + .reorder(score: :desc) + end + def scope_for(key, value) case key.to_s when 'trending' trending_scope(value) + when 'locale' + StatusTrend.where(language: value) else raise "Unknown filter: #{key}" end end def trending_scope(value) - scope = Trends.statuses.query - - scope = scope.in_locale(@params[:locale].to_s) if @params[:locale].present? - scope = scope.allowed if value == 'allowed' - - scope.to_arel + case value + when 'allowed' + StatusTrend.allowed + else + StatusTrend.all + end end end diff --git a/app/models/trends/statuses.rb b/app/models/trends/statuses.rb index 1b9e9259a..14a05e6d8 100644 --- a/app/models/trends/statuses.rb +++ b/app/models/trends/statuses.rb @@ -20,13 +20,27 @@ class Trends::Statuses < Trends::Base clone.filtered_for!(account) end + def to_arel + scope = Status.joins(:trend).reorder(score: :desc) + scope = scope.reorder(language_order_clause.desc, score: :desc) if preferred_languages.present? + scope = scope.merge(StatusTrend.allowed) if @allowed + scope = scope.not_excluded_by_account(@account).not_domain_blocked_by_account(@account) if @account.present? + scope = scope.offset(@offset) if @offset.present? + scope = scope.limit(@limit) if @limit.present? + scope + end + private - def apply_scopes(scope) - if @account.nil? - scope + def language_order_clause + Arel::Nodes::Case.new.when(StatusTrend.arel_table[:language].in(preferred_languages)).then(1).else(0) + end + + def preferred_languages + if @account&.chosen_languages.present? + @account.chosen_languages else - scope.not_excluded_by_account(@account).not_domain_blocked_by_account(@account) + @locale end end end @@ -36,9 +50,6 @@ class Trends::Statuses < Trends::Base end def add(status, _account_id, at_time = Time.now.utc) - # We rely on the total reblogs and favourites count, so we - # don't record which account did the what and when here - record_used_id(status.id, at_time) end @@ -47,18 +58,23 @@ class Trends::Statuses < Trends::Base end def refresh(at_time = Time.now.utc) - statuses = Status.where(id: (recently_used_ids(at_time) + currently_trending_ids(false, -1)).uniq).includes(:account, :media_attachments) + statuses = Status.where(id: (recently_used_ids(at_time) + StatusTrend.pluck(:status_id)).uniq).includes(:status_stat, :account) calculate_scores(statuses, at_time) end def request_review - statuses = Status.where(id: currently_trending_ids(false, -1)).includes(:account) + StatusTrend.pluck('distinct language').flat_map do |language| + score_at_threshold = StatusTrend.where(language: language, allowed: true).order(rank: :desc).where('rank <= ?', options[:review_threshold]).first&.score || 0 + status_trends = StatusTrend.where(language: language, allowed: false).joins(:status).includes(status: :account) - statuses.filter_map do |status| - next unless would_be_trending?(status.id) && !status.trendable? && status.requires_review_notification? + status_trends.filter_map do |trend| + status = trend.status - status.account.touch(:requested_review_at) - status + if trend.score > score_at_threshold && !status.trendable? && status.requires_review_notification? + status.account.touch(:requested_review_at) + status + end + end end end @@ -75,14 +91,11 @@ class Trends::Statuses < Trends::Base private def eligible?(status) - status.public_visibility? && status.account.discoverable? && !status.account.silenced? && (status.spoiler_text.blank? || Setting.trending_status_cw) && !status.sensitive? && !status.reply? + status.public_visibility? && status.account.discoverable? && !status.account.silenced? && (status.spoiler_text.blank? || Setting.trending_status_cw) && !status.sensitive? && !status.reply? && valid_locale?(status.language) end def calculate_scores(statuses, at_time) - global_items = [] - locale_items = Hash.new { |h, key| h[key] = [] } - - statuses.each do |status| + items = statuses.map do |status| expected = 1.0 observed = (status.reblogs_count + status.favourites_count).to_f @@ -94,29 +107,24 @@ class Trends::Statuses < Trends::Base end end - decaying_score = score * (0.5**((at_time.to_f - status.created_at.to_f) / options[:score_halflife].to_f)) - - next unless decaying_score >= options[:decay_threshold] + decaying_score = begin + if score.zero? || !eligible?(status) + 0 + else + score * (0.5**((at_time.to_f - status.created_at.to_f) / options[:score_halflife].to_f)) + end + end - global_items << { score: decaying_score, item: status } - locale_items[status.language] << { account_id: status.account_id, score: decaying_score, item: status } if valid_locale?(status.language) + [decaying_score, status] end - replace_items('', global_items) + to_insert = items.filter { |(score, _)| score >= options[:decay_threshold] } + to_delete = items.filter { |(score, _)| score < options[:decay_threshold] } - Trends.available_locales.each do |locale| - replace_items(":#{locale}", locale_items[locale]) + StatusTrend.transaction do + StatusTrend.upsert_all(to_insert.map { |(score, status)| { status_id: status.id, account_id: status.account_id, score: score, language: status.language, allowed: status.trendable? || false } }, unique_by: :status_id) if to_insert.any? + StatusTrend.where(status_id: to_delete.map { |(_, status)| status.id }).delete_all if to_delete.any? + StatusTrend.connection.exec_update('UPDATE status_trends SET rank = t0.calculated_rank FROM (SELECT id, row_number() OVER w AS calculated_rank FROM status_trends WINDOW w AS (PARTITION BY language ORDER BY score DESC)) t0 WHERE status_trends.id = t0.id') end end - - def filter_for_allowed_items(items) - # Show only one status per account, pick the one with the highest score - # that's also eligible to trend - - items.group_by { |item| item[:account_id] }.values.filter_map { |account_items| account_items.select { |item| item[:item].trendable? && item[:item].account.discoverable? }.max_by { |item| item[:score] } } - end - - def would_be_trending?(id) - score(id) > score_at_rank(options[:review_threshold] - 1) - end end diff --git a/app/models/unavailable_domain.rb b/app/models/unavailable_domain.rb index 5e8870bde..dfc0ef14e 100644 --- a/app/models/unavailable_domain.rb +++ b/app/models/unavailable_domain.rb @@ -16,6 +16,10 @@ class UnavailableDomain < ApplicationRecord after_commit :reset_cache! + def to_log_human_identifier + domain + end + private def reset_cache! diff --git a/app/models/user.rb b/app/models/user.rb index 46f66526e..0e8a87aea 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -181,6 +181,14 @@ class User < ApplicationRecord update!(disabled: false) end + def to_log_human_identifier + account.acct + end + + def to_log_route_param + account_id + end + def confirm new_user = !confirmed? self.approved = true if open_registrations? && !sign_up_from_ip_requires_approval? @@ -273,18 +281,18 @@ class User < ApplicationRecord save! end + def prefers_noindex? + setting_noindex + end + def preferred_posting_language - valid_locale_cascade(settings.default_language, locale) + valid_locale_cascade(settings.default_language, locale, I18n.locale) end def setting_default_privacy settings.default_privacy || (account.locked? ? 'private' : 'public') end - def allows_digest_emails? - settings.notification_emails['digest'] - end - def allows_report_emails? settings.notification_emails['report'] end diff --git a/app/models/user_role.rb b/app/models/user_role.rb index 57a56c0b0..74dfdc220 100644 --- a/app/models/user_role.rb +++ b/app/models/user_role.rb @@ -155,6 +155,10 @@ class UserRole < ApplicationRecord end end + def to_log_human_identifier + name + end + private def in_permissions?(privilege) |