diff options
Diffstat (limited to 'app/models')
26 files changed, 405 insertions, 137 deletions
diff --git a/app/models/account.rb b/app/models/account.rb index f75750838..79939ad9e 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -138,6 +138,7 @@ class Account < ApplicationRecord :role, :locale, :shows_application?, + :prefers_noindex?, to: :user, prefix: true, allow_nil: true @@ -194,10 +195,6 @@ class Account < ApplicationRecord "acct:#{local_username_and_domain}" end - def searchable? - !(suspended? || moved?) && (!local? || (approved? && confirmed?)) - end - def possibly_stale? last_webfingered_at.nil? || last_webfingered_at <= 1.day.ago end diff --git a/app/models/admin/status_batch_action.rb b/app/models/admin/status_batch_action.rb index 7bf6fa6da..0ec4fef82 100644 --- a/app/models/admin/status_batch_action.rb +++ b/app/models/admin/status_batch_action.rb @@ -40,7 +40,7 @@ class Admin::StatusBatchAction end def handle_delete! - statuses.each { |status| authorize(status, :destroy?) } + statuses.each { |status| authorize([:admin, status], :destroy?) } ApplicationRecord.transaction do statuses.each do |status| @@ -75,7 +75,7 @@ class Admin::StatusBatchAction statuses.includes(:media_attachments, :preview_cards).find_each do |status| next unless status.with_media? || status.with_preview_card? - authorize(status, :update?) + authorize([:admin, status], :update?) if target_account.local? UpdateStatusService.new.call(status, representative_account.id, sensitive: true) diff --git a/app/models/admin/status_filter.rb b/app/models/admin/status_filter.rb index 4fba612a6..d7a16f760 100644 --- a/app/models/admin/status_filter.rb +++ b/app/models/admin/status_filter.rb @@ -3,7 +3,6 @@ class Admin::StatusFilter KEYS = %i( media - id report_id ).freeze @@ -28,12 +27,10 @@ class Admin::StatusFilter private - def scope_for(key, value) + def scope_for(key, _value) case key.to_s when 'media' Status.joins(:media_attachments).merge(@account.media_attachments.reorder(nil)).group(:id).reorder('statuses.id desc') - when 'id' - Status.where(id: value) else raise "Unknown filter: #{key}" end diff --git a/app/models/domain_block.rb b/app/models/domain_block.rb index b08687787..ad1dc2a38 100644 --- a/app/models/domain_block.rb +++ b/app/models/domain_block.rb @@ -28,8 +28,8 @@ class DomainBlock < ApplicationRecord delegate :count, to: :accounts, prefix: true scope :matches_domain, ->(value) { where(arel_table[:domain].matches("%#{value}%")) } - scope :with_user_facing_limitations, -> { where(severity: [:silence, :suspend]).or(where(reject_media: true)) } - scope :by_severity, -> { order(Arel.sql('(CASE severity WHEN 0 THEN 1 WHEN 1 THEN 2 WHEN 2 THEN 0 END), reject_media, domain')) } + scope :with_user_facing_limitations, -> { where(severity: [:silence, :suspend]) } + scope :by_severity, -> { order(Arel.sql('(CASE severity WHEN 0 THEN 1 WHEN 1 THEN 2 WHEN 2 THEN 0 END), domain')) } def to_log_human_identifier domain diff --git a/app/models/extended_description.rb b/app/models/extended_description.rb new file mode 100644 index 000000000..6e5c0d1b6 --- /dev/null +++ b/app/models/extended_description.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class ExtendedDescription < ActiveModelSerializers::Model + attributes :updated_at, :text + + def self.current + custom = Setting.find_by(var: 'site_extended_description') + + if custom&.value.present? + new(text: custom.value, updated_at: custom.updated_at) + else + new + end + end +end diff --git a/app/models/featured_tag.rb b/app/models/featured_tag.rb index 201ce75f5..3f5cddce6 100644 --- a/app/models/featured_tag.rb +++ b/app/models/featured_tag.rb @@ -22,10 +22,18 @@ class FeaturedTag < ApplicationRecord before_create :set_tag before_create :reset_data + scope :by_name, ->(name) { joins(:tag).where(tag: { name: HashtagNormalizer.new.normalize(name) }) } + delegate :display_name, to: :tag attr_writer :name + LIMIT = 10 + + def sign? + true + end + def name tag_id.present? ? tag.name : @name end @@ -50,11 +58,12 @@ class FeaturedTag < ApplicationRecord end def validate_featured_tags_limit - errors.add(:base, I18n.t('featured_tags.errors.limit')) if account.featured_tags.count >= 10 + errors.add(:base, I18n.t('featured_tags.errors.limit')) if account.featured_tags.count >= LIMIT end def validate_tag_name errors.add(:name, :blank) if @name.blank? errors.add(:name, :invalid) unless @name.match?(/\A(#{Tag::HASHTAG_NAME_RE})\z/i) + errors.add(:name, :taken) if FeaturedTag.by_name(@name).where(account_id: account_id).exists? end end diff --git a/app/models/form/admin_settings.rb b/app/models/form/admin_settings.rb index 68c98d43f..b53a82db2 100644 --- a/app/models/form/admin_settings.rb +++ b/app/models/form/admin_settings.rb @@ -8,26 +8,22 @@ class Form::AdminSettings site_contact_email site_title site_short_description - site_description site_extended_description site_terms registrations_mode closed_registrations_message - open_deletion timeline_preview bootstrap_timeline_accounts flavour skin activity_api_enabled peers_api_enabled - show_known_fediverse_at_about_page preview_sensitive_media custom_css profile_directory hide_followers_count flavour_and_skin thumbnail - hero mascot show_reblogs_in_public_timelines show_replies_in_public_timelines @@ -45,12 +41,16 @@ class Form::AdminSettings backups_retention_period ).freeze + INTEGER_KEYS = %i( + media_cache_retention_period + content_cache_retention_period + backups_retention_period + ).freeze + BOOLEAN_KEYS = %i( - open_deletion timeline_preview activity_api_enabled peers_api_enabled - show_known_fediverse_at_about_page preview_sensitive_media profile_directory hide_followers_count @@ -66,7 +66,6 @@ class Form::AdminSettings UPLOAD_KEYS = %i( thumbnail - hero mascot ).freeze @@ -76,34 +75,49 @@ class Form::AdminSettings attr_accessor(*KEYS) - validates :site_short_description, :site_description, html: { wrap_with: :p } - validates :site_extended_description, :site_terms, :closed_registrations_message, html: true - validates :registrations_mode, inclusion: { in: %w(open approved none) } - validates :site_contact_email, :site_contact_username, presence: true - validates :site_contact_username, existing_username: true - validates :bootstrap_timeline_accounts, existing_username: { multiple: true } - validates :show_domain_blocks, inclusion: { in: %w(disabled users all) } - validates :show_domain_blocks_rationale, inclusion: { in: %w(disabled users all) } - validates :media_cache_retention_period, :content_cache_retention_period, :backups_retention_period, numericality: { only_integer: true }, allow_blank: true - - def initialize(_attributes = {}) - super - initialize_attributes + validates :registrations_mode, inclusion: { in: %w(open approved none) }, if: -> { defined?(@registrations_mode) } + validates :site_contact_email, :site_contact_username, presence: true, if: -> { defined?(@site_contact_username) || defined?(@site_contact_email) } + validates :site_contact_username, existing_username: true, if: -> { defined?(@site_contact_username) } + validates :bootstrap_timeline_accounts, existing_username: { multiple: true }, if: -> { defined?(@bootstrap_timeline_accounts) } + validates :show_domain_blocks, inclusion: { in: %w(disabled users all) }, if: -> { defined?(@show_domain_blocks) } + validates :show_domain_blocks_rationale, inclusion: { in: %w(disabled users all) }, if: -> { defined?(@show_domain_blocks_rationale) } + validates :media_cache_retention_period, :content_cache_retention_period, :backups_retention_period, numericality: { only_integer: true }, allow_blank: true, if: -> { defined?(@media_cache_retention_period) || defined?(@content_cache_retention_period) || defined?(@backups_retention_period) } + validates :site_short_description, length: { maximum: 200 }, if: -> { defined?(@site_short_description) } + + KEYS.each do |key| + define_method(key) do + return instance_variable_get("@#{key}") if instance_variable_defined?("@#{key}") + + stored_value = begin + if UPLOAD_KEYS.include?(key) + SiteUpload.where(var: key).first_or_initialize(var: key) + else + Setting.public_send(key) + end + end + + instance_variable_set("@#{key}", stored_value) + end + end + + UPLOAD_KEYS.each do |key| + define_method("#{key}=") do |file| + value = public_send(key) + value.file = file + end end def save return false unless valid? KEYS.each do |key| - next if PSEUDO_KEYS.include?(key) - value = instance_variable_get("@#{key}") + next if PSEUDO_KEYS.include?(key) || !instance_variable_defined?("@#{key}") - if UPLOAD_KEYS.include?(key) && !value.nil? - upload = SiteUpload.where(var: key).first_or_initialize(var: key) - upload.update(file: value) + if UPLOAD_KEYS.include?(key) + public_send(key).save else setting = Setting.where(var: key).first_or_initialize(var: key) - setting.update(value: typecast_value(key, value)) + setting.update(value: typecast_value(key, instance_variable_get("@#{key}"))) end end end @@ -118,16 +132,11 @@ class Form::AdminSettings private - def initialize_attributes - KEYS.each do |key| - next if PSEUDO_KEYS.include?(key) - instance_variable_set("@#{key}", Setting.public_send(key)) if instance_variable_get("@#{key}").nil? - end - end - def typecast_value(key, value) if BOOLEAN_KEYS.include?(key) value == '1' + elsif INTEGER_KEYS.include?(key) + value.blank? ? value : Integer(value) else value end diff --git a/app/models/ip_block.rb b/app/models/ip_block.rb index 8666f4248..31343f0e1 100644 --- a/app/models/ip_block.rb +++ b/app/models/ip_block.rb @@ -25,6 +25,7 @@ class IpBlock < ApplicationRecord } validates :ip, :severity, presence: true + validates :ip, uniqueness: true after_commit :reset_cache diff --git a/app/models/preview_card.rb b/app/models/preview_card.rb index c49c51a1b..b5d3f9c8f 100644 --- a/app/models/preview_card.rb +++ b/app/models/preview_card.rb @@ -48,6 +48,7 @@ class PreviewCard < ApplicationRecord enum link_type: [:unknown, :article] has_and_belongs_to_many :statuses + has_one :trend, class_name: 'PreviewCardTrend', inverse_of: :preview_card, dependent: :destroy has_attached_file :image, processors: [:thumbnail, :blurhash_transcoder], styles: ->(f) { image_styles(f) }, convert_options: { all: '-quality 80 -strip' }, validate_media_type: false diff --git a/app/models/preview_card_trend.rb b/app/models/preview_card_trend.rb new file mode 100644 index 000000000..018400dfa --- /dev/null +++ b/app/models/preview_card_trend.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +# == Schema Information +# +# Table name: preview_card_trends +# +# id :bigint(8) not null, primary key +# preview_card_id :bigint(8) not null +# score :float default(0.0), not null +# rank :integer default(0), not null +# allowed :boolean default(FALSE), not null +# language :string +# +class PreviewCardTrend < ApplicationRecord + belongs_to :preview_card + scope :allowed, -> { where(allowed: true) } +end diff --git a/app/models/privacy_policy.rb b/app/models/privacy_policy.rb new file mode 100644 index 000000000..36cbf1882 --- /dev/null +++ b/app/models/privacy_policy.rb @@ -0,0 +1,77 @@ +# frozen_string_literal: true + +class PrivacyPolicy < ActiveModelSerializers::Model + DEFAULT_PRIVACY_POLICY = <<~TXT + This privacy policy describes how %{domain} ("%{domain}", "we", "us") collects, protects and uses the personally identifiable information you may provide through the %{domain} website or its API. The policy also describes the choices available to you regarding our use of your personal information and how you can access and update this information. This policy does not apply to the practices of companies that %{domain} does not own or control, or to individuals that %{domain} does not employ or manage. + + # What information do we collect? + + - **Basic account information**: If you register on this server, you may be asked to enter a username, an e-mail address and a password. You may also enter additional profile information such as a display name and biography, and upload a profile picture and header image. The username, display name, biography, profile picture and header image are always listed publicly. + - **Posts, following and other public information**: The list of people you follow is listed publicly, the same is true for your followers. When you submit a message, the date and time is stored as well as the application you submitted the message from. Messages may contain media attachments, such as pictures and videos. Public and unlisted posts are available publicly. When you feature a post on your profile, that is also publicly available information. Your posts are delivered to your followers, in some cases it means they are delivered to different servers and copies are stored there. When you delete posts, this is likewise delivered to your followers. The action of reblogging or favouriting another post is always public. + - **Direct and followers-only posts**: All posts are stored and processed on the server. Followers-only posts are delivered to your followers and users who are mentioned in them, and direct posts are delivered only to users mentioned in them. In some cases it means they are delivered to different servers and copies are stored there. We make a good faith effort to limit the access to those posts only to authorized persons, but other servers may fail to do so. Therefore it's important to review servers your followers belong to. You may toggle an option to approve and reject new followers manually in the settings. **Please keep in mind that the operators of the server and any receiving server may view such messages**, and that recipients may screenshot, copy or otherwise re-share them. **Do not share any sensitive information over Mastodon.** + - **IPs and other metadata**: When you log in, we record the IP address you log in from, as well as the name of your browser application. All the logged in sessions are available for your review and revocation in the settings. The latest IP address used is stored for up to 12 months. We also may retain server logs which include the IP address of every request to our server. + + # What do we use your information for? + + Any of the information we collect from you may be used in the following ways: + + - To provide the core functionality of Mastodon. You can only interact with other people's content and post your own content when you are logged in. For example, you may follow other people to view their combined posts in your own personalized home timeline. + - To aid moderation of the community, for example comparing your IP address with other known ones to determine ban evasion or other violations. + - The email address you provide may be used to send you information, notifications about other people interacting with your content or sending you messages, and to respond to inquiries, and/or other requests or questions. + + # How do we protect your information? + + We implement a variety of security measures to maintain the safety of your personal information when you enter, submit, or access your personal information. Among other things, your browser session, as well as the traffic between your applications and the API, are secured with SSL, and your password is hashed using a strong one-way algorithm. You may enable two-factor authentication to further secure access to your account. + + # What is our data retention policy? + + We will make a good faith effort to: + + - Retain server logs containing the IP address of all requests to this server, in so far as such logs are kept, no more than 90 days. + - Retain the IP addresses associated with registered users no more than 12 months. + + You can request and download an archive of your content, including your posts, media attachments, profile picture, and header image. + + You may irreversibly delete your account at any time. + + # Do we use cookies? + + Yes. Cookies are small files that a site or its service provider transfers to your computer's hard drive through your Web browser (if you allow). These cookies enable the site to recognize your browser and, if you have a registered account, associate it with your registered account. + + We use cookies to understand and save your preferences for future visits. + + # Do we disclose any information to outside parties? + + We do not sell, trade, or otherwise transfer to outside parties your personally identifiable information. This does not include trusted third parties who assist us in operating our site, conducting our business, or servicing you, so long as those parties agree to keep this information confidential. We may also release your information when we believe release is appropriate to comply with the law, enforce our site policies, or protect ours or others rights, property, or safety. + + Your public content may be downloaded by other servers in the network. Your public and followers-only posts are delivered to the servers where your followers reside, and direct messages are delivered to the servers of the recipients, in so far as those followers or recipients reside on a different server than this. + + When you authorize an application to use your account, depending on the scope of permissions you approve, it may access your public profile information, your following list, your followers, your lists, all your posts, and your favourites. Applications can never access your e-mail address or password. + + # Site usage by children + + If this server is in the EU or the EEA: Our site, products and services are all directed to people who are at least 16 years old. If you are under the age of 16, per the requirements of the GDPR (General Data Protection Regulation) do not use this site. + + If this server is in the USA: Our site, products and services are all directed to people who are at least 13 years old. If you are under the age of 13, per the requirements of COPPA (Children's Online Privacy Protection Act) do not use this site. + + Law requirements can be different if this server is in another jurisdiction. + + ___ + + This document is CC-BY-SA. Originally adapted from the [Discourse privacy policy](https://github.com/discourse/discourse). + TXT + + DEFAULT_UPDATED_AT = DateTime.new(2022, 10, 7).freeze + + attributes :updated_at, :text + + def self.current + custom = Setting.find_by(var: 'site_terms') + + if custom&.value.present? + new(text: custom.value, updated_at: custom.updated_at) + else + new(text: DEFAULT_PRIVACY_POLICY, updated_at: DEFAULT_UPDATED_AT) + end + end +end diff --git a/app/models/public_feed.rb b/app/models/public_feed.rb index 2528ef1b6..bc8281ef2 100644 --- a/app/models/public_feed.rb +++ b/app/models/public_feed.rb @@ -9,6 +9,7 @@ class PublicFeed # @option [Boolean] :remote # @option [Boolean] :only_media # @option [Boolean] :allow_local_only + # @option [String] :locale def initialize(account, options = {}) @account = account @options = options @@ -29,6 +30,7 @@ class PublicFeed scope.merge!(remote_only_scope) if remote_only? scope.merge!(account_filters_scope) if account? scope.merge!(media_only_scope) if media_only? + scope.merge!(language_scope) scope.cache_ids.to_a_paginated_by_id(limit, max_id: max_id, since_id: since_id, min_id: min_id) end @@ -97,10 +99,19 @@ class PublicFeed Status.not_local_only end + def language_scope + if account&.chosen_languages.present? + Status.where(language: account.chosen_languages) + elsif @options[:locale].present? + Status.where(language: @options[:locale]) + else + Status.all + end + end + def account_filters_scope Status.not_excluded_by_account(account).tap do |scope| scope.merge!(Status.not_domain_blocked_by_account(account)) unless local_only? - scope.merge!(Status.in_chosen_languages(account)) if account.chosen_languages.present? end end end diff --git a/app/models/report.rb b/app/models/report.rb index 42c869dd4..525d22ad5 100644 --- a/app/models/report.rb +++ b/app/models/report.rb @@ -33,6 +33,7 @@ class Report < ApplicationRecord belongs_to :assigned_account, class_name: 'Account', optional: true has_many :notes, class_name: 'ReportNote', foreign_key: :report_id, inverse_of: :report, dependent: :destroy + has_many :notifications, as: :activity, dependent: :destroy scope :unresolved, -> { where(action_taken_at: nil) } scope :resolved, -> { where.not(action_taken_at: nil) } diff --git a/app/models/site_upload.rb b/app/models/site_upload.rb index cf10b30fc..d3b81d4d5 100644 --- a/app/models/site_upload.rb +++ b/app/models/site_upload.rb @@ -12,10 +12,35 @@ # meta :json # created_at :datetime not null # updated_at :datetime not null +# blurhash :string # class SiteUpload < ApplicationRecord - has_attached_file :file + include Attachmentable + + STYLES = { + thumbnail: { + '@1x': { + format: 'png', + geometry: '1200x630#', + file_geometry_parser: FastGeometryParser, + blurhash: { + x_comp: 4, + y_comp: 4, + }.freeze, + }, + + '@2x': { + format: 'png', + geometry: '2400x1260#', + file_geometry_parser: FastGeometryParser, + }.freeze, + }.freeze, + + mascot: {}.freeze, + }.freeze + + has_attached_file :file, styles: ->(file) { STYLES[file.instance.var.to_sym] }, convert_options: { all: '-coalesce -strip' }, processors: [:lazy_thumbnail, :blurhash_transcoder, :type_corrector] validates_attachment_content_type :file, content_type: /\Aimage\/.*\z/ validates :file, presence: true diff --git a/app/models/status.rb b/app/models/status.rb index c1e8862ca..745a1401c 100644 --- a/app/models/status.rb +++ b/app/models/status.rb @@ -77,6 +77,7 @@ class Status < ApplicationRecord has_one :notification, as: :activity, dependent: :destroy has_one :status_stat, inverse_of: :status has_one :poll, inverse_of: :status, dependent: :destroy + has_one :trend, class_name: 'StatusTrend', inverse_of: :status validates :uri, uniqueness: true, presence: true, unless: :local? validates :text, presence: true, unless: -> { with_media? || reblog? } @@ -98,7 +99,6 @@ class Status < ApplicationRecord scope :without_reblogs, -> { where('statuses.reblog_of_id IS NULL') } scope :with_public_visibility, -> { where(visibility: :public) } scope :tagged_with, ->(tag_ids) { joins(:statuses_tags).where(statuses_tags: { tag_id: tag_ids }) } - scope :in_chosen_languages, ->(account) { where(language: nil).or where(language: account.chosen_languages) } scope :excluding_silenced_accounts, -> { left_outer_joins(:account).where(accounts: { silenced_at: nil }) } scope :including_silenced_accounts, -> { left_outer_joins(:account).where.not(accounts: { silenced_at: nil }) } scope :not_excluded_by_account, ->(account) { where.not(account_id: account.excluded_from_timeline_account_ids) } diff --git a/app/models/status_edit.rb b/app/models/status_edit.rb index 33528eb0d..c2330c04f 100644 --- a/app/models/status_edit.rb +++ b/app/models/status_edit.rb @@ -31,7 +31,7 @@ class StatusEdit < ApplicationRecord :preview_remote_url, :text_url, :meta, :blurhash, :not_processed?, :needs_redownload?, :local?, :file, :thumbnail, :thumbnail_remote_url, - :shortcode, to: :media_attachment + :shortcode, :video?, :audio?, to: :media_attachment end rate_limit by: :account, family: :statuses @@ -41,7 +41,8 @@ class StatusEdit < ApplicationRecord default_scope { order(id: :asc) } - delegate :local?, to: :status + delegate :local?, :application, :edited?, :edited_at, + :discarded?, :visibility, to: :status def emojis return @emojis if defined?(@emojis) @@ -60,4 +61,12 @@ class StatusEdit < ApplicationRecord end end end + + def proper + self + end + + def reblog? + false + end end diff --git a/app/models/status_trend.rb b/app/models/status_trend.rb new file mode 100644 index 000000000..b0f1b6942 --- /dev/null +++ b/app/models/status_trend.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +# == Schema Information +# +# Table name: status_trends +# +# id :bigint(8) not null, primary key +# status_id :bigint(8) not null +# account_id :bigint(8) not null +# score :float default(0.0), not null +# rank :integer default(0), not null +# allowed :boolean default(FALSE), not null +# language :string +# + +class StatusTrend < ApplicationRecord + belongs_to :status + belongs_to :account + + scope :allowed, -> { joins('INNER JOIN (SELECT account_id, MAX(score) AS max_score FROM status_trends GROUP BY account_id) AS grouped_status_trends ON status_trends.account_id = grouped_status_trends.account_id AND status_trends.score = grouped_status_trends.max_score').where(allowed: true) } +end diff --git a/app/models/tag_feed.rb b/app/models/tag_feed.rb index fbbdbaae2..64d48ea72 100644 --- a/app/models/tag_feed.rb +++ b/app/models/tag_feed.rb @@ -12,6 +12,7 @@ class TagFeed < PublicFeed # @option [Boolean] :local # @option [Boolean] :remote # @option [Boolean] :only_media + # @option [String] :locale def initialize(tag, account, options = {}) @tag = tag super(account, options) @@ -33,6 +34,7 @@ class TagFeed < PublicFeed scope.merge!(remote_only_scope) if remote_only? scope.merge!(account_filters_scope) if account? scope.merge!(media_only_scope) if media_only? + scope.merge!(language_scope) scope.cache_ids.to_a_paginated_by_id(limit, max_id: max_id, since_id: since_id, min_id: min_id) end diff --git a/app/models/trends.rb b/app/models/trends.rb index 5d5f2eb22..b09db940e 100644 --- a/app/models/trends.rb +++ b/app/models/trends.rb @@ -26,7 +26,7 @@ module Trends end def self.request_review! - return unless enabled? + return if skip_review? || !enabled? links_requiring_review = links.request_review tags_requiring_review = tags.request_review @@ -46,6 +46,10 @@ module Trends Setting.trends end + def self.skip_review? + Setting.trendable_by_default + end + def self.available_locales @available_locales ||= I18n.available_locales.map { |locale| locale.to_s.split(/[_-]/).first }.uniq end diff --git a/app/models/trends/base.rb b/app/models/trends/base.rb index 047111248..a189f11f2 100644 --- a/app/models/trends/base.rb +++ b/app/models/trends/base.rb @@ -98,4 +98,8 @@ class Trends::Base pipeline.rename(from_key, to_key) end end + + def skip_review? + Setting.trendable_by_default + end end diff --git a/app/models/trends/links.rb b/app/models/trends/links.rb index 604894cd6..8808b3ab6 100644 --- a/app/models/trends/links.rb +++ b/app/models/trends/links.rb @@ -11,6 +11,40 @@ class Trends::Links < Trends::Base decay_threshold: 1, } + class Query < Trends::Query + def filtered_for!(account) + @account = account + self + end + + def filtered_for(account) + clone.filtered_for!(account) + end + + def to_arel + scope = PreviewCard.joins(:trend).reorder(score: :desc) + scope = scope.reorder(language_order_clause.desc, score: :desc) if preferred_languages.present? + scope = scope.merge(PreviewCardTrend.allowed) if @allowed + scope = scope.offset(@offset) if @offset.present? + scope = scope.limit(@limit) if @limit.present? + scope + end + + private + + def language_order_clause + Arel::Nodes::Case.new.when(PreviewCardTrend.arel_table[:language].in(preferred_languages)).then(1).else(0) + end + + def preferred_languages + if @account&.chosen_languages.present? + @account.chosen_languages + else + @locale + end + end + end + def register(status, at_time = Time.now.utc) original_status = status.proper @@ -28,24 +62,33 @@ class Trends::Links < Trends::Base record_used_id(preview_card.id, at_time) end + def query + Query.new(key_prefix, klass) + end + def refresh(at_time = Time.now.utc) - preview_cards = PreviewCard.where(id: (recently_used_ids(at_time) + currently_trending_ids(false, -1)).uniq) + preview_cards = PreviewCard.where(id: (recently_used_ids(at_time) + PreviewCardTrend.pluck(:preview_card_id)).uniq) calculate_scores(preview_cards, at_time) end def request_review - preview_cards = PreviewCard.where(id: currently_trending_ids(false, -1)) + PreviewCardTrend.pluck('distinct language').flat_map do |language| + score_at_threshold = PreviewCardTrend.where(language: language, allowed: true).order(rank: :desc).where('rank <= ?', options[:review_threshold]).first&.score || 0 + preview_card_trends = PreviewCardTrend.where(language: language, allowed: false).joins(:preview_card) - preview_cards.filter_map do |preview_card| - next unless would_be_trending?(preview_card.id) && !preview_card.trendable? && preview_card.requires_review_notification? + preview_card_trends.filter_map do |trend| + preview_card = trend.preview_card - if preview_card.provider.nil? - preview_card.provider = PreviewCardProvider.create(domain: preview_card.domain, requested_review_at: Time.now.utc) - else - preview_card.provider.touch(:requested_review_at) - end + next unless trend.score > score_at_threshold && !preview_card.trendable? && preview_card.requires_review_notification? + + if preview_card.provider.nil? + preview_card.provider = PreviewCardProvider.create(domain: preview_card.domain, requested_review_at: Time.now.utc) + else + preview_card.provider.touch(:requested_review_at) + end - preview_card + preview_card + end end end @@ -62,10 +105,7 @@ class Trends::Links < Trends::Base private def calculate_scores(preview_cards, at_time) - global_items = [] - locale_items = Hash.new { |h, key| h[key] = [] } - - preview_cards.each do |preview_card| + items = preview_cards.map do |preview_card| expected = preview_card.history.get(at_time - 1.day).accounts.to_f expected = 1.0 if expected.zero? observed = preview_card.history.get(at_time).accounts.to_f @@ -89,26 +129,24 @@ class Trends::Links < Trends::Base preview_card.update_columns(max_score: max_score, max_score_at: max_time) end - decaying_score = max_score * (0.5**((at_time.to_f - max_time.to_f) / options[:max_score_halflife].to_f)) - - next unless decaying_score >= options[:decay_threshold] + decaying_score = begin + if max_score.zero? || !valid_locale?(preview_card.language) + 0 + else + max_score * (0.5**((at_time.to_f - max_time.to_f) / options[:max_score_halflife].to_f)) + end + end - global_items << { score: decaying_score, item: preview_card } - locale_items[preview_card.language] << { score: decaying_score, item: preview_card } if valid_locale?(preview_card.language) + [decaying_score, preview_card] end - replace_items('', global_items) + to_insert = items.filter { |(score, _)| score >= options[:decay_threshold] } + to_delete = items.filter { |(score, _)| score < options[:decay_threshold] } - Trends.available_locales.each do |locale| - replace_items(":#{locale}", locale_items[locale]) + PreviewCardTrend.transaction do + PreviewCardTrend.upsert_all(to_insert.map { |(score, preview_card)| { preview_card_id: preview_card.id, score: score, language: preview_card.language, allowed: preview_card.trendable? || false } }, unique_by: :preview_card_id) if to_insert.any? + PreviewCardTrend.where(preview_card_id: to_delete.map { |(_, preview_card)| preview_card.id }).delete_all if to_delete.any? + PreviewCardTrend.connection.exec_update('UPDATE preview_card_trends SET rank = t0.calculated_rank FROM (SELECT id, row_number() OVER w AS calculated_rank FROM preview_card_trends WINDOW w AS (PARTITION BY language ORDER BY score DESC)) t0 WHERE preview_card_trends.id = t0.id') end end - - def filter_for_allowed_items(items) - items.select { |item| item[:item].trendable? } - end - - def would_be_trending?(id) - score(id) > score_at_rank(options[:review_threshold] - 1) - end end diff --git a/app/models/trends/preview_card_filter.rb b/app/models/trends/preview_card_filter.rb index 25add58c8..0a81146d4 100644 --- a/app/models/trends/preview_card_filter.rb +++ b/app/models/trends/preview_card_filter.rb @@ -13,10 +13,10 @@ class Trends::PreviewCardFilter end def results - scope = PreviewCard.unscoped + scope = initial_scope params.each do |key, value| - next if %w(page locale).include?(key.to_s) + next if %w(page).include?(key.to_s) scope.merge!(scope_for(key, value.to_s.strip)) if value.present? end @@ -26,21 +26,30 @@ class Trends::PreviewCardFilter private + def initial_scope + PreviewCard.select(PreviewCard.arel_table[Arel.star]) + .joins(:trend) + .eager_load(:trend) + .reorder(score: :desc) + end + def scope_for(key, value) case key.to_s when 'trending' trending_scope(value) + when 'locale' + PreviewCardTrend.where(language: value) else raise "Unknown filter: #{key}" end end def trending_scope(value) - scope = Trends.links.query - - scope = scope.in_locale(@params[:locale].to_s) if @params[:locale].present? - scope = scope.allowed if value == 'allowed' - - scope.to_arel + case value + when 'allowed' + PreviewCardTrend.allowed + else + PreviewCardTrend.all + end end end diff --git a/app/models/trends/status_batch.rb b/app/models/trends/status_batch.rb index 78d93bed4..f9b97b224 100644 --- a/app/models/trends/status_batch.rb +++ b/app/models/trends/status_batch.rb @@ -30,7 +30,7 @@ class Trends::StatusBatch end def approve! - statuses.each { |status| authorize(status, :review?) } + statuses.each { |status| authorize([:admin, status], :review?) } statuses.update_all(trendable: true) end @@ -45,7 +45,7 @@ class Trends::StatusBatch end def reject! - statuses.each { |status| authorize(status, :review?) } + statuses.each { |status| authorize([:admin, status], :review?) } statuses.update_all(trendable: false) end diff --git a/app/models/trends/status_filter.rb b/app/models/trends/status_filter.rb index 7c453e339..cb0f75d67 100644 --- a/app/models/trends/status_filter.rb +++ b/app/models/trends/status_filter.rb @@ -13,10 +13,10 @@ class Trends::StatusFilter end def results - scope = Status.unscoped.kept + scope = initial_scope params.each do |key, value| - next if %w(page locale).include?(key.to_s) + next if %w(page).include?(key.to_s) scope.merge!(scope_for(key, value.to_s.strip)) if value.present? end @@ -26,21 +26,30 @@ class Trends::StatusFilter private + def initial_scope + Status.select(Status.arel_table[Arel.star]) + .joins(:trend) + .eager_load(:trend) + .reorder(score: :desc) + end + def scope_for(key, value) case key.to_s when 'trending' trending_scope(value) + when 'locale' + StatusTrend.where(language: value) else raise "Unknown filter: #{key}" end end def trending_scope(value) - scope = Trends.statuses.query - - scope = scope.in_locale(@params[:locale].to_s) if @params[:locale].present? - scope = scope.allowed if value == 'allowed' - - scope.to_arel + case value + when 'allowed' + StatusTrend.allowed + else + StatusTrend.all + end end end diff --git a/app/models/trends/statuses.rb b/app/models/trends/statuses.rb index 1b9e9259a..14a05e6d8 100644 --- a/app/models/trends/statuses.rb +++ b/app/models/trends/statuses.rb @@ -20,13 +20,27 @@ class Trends::Statuses < Trends::Base clone.filtered_for!(account) end + def to_arel + scope = Status.joins(:trend).reorder(score: :desc) + scope = scope.reorder(language_order_clause.desc, score: :desc) if preferred_languages.present? + scope = scope.merge(StatusTrend.allowed) if @allowed + scope = scope.not_excluded_by_account(@account).not_domain_blocked_by_account(@account) if @account.present? + scope = scope.offset(@offset) if @offset.present? + scope = scope.limit(@limit) if @limit.present? + scope + end + private - def apply_scopes(scope) - if @account.nil? - scope + def language_order_clause + Arel::Nodes::Case.new.when(StatusTrend.arel_table[:language].in(preferred_languages)).then(1).else(0) + end + + def preferred_languages + if @account&.chosen_languages.present? + @account.chosen_languages else - scope.not_excluded_by_account(@account).not_domain_blocked_by_account(@account) + @locale end end end @@ -36,9 +50,6 @@ class Trends::Statuses < Trends::Base end def add(status, _account_id, at_time = Time.now.utc) - # We rely on the total reblogs and favourites count, so we - # don't record which account did the what and when here - record_used_id(status.id, at_time) end @@ -47,18 +58,23 @@ class Trends::Statuses < Trends::Base end def refresh(at_time = Time.now.utc) - statuses = Status.where(id: (recently_used_ids(at_time) + currently_trending_ids(false, -1)).uniq).includes(:account, :media_attachments) + statuses = Status.where(id: (recently_used_ids(at_time) + StatusTrend.pluck(:status_id)).uniq).includes(:status_stat, :account) calculate_scores(statuses, at_time) end def request_review - statuses = Status.where(id: currently_trending_ids(false, -1)).includes(:account) + StatusTrend.pluck('distinct language').flat_map do |language| + score_at_threshold = StatusTrend.where(language: language, allowed: true).order(rank: :desc).where('rank <= ?', options[:review_threshold]).first&.score || 0 + status_trends = StatusTrend.where(language: language, allowed: false).joins(:status).includes(status: :account) - statuses.filter_map do |status| - next unless would_be_trending?(status.id) && !status.trendable? && status.requires_review_notification? + status_trends.filter_map do |trend| + status = trend.status - status.account.touch(:requested_review_at) - status + if trend.score > score_at_threshold && !status.trendable? && status.requires_review_notification? + status.account.touch(:requested_review_at) + status + end + end end end @@ -75,14 +91,11 @@ class Trends::Statuses < Trends::Base private def eligible?(status) - status.public_visibility? && status.account.discoverable? && !status.account.silenced? && (status.spoiler_text.blank? || Setting.trending_status_cw) && !status.sensitive? && !status.reply? + status.public_visibility? && status.account.discoverable? && !status.account.silenced? && (status.spoiler_text.blank? || Setting.trending_status_cw) && !status.sensitive? && !status.reply? && valid_locale?(status.language) end def calculate_scores(statuses, at_time) - global_items = [] - locale_items = Hash.new { |h, key| h[key] = [] } - - statuses.each do |status| + items = statuses.map do |status| expected = 1.0 observed = (status.reblogs_count + status.favourites_count).to_f @@ -94,29 +107,24 @@ class Trends::Statuses < Trends::Base end end - decaying_score = score * (0.5**((at_time.to_f - status.created_at.to_f) / options[:score_halflife].to_f)) - - next unless decaying_score >= options[:decay_threshold] + decaying_score = begin + if score.zero? || !eligible?(status) + 0 + else + score * (0.5**((at_time.to_f - status.created_at.to_f) / options[:score_halflife].to_f)) + end + end - global_items << { score: decaying_score, item: status } - locale_items[status.language] << { account_id: status.account_id, score: decaying_score, item: status } if valid_locale?(status.language) + [decaying_score, status] end - replace_items('', global_items) + to_insert = items.filter { |(score, _)| score >= options[:decay_threshold] } + to_delete = items.filter { |(score, _)| score < options[:decay_threshold] } - Trends.available_locales.each do |locale| - replace_items(":#{locale}", locale_items[locale]) + StatusTrend.transaction do + StatusTrend.upsert_all(to_insert.map { |(score, status)| { status_id: status.id, account_id: status.account_id, score: score, language: status.language, allowed: status.trendable? || false } }, unique_by: :status_id) if to_insert.any? + StatusTrend.where(status_id: to_delete.map { |(_, status)| status.id }).delete_all if to_delete.any? + StatusTrend.connection.exec_update('UPDATE status_trends SET rank = t0.calculated_rank FROM (SELECT id, row_number() OVER w AS calculated_rank FROM status_trends WINDOW w AS (PARTITION BY language ORDER BY score DESC)) t0 WHERE status_trends.id = t0.id') end end - - def filter_for_allowed_items(items) - # Show only one status per account, pick the one with the highest score - # that's also eligible to trend - - items.group_by { |item| item[:account_id] }.values.filter_map { |account_items| account_items.select { |item| item[:item].trendable? && item[:item].account.discoverable? }.max_by { |item| item[:score] } } - end - - def would_be_trending?(id) - score(id) > score_at_rank(options[:review_threshold] - 1) - end end diff --git a/app/models/user.rb b/app/models/user.rb index de59fe4b3..0e8a87aea 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -281,6 +281,10 @@ class User < ApplicationRecord save! end + def prefers_noindex? + setting_noindex + end + def preferred_posting_language valid_locale_cascade(settings.default_language, locale, I18n.locale) end |