diff options
Diffstat (limited to 'app/lib')
27 files changed, 1571 insertions, 198 deletions
diff --git a/app/lib/activitypub/activity.rb b/app/lib/activitypub/activity.rb index 2b5d3ffc2..968dd3f67 100644 --- a/app/lib/activitypub/activity.rb +++ b/app/lib/activitypub/activity.rb @@ -4,8 +4,8 @@ class ActivityPub::Activity include JsonLdHelper include Redisable - SUPPORTED_TYPES = %w(Note Question).freeze - CONVERTED_TYPES = %w(Image Audio Video Article Page Event).freeze + SUPPORTED_TYPES = %w(Note Question Article).freeze + CONVERTED_TYPES = %w(Image Audio Video Page Event).freeze def initialize(json, account, **options) @json = json @@ -190,7 +190,7 @@ class ActivityPub::Activity end def first_local_follower - @account.followers.local.first + @account.followers.local.random.first end def follow_request_from_object @@ -204,9 +204,9 @@ class ActivityPub::Activity def fetch_remote_original_status if object_uri.start_with?('http') return if ActivityPub::TagManager.instance.local_uri?(object_uri) - ActivityPub::FetchRemoteStatusService.new.call(object_uri, id: true, on_behalf_of: @account.followers.local.first) + ActivityPub::FetchRemoteStatusService.new.call(object_uri, id: true, on_behalf_of: signed_fetch_account) elsif @object['url'].present? - ::FetchRemoteStatusService.new.call(@object['url']) + ::FetchRemoteStatusService.new.call(@object['url'], nil, signed_fetch_account) end end diff --git a/app/lib/activitypub/activity/add.rb b/app/lib/activitypub/activity/add.rb index 688ab00b3..03b584302 100644 --- a/app/lib/activitypub/activity/add.rb +++ b/app/lib/activitypub/activity/add.rb @@ -9,6 +9,6 @@ class ActivityPub::Activity::Add < ActivityPub::Activity return unless !status.nil? && status.account_id == @account.id && !@account.pinned?(status) - StatusPin.create!(account: @account, status: status) + StatusPin.create(account: @account, status: status) end end diff --git a/app/lib/activitypub/activity/announce.rb b/app/lib/activitypub/activity/announce.rb index 349e8f77e..327def623 100644 --- a/app/lib/activitypub/activity/announce.rb +++ b/app/lib/activitypub/activity/announce.rb @@ -2,7 +2,7 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity def perform - return reject_payload! if delete_arrived_first?(@json['id']) || !related_to_local_activity? + return reject_payload! if delete_arrived_first?(@json['id']) RedisLock.acquire(lock_options) do |lock| if lock.acquired? @@ -50,7 +50,7 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity elsif audience_to.include?(@account.followers_url) :private else - :direct + :limited end end @@ -58,18 +58,6 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity status.account_id == @account.id || status.distributable? end - def related_to_local_activity? - followed_by_local_accounts? || requested_through_relay? || reblog_of_local_status? - end - - def requested_through_relay? - super || Relay.find_by(inbox_url: @account.inbox_url)&.enabled? - end - - def reblog_of_local_status? - status_from_uri(object_uri)&.account&.local? - end - def lock_options { redis: Redis.current, key: "announce:#{@object['id']}" } end diff --git a/app/lib/activitypub/activity/block.rb b/app/lib/activitypub/activity/block.rb index 90477bf33..d8ca9951e 100644 --- a/app/lib/activitypub/activity/block.rb +++ b/app/lib/activitypub/activity/block.rb @@ -11,7 +11,7 @@ class ActivityPub::Activity::Block < ActivityPub::Activity return end - UnfollowService.new.call(target_account, @account) if target_account.following?(@account) + BlockService.new.call(target_account, @account) @account.block!(target_account, uri: @json['id']) unless delete_arrived_first?(@json['id']) end diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index d56d47a2d..98bcada7a 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -1,6 +1,10 @@ # frozen_string_literal: true +# rubocop:disable Metrics/ClassLength class ActivityPub::Activity::Create < ActivityPub::Activity + include ImgProxyHelper + include DomainControlHelper + def perform dereference_object! @@ -43,7 +47,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity end def create_status - return reject_payload! if unsupported_object_type? || invalid_origin?(object_uri) || tombstone_exists? || !related_to_local_activity? + return reject_payload! if unsupported_object_type? || invalid_origin?(object_uri) || tombstone_exists? || !related_to_local_activity? || twitter_retweet? RedisLock.acquire(lock_options) do |lock| if lock.acquired? @@ -51,7 +55,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity @status = find_existing_status - if @status.nil? + if @status.nil? || @options[:update] process_status elsif @options[:delivered_to_account_id].present? postprocess_audience_and_deliver @@ -72,22 +76,38 @@ class ActivityPub::Activity::Create < ActivityPub::Activity as_array(@object['cc'] || @json['cc']).map { |x| value_or_id(x) } end + def object_uri + @object['id'] || super + end + def process_status @tags = [] @mentions = [] @params = {} - process_status_params + unless @status.nil? + reblog_uri.blank? ? process_status_update_params : process_reblog_update_params + process_tags + process_audience + + @status = UpdateStatusService.new.call(@status, @params, @mentions, @tags) + resolve_thread(@status) + fetch_replies(@status) unless @account.silenced? + return @status + end + + reblog_uri.blank? ? process_status_params : process_reblog_params process_tags process_audience ApplicationRecord.transaction do @status = Status.create!(@params) + process_inline_images! attach_tags(@status) end resolve_thread(@status) - fetch_replies(@status) + fetch_replies(@status) unless @account.silenced? check_for_spam distribute(@status) forward_for_reply @@ -108,7 +128,10 @@ class ActivityPub::Activity::Create < ActivityPub::Activity text: text_from_content || '', language: detected_language, spoiler_text: converted_object_type? ? '' : (text_from_summary || ''), + title: text_from_title, + reblog: reblogged_status, created_at: @object['published'], + expires_at: @object['expires'], override_timestamps: @options[:override_timestamps], reply: @object['inReplyTo'].present?, sensitive: @account.sensitized? || @object['sensitive'] || false, @@ -121,9 +144,61 @@ class ActivityPub::Activity::Create < ActivityPub::Activity end end + def process_status_update_params + @params = begin + { + text: text_from_content || '', + language: detected_language, + spoiler_text: converted_object_type? ? '' : (text_from_summary || ''), + title: text_from_title, + sensitive: @object['sensitive'] || false, + visibility: visibility_from_audience, + expires_at: @object['expires'], + media_attachment_ids: process_attachments.take(4).map(&:id), + } + end + end + + def process_reblog_params + @params = begin + { + uri: object_uri, + url: object_url || object_uri, + account: @account, + text: text_from_content || '', + language: detected_language, + spoiler_text: converted_object_type? ? '' : (text_from_summary || ''), + title: text_from_title, + reblog: reblogged_status, + created_at: @object['published'], + override_timestamps: @options[:override_timestamps], + reply: @object['inReplyTo'].present?, + sensitive: @object['sensitive'] || false, + visibility: visibility_from_audience, + thread: replied_to_status, + } + end + end + + def process_reblog_update_params + @params = begin + { + text: text_from_content || '', + language: detected_language, + spoiler_text: converted_object_type? ? '' : (text_from_summary || ''), + title: text_from_title, + sensitive: @object['sensitive'] || false, + visibility: visibility_from_audience, + } + end + end + def process_audience + @params[:visibility] = :unlisted if @account.silenced? && @params[:visibility] == :public + (audience_to + audience_cc).uniq.each do |audience| next if audience == ActivityPub::TagManager::COLLECTIONS[:public] + next (@params[:visibility] = :limited) if domain_not_allowed?(audience) # Unlike with tags, there is no point in resolving accounts we don't already # know here, because silent mentions would only be used for local access @@ -133,15 +208,18 @@ class ActivityPub::Activity::Create < ActivityPub::Activity next if account.nil? || @mentions.any? { |mention| mention.account_id == account.id } @mentions << Mention.new(account: account, silent: true) + @params[:visibility] = :unlisted if account.silenced? && @params[:visibility] == :public # If there is at least one silent mention, then the status can be considered # as a limited-audience status, and not strictly a direct message, but only # if we considered a direct message in the first place - next unless @params[:visibility] == :direct && direct_message.nil? + next unless account.suspended? || (@params[:visibility] == :direct && direct_message.nil?) @params[:visibility] = :limited end + @params[:visibility] = :limited if @params[:reply] && @params[:visibility] == :private && @mentions.pluck(:account_id).without(@account.id).present? + # If the payload was delivered to a specific inbox, the inbox owner must have # access to it, unless they already have access to it anyway return if @options[:delivered_to_account_id].nil? || @mentions.any? { |mention| mention.account_id == @options[:delivered_to_account_id] } @@ -204,11 +282,12 @@ class ActivityPub::Activity::Create < ActivityPub::Activity def process_mention(tag) return if tag['href'].blank? + return (@params[:visibility] = :limited) if domain_not_allowed?(tag['href']) account = account_from_uri(tag['href']) account = ActivityPub::FetchRemoteAccountService.new.call(tag['href']) if account.nil? - return if account.nil? + return (@params[:visibility] = :limited) if account.nil? || account.suspended? @mentions << Mention.new(account: account, silent: false) end @@ -240,7 +319,21 @@ class ActivityPub::Activity::Create < ActivityPub::Activity begin href = Addressable::URI.parse(attachment['url']).normalize.to_s - media_attachment = MediaAttachment.create(account: @account, remote_url: href, thumbnail_remote_url: icon_url_from_attachment(attachment), description: attachment['summary'].presence || attachment['name'].presence, focus: attachment['focalPoint'], blurhash: supported_blurhash?(attachment['blurhash']) ? attachment['blurhash'] : nil) + media_attachment = MediaAttachment.find_by(account: @account, remote_url: href) + + if media_attachment.nil? + media_attachment = MediaAttachment.create(account: @account, remote_url: href, thumbnail_remote_url: icon_url_from_attachment(attachment), description: attachment['summary'].presence || attachment['name'].presence, focus: attachment['focalPoint'], blurhash: supported_blurhash?(attachment['blurhash']) ? attachment['blurhash'] : nil) + else + updated_description = attachment['summary'].presence || media_attachment[:description].presence || attachment['name'].presence || media_attachment[:name].presence + updated_focus = attachment['focalPoint'].presence || media_attachment['focalPoint'].presence + updated_blurhash = supported_blurhash?(attachment['blurhash']) ? attachment['blurhash'] : media_attachment[:blurhash] + + media_attachment.update(description: updated_description, focus: updated_focus, blurhash: updated_blurhash) + + media_attachments << media_attachment + next + end + media_attachments << media_attachment next if unsupported_media_type?(attachment['mediaType']) || skip_download? @@ -330,22 +423,38 @@ class ActivityPub::Activity::Create < ActivityPub::Activity end def fetch_replies(status) + FetchReplyWorker.perform_async(@object['root']) unless invalid_root_uri? + collection = @object['replies'] return if collection.nil? - replies = ActivityPub::FetchRepliesService.new.call(status, collection, false) - return unless replies.nil? - - uri = value_or_id(collection) - ActivityPub::FetchRepliesWorker.perform_async(status.id, uri) unless uri.nil? + if collection.is_a?(Hash) + ActivityPub::FetchRepliesService.new.call(status, collection) + else + uri = value_or_id(collection) + ActivityPub::FetchRepliesWorker.perform_async(status.id, uri) unless uri.nil? + end end def conversation_from_uri(uri) return nil if uri.nil? - return Conversation.find_by(id: OStatus::TagManager.instance.unique_tag_to_local_id(uri, 'Conversation')) if OStatus::TagManager.instance.local_id?(uri) + + conversation = OStatus::TagManager.instance.local_id?(uri) ? Conversation.find_by(id: OStatus::TagManager.instance.unique_tag_to_local_id(uri, 'Conversation')) : nil begin - Conversation.find_or_create_by!(uri: uri) + conversation = Conversation.find_by(uri: uri) if conversation.blank? + + if @object['inReplyTo'].blank? && replied_to_status.blank? + if conversation.blank? + conversation = Conversation.create!(uri: uri, root: object_uri) + elsif conversation.root.blank? + conversation.update!(uri: uri, root: object_uri) + end + elsif conversation.blank? + conversation = Conversation.create!(uri: uri) + end + + conversation rescue ActiveRecord::RecordInvalid, ActiveRecord::RecordNotUnique retry end @@ -377,7 +486,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity def replied_to_status return @replied_to_status if defined?(@replied_to_status) - if in_reply_to_uri.blank? + if in_reply_to_uri.blank? || in_reply_to_uri == object_uri @replied_to_status = nil else @replied_to_status = status_from_uri(in_reply_to_uri) @@ -390,13 +499,28 @@ class ActivityPub::Activity::Create < ActivityPub::Activity value_or_id(@object['inReplyTo']) end + def reblogged_status + FetchRemoteStatusService.new.call(reblog_uri) if reblog_uri.present? + end + + def reblog_uri + return @reblog_uri if defined?(@reblog_uri) + + @reblog_uri = @object['reblog'].presence || @object['_misskey_quote'].presence + end + + def twitter_retweet? + text_from_content.present? && (text_from_content.include?('<p>🐦🔗') || text_from_content.include?('<p>RT @')) + end + def text_from_content - return Formatter.instance.linkify([[text_from_name, text_from_summary.presence].compact.join("\n\n"), object_url || object_uri].join(' ')) if converted_object_type? + return @status_text if defined?(@status_text) + return @status_text = Formatter.instance.linkify([[text_from_name, text_from_summary.presence].compact.join("\n\n"), object_url || object_uri].join(' ')) if converted_object_type? if @object['content'].present? - @object['content'] + @status_text = @object['type'] == 'Article' ? Formatter.instance.format_article(@object['content']) : @object['content'] elsif content_language_map? - @object['contentMap'].values.first + @status_text = @object['contentMap'].values.first end end @@ -408,6 +532,14 @@ class ActivityPub::Activity::Create < ActivityPub::Activity end end + def text_from_title + if @object['title'].present? + @object['title'] + elsif title_language_map? + @object['titleMap'].values.first + end + end + def text_from_name if @object['name'].present? @object['name'] @@ -444,6 +576,10 @@ class ActivityPub::Activity::Create < ActivityPub::Activity @object['summaryMap'].is_a?(Hash) && !@object['summaryMap'].empty? end + def title_language_map? + @object['titleMap'].is_a?(Hash) && !@object['titleMap'].empty? + end + def content_language_map? @object['contentMap'].is_a?(Hash) && !@object['contentMap'].empty? end @@ -490,6 +626,10 @@ class ActivityPub::Activity::Create < ActivityPub::Activity Account.local.where(username: local_usernames).exists? end + def invalid_root_uri? + @object['root'].blank? || [object_uri, @object['url']].include?(@object['root']) || status_from_uri(@object['root']) + end + def tombstone_exists? Tombstone.exists?(uri: object_uri) end @@ -524,3 +664,4 @@ class ActivityPub::Activity::Create < ActivityPub::Activity { redis: Redis.current, key: "vote:#{replied_to_status.poll_id}:#{@account.id}" } end end +# rubocop:enable Metrics/ClassLength diff --git a/app/lib/activitypub/activity/delete.rb b/app/lib/activitypub/activity/delete.rb index 09b9e5e0e..ab2c34cfd 100644 --- a/app/lib/activitypub/activity/delete.rb +++ b/app/lib/activitypub/activity/delete.rb @@ -51,15 +51,12 @@ class ActivityPub::Activity::Delete < ActivityPub::Activity def replied_to_status return @replied_to_status if defined?(@replied_to_status) - @replied_to_status = @status.thread - end - def reply_to_local? - !replied_to_status.nil? && replied_to_status.account.local? + @replied_to_status = @status.thread end def forward_for_reply - return unless @json['signature'].present? && reply_to_local? + return if @json['signature'].blank? || replied_to_status.blank? inboxes = replied_to_status.account.followers.inboxes - [@account.preferred_inbox_url] diff --git a/app/lib/activitypub/activity/update.rb b/app/lib/activitypub/activity/update.rb index 018e2df54..d1dba5196 100644 --- a/app/lib/activitypub/activity/update.rb +++ b/app/lib/activitypub/activity/update.rb @@ -2,6 +2,7 @@ class ActivityPub::Activity::Update < ActivityPub::Activity SUPPORTED_TYPES = %w(Application Group Organization Person Service).freeze + SUPPORTED_OBJECT_TYPES = (ActivityPub::Activity::SUPPORTED_TYPES + ActivityPub::Activity::CONVERTED_TYPES).freeze def perform dereference_object! @@ -10,6 +11,9 @@ class ActivityPub::Activity::Update < ActivityPub::Activity update_account elsif equals_or_includes_any?(@object['type'], %w(Question)) update_poll + elsif equals_or_includes_any?(@object['type'], SUPPORTED_OBJECT_TYPES) + @options[:update] = true + ActivityPub::Activity::Create.new(@json, @account, @options).perform end end diff --git a/app/lib/activitypub/adapter.rb b/app/lib/activitypub/adapter.rb index ef00a4e2e..bf5a49f05 100644 --- a/app/lib/activitypub/adapter.rb +++ b/app/lib/activitypub/adapter.rb @@ -8,6 +8,18 @@ class ActivityPub::Adapter < ActiveModelSerializers::Adapter::Base CONTEXT_EXTENSION_MAP = { direct_message: { 'litepub': 'http://litepub.social/ns#', 'directMessage': 'litepub:directMessage' }, + edited: { 'mp' => 'https://the.monsterpit.net/ns#', 'edited' => 'mp:edited' }, + require_dereference: { 'mp' => 'https://the.monsterpit.net/ns#', 'requireDereference' => 'mp:requireDereference' }, + show_replies: { 'mp' => 'https://the.monsterpit.net/ns#', 'showReplies' => 'mp:showReplies' }, + show_unlisted: { 'mp' => 'https://the.monsterpit.net/ns#', 'showUnlisted' => 'mp:showUnlisted' }, + private: { 'mp' => 'https://the.monsterpit.net/ns#', 'private' => 'mp:private' }, + require_auth: { 'mp' => 'https://the.monsterpit.net/ns#', 'requireAuth' => 'mp:requireAuth' }, + metadata: { 'mp' => 'https://the.monsterpit.net/ns#', 'metadata' => { '@id' => 'mp:metadata', '@type' => '@id' } }, + server_metadata: { 'mp' => 'https://the.monsterpit.net/ns#', 'serverMetadata' => { '@id' => 'mp:serverMetadata', '@type' => '@id' } }, + root: { 'mp' => 'https://the.monsterpit.net/ns#', 'root' => { '@id' => 'mp:root', '@type' => '@id' } }, + reblog: { 'mp' => 'https://the.monsterpit.net/ns#', 'reblog' => { '@id' => 'mp:reblog', '@type' => '@id' }, + 'misskey' => 'https://misskey.io/ns#', '_misskey_quote' => { '@id' => 'misskey:_misskey_quote', '@type' => '@id' } }, + expires: { 'mp' => 'https://the.monsterpit.net/ns#', 'expires' => 'mp:expires' }, manually_approves_followers: { 'manuallyApprovesFollowers' => 'as:manuallyApprovesFollowers' }, sensitive: { 'sensitive' => 'as:sensitive' }, hashtag: { 'Hashtag' => 'as:Hashtag' }, @@ -15,7 +27,7 @@ class ActivityPub::Adapter < ActiveModelSerializers::Adapter::Base also_known_as: { 'alsoKnownAs' => { '@id' => 'as:alsoKnownAs', '@type' => '@id' } }, emoji: { 'toot' => 'http://joinmastodon.org/ns#', 'Emoji' => 'toot:Emoji' }, featured: { 'toot' => 'http://joinmastodon.org/ns#', 'featured' => { '@id' => 'toot:featured', '@type' => '@id' }, 'featuredTags' => { '@id' => 'toot:featuredTags', '@type' => '@id' } }, - property_value: { 'schema' => 'http://schema.org#', 'PropertyValue' => 'schema:PropertyValue', 'value' => 'schema:value' }, + property_value: { 'schema' => 'http://schema.org', 'PropertyValue' => 'schema:PropertyValue', 'value' => 'schema:value' }, atom_uri: { 'ostatus' => 'http://ostatus.org#', 'atomUri' => 'ostatus:atomUri' }, conversation: { 'ostatus' => 'http://ostatus.org#', 'inReplyToAtomUri' => 'ostatus:inReplyToAtomUri', 'conversation' => 'ostatus:conversation' }, focal_point: { 'toot' => 'http://joinmastodon.org/ns#', 'focalPoint' => { '@container' => '@list', '@id' => 'toot:focalPoint' } }, diff --git a/app/lib/activitypub/case_transform.rb b/app/lib/activitypub/case_transform.rb index 7f716f862..7f31fabda 100644 --- a/app/lib/activitypub/case_transform.rb +++ b/app/lib/activitypub/case_transform.rb @@ -14,8 +14,10 @@ module ActivityPub::CaseTransform when String camel_lower_cache[value] ||= if value.start_with?('_:') '_:' + value.gsub(/\A_:/, '').underscore.camelize(:lower) - else + elsif value != '_misskey_quote' value.underscore.camelize(:lower) + else + value end else value end diff --git a/app/lib/activitypub/tag_manager.rb b/app/lib/activitypub/tag_manager.rb index 3f2ae1106..fb1c9d7b2 100644 --- a/app/lib/activitypub/tag_manager.rb +++ b/app/lib/activitypub/tag_manager.rb @@ -64,29 +64,21 @@ class ActivityPub::TagManager # Public statuses go out to primarily the public collection # Unlisted and private statuses go out primarily to the followers collection # Others go out only to the people they mention - def to(status) - case status.visibility - when 'public' - [COLLECTIONS[:public]] - when 'unlisted', 'private' - [account_followers_url(status.account)] - when 'direct', 'limited' - if status.account.silenced? - # Only notify followers if the account is locally silenced - account_ids = status.active_mentions.pluck(:account_id) - to = status.account.followers.where(id: account_ids).each_with_object([]) do |account, result| - result << uri_for(account) - result << account_followers_url(account) if account.group? - end - to.concat(FollowRequest.where(target_account_id: status.account_id, account_id: account_ids).each_with_object([]) do |request, result| - result << uri_for(request.account) - result << account_followers_url(request.account) if request.account.group? - end) - else - status.active_mentions.each_with_object([]) do |mention, result| - result << uri_for(mention.account) - result << account_followers_url(mention.account) if mention.account.group? - end + def to(status, target_domain: nil) + visibility = status.visibility_for_domain(target_domain) + case visibility + when 'public', 'unlisted' + [status.tags.present? ? COLLECTIONS[:public] : account_followers_url(status.account)] + else + account_ids = status.active_mentions.pluck(:account_id) + account_ids |= status.account.follower_ids if visibility == 'private' + + accounts = status.account.silenced? ? status.account.followers.where(id: account_ids) : Account.where(id: account_ids) + accounts = accounts.where(domain: target_domain) if target_domain.present? + + accounts.each_with_object([]) do |account, result| + result << uri_for(account) + result << account_followers_url(account) if account.group? end end end @@ -96,36 +88,32 @@ class ActivityPub::TagManager # Unlisted statuses go to the public as well # Both of those and private statuses also go to the people mentioned in them # Direct ones don't have a secondary audience - def cc(status) + def cc(status, target_domain: nil) cc = [] - cc << uri_for(status.reblog.account) if status.reblog? - case status.visibility - when 'public' - cc << account_followers_url(status.account) - when 'unlisted' - cc << COLLECTIONS[:public] + visibility = status.visibility_for_domain(target_domain) + + case visibility + when 'public', 'unlisted' + cc << (status.tags.present? ? account_followers_url(status.account) : COLLECTIONS[:public]) + account_ids = status.active_mentions.pluck(:account_id) + when 'private', 'limited' + # Work around Mastodon visibility heuritic bug by addressing instance actor. + cc << instance_actor_url + account_ids = status.silent_mentions.pluck(:account_id) + else + account_ids = [] end - unless status.direct_visibility? || status.limited_visibility? - if status.account.silenced? - # Only notify followers if the account is locally silenced - account_ids = status.active_mentions.pluck(:account_id) - cc.concat(status.account.followers.where(id: account_ids).each_with_object([]) do |account, result| - result << uri_for(account) - result << account_followers_url(account) if account.group? - end) - cc.concat(FollowRequest.where(target_account_id: status.account_id, account_id: account_ids).each_with_object([]) do |request, result| - result << uri_for(request.account) - result << account_followers_url(request.account) if request.account.group? - end) - else - cc.concat(status.active_mentions.each_with_object([]) do |mention, result| - result << uri_for(mention.account) - result << account_followers_url(mention.account) if mention.account.group? - end) - end + if account_ids.present? + accounts = status.account.silenced? ? status.account.followers.where(id: account_ids) : Account.where(id: account_ids) + accounts = accounts.where(domain: target_domain) if target_domain.present? + + cc.concat(accounts.each_with_object([]) do |account, result| + result << uri_for(account) + result << account_followers_url(account) if account.group? + end) end cc diff --git a/app/lib/command_tag/command/account_tools.rb b/app/lib/command_tag/command/account_tools.rb new file mode 100644 index 000000000..ac38f19a1 --- /dev/null +++ b/app/lib/command_tag/command/account_tools.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true +module CommandTag::Command::AccountTools + def handle_account_at_start(args) + return if args[0].blank? + + case args[0].downcase + when 'set' + handle_account_set(args[1..-1]) + end + end + + alias handle_acct_at_start handle_account_at_start + + private + + def handle_account_set(args) + return if args[0].blank? + + case args[0].downcase + when 'v', 'p', 'visibility', 'privacy', 'default-visibility', 'default-privacy' + args[1] = read_visibility_from(args[1]) + return if args[1].blank? + + if args[2].blank? + @account.user.settings.default_privacy = args[1] + elsif args[1] == 'public' + domains = args[2..-1].map { |domain| normalize_domain(domain) unless domain == '*' }.uniq.compact + @account.domain_permissions.where(domain: domains, sticky: false).destroy_all if domains.present? + elsif args[1] != 'cc' + args[2..-1].flat_map(&:split).uniq.each do |domain| + domain = normalize_domain(domain) unless domain == '*' + @account.domain_permissions.create_or_update(domain: domain, visibility: args[1]) if domain.present? + end + end + end + end +end diff --git a/app/lib/command_tag/command/footer_tools.rb b/app/lib/command_tag/command/footer_tools.rb new file mode 100644 index 000000000..73e2f05bd --- /dev/null +++ b/app/lib/command_tag/command/footer_tools.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true +module CommandTag::Command::FooterTools + def handle_999_footertools_startup + @status.footer = var('persist:footer:default')[0] + end + + def handle_footer_before_save(args) + return if args.blank? + + name = normalize(args.shift) + return (@status.footer = nil) if read_falsy_from(name) + + var_name = "persist:footer:#{name}" + return @status.footer = var(var_name)[0] if args.blank? + + if read_falsy_from(normalize(args[0])) + @status.footer = nil if ['default', var(var_name)[0]].include?(name) + @vars.delete(var_name) + return + end + + if name == 'default' + name = normalize(args.shift) + var_name = "persist:footer:#{name}" + @vars[var_name] = [args.join(' ').strip] if args.present? + @vars['persist:footer:default'] = var(var_name) + elsif %w(default DEFAULT).include?(args[0]) + @vars['persist:footer:default'] = var(var_name) + else + @vars[var_name] = [args.join(' ').strip] + end + + @status.footer = var(var_name)[0] + end + + # Monsterfork v1 familiarity. + def handle_i_before_save(args) + return if args.blank? + + handle_footer_before_save(args[1..-1]) if %w(am are).include?(normalize(args[0])) + end + + alias handle_we_before_save handle_i_before_save + alias handle_signature_before_save handle_footer_before_save + alias handle_signed_before_save handle_footer_before_save + alias handle_sign_before_save handle_footer_before_save + alias handle_sig_before_save handle_footer_before_save + alias handle_am_before_save handle_footer_before_save + alias handle_are_before_save handle_footer_before_save +end diff --git a/app/lib/command_tag/command/hello_world.rb b/app/lib/command_tag/command/hello_world.rb new file mode 100644 index 000000000..ab10b495b --- /dev/null +++ b/app/lib/command_tag/command/hello_world.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module CommandTag::Command::HelloWorld + def handle_helloworld_startup + @vars['hello_world'] = ['Hello, world!'] + end + + def handle_hello_world_with_return(_) + 'Hello, world!' + end +end diff --git a/app/lib/command_tag/command/parent_status_tools.rb b/app/lib/command_tag/command/parent_status_tools.rb new file mode 100644 index 000000000..2fdee2fb8 --- /dev/null +++ b/app/lib/command_tag/command/parent_status_tools.rb @@ -0,0 +1,80 @@ +# frozen_string_literal: true +module CommandTag::Command::ParentStatusTools + def handle_publish_once_at_end(_) + is_blank = status_text_blank? + return PublishStatusService.new.call(@status) if @parent.blank? || !is_blank + return unless is_blank && author_of_parent? && !@parent.published? + + PublishStatusService.new.call(@parent) + end + + alias handle_publish_post_once_at_end handle_publish_once_at_end + alias handle_publish_roar_once_at_end handle_publish_once_at_end + alias handle_publish_toot_once_at_end handle_publish_once_at_end + + def handle_edit_once_before_save(_) + return unless author_of_parent? + + params = @parent.slice(*UpdateStatusService::ALLOWED_ATTRIBUTES).with_indifferent_access.compact + params[:text] = @text + UpdateStatusService.new.call(@parent, params) + destroy_status! + end + + alias handle_edit_post_once_before_save handle_edit_once_before_save + alias handle_edit_roar_once_before_save handle_edit_once_before_save + alias handle_edit_toot_once_before_save handle_edit_once_before_save + alias handle_edit_parent_once_before_save handle_edit_once_before_save + + def handle_mute_once_at_end(_) + return if author_of_parent? + + MuteStatusService.new.call(@account, @parent) + end + + alias handle_mute_post_once_at_end handle_mute_once_at_end + alias handle_mute_roar_once_at_end handle_mute_once_at_end + alias handle_mute_toot_once_at_end handle_mute_once_at_end + alias handle_mute_parent_once_at_end handle_mute_once_at_end + alias handle_hide_once_at_end handle_mute_once_at_end + alias handle_hide_post_once_at_end handle_mute_once_at_end + alias handle_hide_roar_once_at_end handle_mute_once_at_end + alias handle_hide_toot_once_at_end handle_mute_once_at_end + alias handle_hide_parent_once_at_end handle_mute_once_at_end + + def handle_unmute_once_at_end(_) + return if author_of_parent? + + @account.unmute_status!(@parent) + end + + alias handle_unmute_post_once_at_end handle_unmute_once_at_end + alias handle_unmute_roar_once_at_end handle_unmute_once_at_end + alias handle_unmute_toot_once_at_end handle_unmute_once_at_end + alias handle_unmute_parent_once_at_end handle_unmute_once_at_end + alias handle_unhide_once_at_end handle_unmute_once_at_end + alias handle_unhide_post_once_at_end handle_unmute_once_at_end + alias handle_unhide_roar_once_at_end handle_unmute_once_at_end + alias handle_unhide_toot_once_at_end handle_unmute_once_at_end + alias handle_unhide_parent_once_at_end handle_unmute_once_at_end + + def handle_mute_thread_once_at_end(_) + return if author_of_parent? + + MuteConversationService.new.call(@account, @conversation) + end + + alias handle_mute_conversation_once_at_end handle_mute_thread_once_at_end + alias handle_hide_thread_once_at_end handle_mute_thread_once_at_end + alias handle_hide_conversation_once_at_end handle_mute_thread_once_at_end + + def handle_unmute_thread_once_at_end(_) + return if author_of_parent? || @conversation.blank? + + @account.unmute_conversation!(@conversation) + end + + alias handle_unmute_conversation_once_at_end handle_unmute_thread_once_at_end + alias handle_unhide_thread_once_at_end handle_unmute_thread_once_at_end + alias handle_unhide_conversation_once_at_end handle_unmute_thread_once_at_end +end diff --git a/app/lib/command_tag/command/status_tools.rb b/app/lib/command_tag/command/status_tools.rb new file mode 100644 index 000000000..b2ddca422 --- /dev/null +++ b/app/lib/command_tag/command/status_tools.rb @@ -0,0 +1,239 @@ +# frozen_string_literal: true +module CommandTag::Command::StatusTools + def handle_boost_once_at_start(args) + return unless @parent.present? && StatusPolicy.new(@account, @parent).reblog? + + status = ReblogService.new.call( + @account, @parent, + visibility: @status.visibility, + spoiler_text: args.join(' ').presence || @status.spoiler_text + ) + end + + alias handle_reblog_at_start handle_boost_once_at_start + alias handle_rb_at_start handle_boost_once_at_start + alias handle_rt_at_start handle_boost_once_at_start + + def handle_article_before_save(args) + return unless author_of_status? && args.present? + + case args.shift.downcase + when 'title', 'name', 't' + status.title = args.join(' ') + when 'summary', 'abstract', 'cw', 'cn', 's', 'a' + @status.title = @status.spoiler_text if @status.title.blank? + @status.spoiler_text = args.join(' ') + end + end + + def handle_title_before_save(args) + args.unshift('title') + handle_article_before_save(args) + end + + def handle_summary_before_save(args) + args.unshift('summary') + handle_article_before_save(args) + end + + alias handle_abstract_before_save handle_summary_before_save + + def handle_visibility_before_save(args) + return unless author_of_status? && args[0].present? + + args[0] = read_visibility_from(args[0]) + return if args[0].blank? + + if args[1].blank? + @status.visibility = args[0].to_sym + elsif args[0] == @status.visibility.to_s + domains = args[1..-1].map { |domain| normalize_domain(domain) unless domain == '*' }.uniq.compact + @status.domain_permissions.where(domain: domains).destroy_all if domains.present? + elsif args[0] == 'cc' + expect_list = false + args[1..-1].uniq.each do |target| + if expect_list + expect_list = false + address_to_list(target) + elsif %w(list list:).include?(target.downcase) + expect_list = true + else + mention(resolve_mention(target)) + end + end + elsif args[0] == 'community' + @status.visibility = :public + @status.domain_permissions.create_or_update(domain: '*', visibility: :unlisted) + else + args[1..-1].flat_map(&:split).uniq.each do |domain| + domain = normalize_domain(domain) unless domain == '*' + @status.domain_permissions.create_or_update(domain: domain, visibility: args[0]) if domain.present? + end + end + end + + alias handle_v_before_save handle_visibility_before_save + alias handle_p_before_save handle_visibility_before_save + alias handle_privacy_before_save handle_visibility_before_save + + def handle_local_only_before_save(args) + @status.local_only = args.present? ? read_boolean_from(args[0]) : true + @status.originally_local_only = @status.local_only? + end + + def handle_federate_before_save(args) + @status.local_only = args.present? ? !read_boolean_from(args[0]) : false + @status.originally_local_only = @status.local_only? + end + + def handle_notify_before_save(args) + return if args[0].blank? + + @status.notify = read_boolean_from(args[0]) + end + + alias handle_notice_before_save handle_notify_before_save + + def handle_tags_before_save(args) + return if args.blank? + + cmd = args.shift.downcase + args.select! { |tag| tag =~ /\A(#{Tag::HASHTAG_NAME_RE})\z/i } + + case cmd + when 'add', 'a', '+' + ProcessHashtagsService.new.call(@status, args) + when 'del', 'remove', 'rm', 'r', 'd', '-' + RemoveHashtagsService.new.call(@status, args) + end + end + + def handle_tag_before_save(args) + args.unshift('add') + handle_tags_before_save(args) + end + + def handle_untag_before_save(args) + args.unshift('del') + handle_tags_before_save(args) + end + + def handle_delete_before_save(args) + unless args + RemovalWorker.perform_async(@parent.id, immediate: true) if author_of_parent? && status_text_blank? + return + end + + args.flat_map(&:split).uniq.each do |id| + if id.match?(/\A\d+\z/) + object = @account.statuses.find_by(id: id) + elsif id.start_with?('https://') + begin + object = ActivityPub::TagManager.instance.uri_to_resource(id, Status) + if object.blank? && ActivityPub::TagManager.instance.local_uri?(id) + id = Addressable::URI.parse(id)&.normalized_path&.sub(/\A.*\/([^\/]*)\/*/, '\1') + next unless id.present? && id.match?(/\A\d+\z/) + + object = find_status_or_create_stub(id) + end + rescue Addressable::URI::InvalidURIError + next + end + end + + next if object.blank? || object.account_id != @account.id + + RemovalWorker.perform_async(object.id, immediate: true, unpublished: true) + end + end + + alias handle_destroy_before_save handle_delete_before_save + alias handle_redraft_before_save handle_delete_before_save + + def handle_expires_before_save(args) + return if args.blank? + + @status.expires_at = Time.now.utc + to_datetime(args) + end + + alias handle_expires_in_before_save handle_expires_before_save + alias handle_delete_in_before_save handle_expires_before_save + alias handle_unpublish_in_before_save handle_expires_before_save + + def handle_publish_before_save(args) + return if args.blank? + + @status.published = false + @status.publish_at = Time.now.utc + to_datetime(args) + end + + alias handle_publish_in_before_save handle_publish_before_save + + private + + def resolve_mention(mention_text) + return unless (match = mention_text.match(Account::MENTION_RE)) + + username, domain = match[1].split('@') + domain = begin + if TagManager.instance.local_domain?(domain) + nil + else + TagManager.instance.normalize_domain(domain) + end + end + + Account.find_remote(username, domain) + end + + def mention(target_account) + return if target_account.blank? || target_account.mentions.where(status: @status).exists? + + target_account.mentions.create(status: @status, silent: true) + end + + def address_to_list(list_name) + return if list_name.blank? + + list_accounts = ListAccount.joins(:list).where(lists: { account: @account }).where('LOWER(lists.title) = ?', list_name.mb_chars.downcase).includes(:account).map(&:account) + list_accounts.each { |target_account| mention(target_account) } + end + + def find_status_or_create_stub(id) + status_params = { + id: id, + account: @account, + text: '(Deleted)', + local: true, + visibility: :public, + local_only: false, + published: false, + } + Status.where(id: id).first_or_create(status_params) + end + + def to_datetime(args) + total = 0.seconds + args.reject { |arg| arg.blank? || %w(in at , and).include?(arg) }.in_groups_of(2) { |i, unit| total += to_duration(i.to_i, unit) } + total + end + + def to_duration(amount, unit) + case unit + when nil, 's', 'sec', 'secs', 'second', 'seconds' + amount.seconds + when 'm', 'min', 'mins', 'minute', 'minutes' + amount.minutes + when 'h', 'hr', 'hrs', 'hour', 'hours' + amount.hours + when 'd', 'day', 'days' + amount.days + when 'w', 'wk', 'wks', 'week', 'weeks' + amount.weeks + when 'mo', 'mos', 'mn', 'mns', 'month', 'months' + amount.months + when 'y', 'yr', 'yrs', 'year', 'years' + amount.years + end + end +end diff --git a/app/lib/command_tag/command/text_tools.rb b/app/lib/command_tag/command/text_tools.rb new file mode 100644 index 000000000..6b37b66b7 --- /dev/null +++ b/app/lib/command_tag/command/text_tools.rb @@ -0,0 +1,89 @@ +# frozen_string_literal: true + +module CommandTag::Command::TextTools + def handle_code_at_start(args) + return if args.count < 2 + + name = normalize(args[0]) + value = args.last.presence || '' + @vars[name] = case @status.content_type + when 'text/markdown' + ["```\n#{value}\n```"] + when 'text/html' + ["<pre><code>#{html_encode(value).gsub("\n", '<br/>')}</code></pre>"] + else + ["----------\n#{value}\n----------"] + end + end + + def handle_code_with_return(args) + return if args.count > 1 + + value = args.last.presence || '' + case @status.content_type + when 'text/markdown' + ["```\n#{value}\n```"] + when 'text/html' + ["<pre><code>#{html_encode(value).gsub("\n", '<br/>')}</code></pre>"] + else + ["----------\n#{value}\n----------"] + end + end + + def handle_prepend_before_save(args) + args.each { |arg| @text = "#{arg}\n#{text}" } + end + + def handle_append_before_save(args) + args.each { |arg| @text << "\n#{arg}" } + end + + def handle_replace_before_save(args) + @text.gsub!(args[0], args[1] || '') + end + + alias handle_sub_before_save handle_replace_before_save + + def handle_regex_replace_before_save(args) + flags = normalize(args[2]) + re_opts = (flags.include?('i') ? Regexp::IGNORECASE : 0) + re_opts |= (flags.include?('x') ? Regexp::EXTENDED : 0) + re_opts |= (flags.include?('m') ? Regexp::MULTILINE : 0) + + @text.gsub!(Regexp.new(args[0], re_opts), args[1] || '') + end + + alias handle_resub_before_save handle_replace_before_save + alias handle_regex_sub_before_save handle_replace_before_save + + def handle_keysmash_with_return(args) + keyboard = [ + 'asdf', 'jkl;', + 'gh', "'", + 'we', 'io', + 'r', 'u', + 'cv', 'nm', + 't', 'x', ',', + 'q', 'z', + 'y', 'b', + 'p', '.', + '[', ']' + ] + + min_size = [[5, args[1].to_i].max, 100].min + max_size = [args[0].to_i, 100].min + max_size = 33 unless max_size.positive? + + min_size, max_size = [max_size, min_size] if min_size > max_size + + chunk = rand(min_size..max_size).times.map do + keyboard[(keyboard.size * (rand**3)).floor].split('').sample + end + + chunk.join + end + + def transform_keysmash_template_return(_, args) + handle_keysmash_with_return([args[0], args[2]]) + end +end diff --git a/app/lib/command_tag/command/variables.rb b/app/lib/command_tag/command/variables.rb new file mode 100644 index 000000000..6ba32ea41 --- /dev/null +++ b/app/lib/command_tag/command/variables.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module CommandTag::Command::Variables + def handle_000_variables_startup + @vars.merge!(persistent_vars_from(@account.metadata.fields)) if @account.metadata.present? + end + + def handle_999_variables_shutdown + @account.metadata.update!(fields: nonpersistent_vars_from(@account.metadata.fields).merge(persistent_vars_from(@vars))) + end + + def handle_set_at_start(args) + return if args.blank? + + args[0] = normalize(args[0]) + + case args.count + when 1 + @vars.delete(args[0]) + else + @vars[args[0]] = args[1..-1] + end + end + + def do_unset_at_start(args) + args.each do |arg| + @vars.delete(normalize(arg)) + end + end + + private + + def persistent_vars_from(vars) + vars.select { |key, value| key.start_with?('persist:') && value.present? && value.is_a?(Array) } + end + + def nonpersistent_vars_from(vars) + vars.reject { |key, value| key.start_with?('persist:') || value.blank? } + end +end diff --git a/app/lib/command_tag/commands.rb b/app/lib/command_tag/commands.rb new file mode 100644 index 000000000..f27486427 --- /dev/null +++ b/app/lib/command_tag/commands.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +Dir[File.join(__dir__, 'command', '*.rb')].sort.each { |file| require file } + +module CommandTag::Commands + def self.included(base) + CommandTag::Command.constants.map(&CommandTag::Command.method(:const_get)).grep(Module) do |mod| + base.include(mod) + end + end +end diff --git a/app/lib/command_tag/processor.rb b/app/lib/command_tag/processor.rb new file mode 100644 index 000000000..77be29eba --- /dev/null +++ b/app/lib/command_tag/processor.rb @@ -0,0 +1,343 @@ +# frozen_string_literal: true + +# .~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~. # +################### Cthulhu Code! ################### +# `~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~` # +# - Interprets and executes user input. THIS CAN BE VERY DANGEROUS! # +# - Has a high complexity level and needs tests. # +# - May destroy objects passed to it. # +# - Incurs a high performance penalty. # +# # +############################################################################### + +require_relative 'commands' + +class CommandTag::Break < Mastodon::Error + def initialize(msg = 'A handler stopped execution.') + super + end +end + +class CommandTag::Processor + include Redisable + include ImgProxyHelper + include CommandTag::Commands + + MENTIONS_OR_HASHTAGS_RE = /(?:(?:#{Account::MENTION_RE}|#{Tag::HASHTAG_RE})\s*)+/.freeze + PARSEABLE_RE = /^\s*(?:#{MENTIONS_OR_HASHTAGS_RE})?#!|%%.+?%%/.freeze + STATEMENT_RE = /^\s*#!\s*[^\n]+ (?:start|begin|do)$.*?\n\s*#!\s*(?:end|stop|done)\s*$|^\s*#!\s*.*?\s*$/im.freeze + STATEMENT_PARSE_RE = /'([^']*)'|"([^"]*)"|(\S+)|\s+(?:start|begin|do)\s*$\n+(.*)\n\s*#!\s*(?:end|stop|done)\s*\z/im.freeze + TEMPLATE_RE = /%%\s*(\S+.*?)\s*%%/.freeze + ESCAPE_MAP = { + '\n' => "\n", + '\r' => "\r", + '\t' => "\t", + '\\\\' => '\\', + '\%' => '%', + }.freeze + + def initialize(account, status) + @account = account + @status = status + @parent = status.thread + @conversation = status.conversation + @text = status.text + @run_once = Set[] + @vars = { 'statement_uuid' => [nil] } + @statements = {} + + return unless @account.present? && @account.local? && @status.present? + end + + def process! + reset_status_caches + all_handlers!(:startup) + + unless @text.match?(PARSEABLE_RE) + process_inline_images! + @status.save! + return + end + + @text = parse_statements_from!(@text, @statements) + + execute_statements(:at_start) + execute_statements(:with_return, true) + @text = replace_templates(@text) + execute_statements(:before_save) + + if status_text_blank? + execute_statements(:when_blank) + + unless (@status.published? && !@status.edited.zero?) || @text.present? + execute_statements(:before_destroy) + @status.update(published: false) + @status.destroy + execute_statements(:after_destroy) + end + elsif @status.destroyed? + execute_statements(:after_destroy) + else + @status.text = @text + process_inline_images! + if @status.save + execute_statements(:after_save) + else + execute_statements(:after_save_fail) + end + end + + execute_statements(:at_end) + all_handlers!(:shutdown) + rescue CommandTag::Break + nil + rescue StandardError + @status.update(published: false) + @status.destroy + raise + ensure + reset_status_caches + end + + private + + def all_handlers!(affix) + self.class.instance_methods.grep(/\Ahandle_\w+_#{affix}\z/).sort.each do |name| + public_send(name) + end + end + + # Calls an arbitary public method (if it exists) on a given value and returns the result. + def transform_using(name, value, args = []) + respond_to?(name) ? public_send(name, value, args) : value + end + + # Moves command tags placed after hashtags and mentions to their own line. + def prepare_input(text) + text.gsub(/\r\n|\n\r|\r/, "\n").gsub(/^\s*(#{MENTIONS_OR_HASHTAGS_RE})#!/, "\\1\n#!") + end + + # Translates %%...%% templates. + def replace_templates(text) + text.gsub(TEMPLATE_RE) do + template = unescape_literals(Regexp.last_match(1)) + next if template.blank? + next template[1..-2] if template.match?(/\A'.*'\z/) + + template = template.match?(/\A".*"\z/) ? template[1..-2] : "\#{#{template}}" + template.gsub(/#\{\s*(.*?)\s*\}/) do + next if Regexp.last_match(1).blank? + + parts = Regexp.last_match(1).scan(/'([^']*)'|"([^"]*)"|(\S+)/).flatten.compact + name = normalize(parts[0]) + separator = "\n" + + if parts.count > 2 + if %w(: by: with: using: sep: separator: delim: delimiter:).include?(parts[-2].downcase) + separator = parts[-1] + parts = parts[0..-3] + elsif !parts[-1].match?(/\A[-+]?[0-9]+\z/) + separator = parts[-1] + parts.pop + end + end + + index = to_integer(parts[1]) + str_start = to_integer(parts[2]) + str_end = to_integer(parts[3]) + + str_start, str_end = [str_end, str_start] if str_start > str_end + + old_value = (['all', '[]'].include?(parts[1]) ? var(name).join(separator) : var(name)[index].to_s) + name = name.gsub(/[^\w_]+/, '_') + new_value = transform_using("transform_#{name}_template_return", old_value, [index, str_start, str_end]) + next new_value if new_value != old_value + + new_value = transform_using("transform_#{name}_template_value", new_value, [index, str_start, str_end]) + (str_end - str_start).zero? ? new_value : new_value[str_start..str_end] + end + end.rstrip + end + + # Parses statements from text and merges them into statement queues. + # Mutates statement queues hash! + def parse_statements_from!(text, statement_queues) + @run_once.clear + + text = prepare_input(text) + text.gsub!(STATEMENT_RE) do + statement = unescape_literals(Regexp.last_match(0).strip[2..-1]) + next if statement.blank? + + statement_array = statement.scan(STATEMENT_PARSE_RE).flatten.compact.map { |arg| arg.gsub('\#!', '#!') } + statement_array[0] = statement_array[0].strip.tr(':.\- ', '_').gsub(/__+/, '_').downcase + next unless statement_array[0].match?(/\A[\w_]+\z/) + + statement_array[-1].rstrip! if statement_array.count > 1 + add_statement_handlers_for!(statement_array, statement_queues) + end + + @run_once.clear + text + end + + # Yields all possible handler names for a command. + def potential_handlers_for(name) + ['_once', ''].each_with_index do |count_affix, index| + %w(at_start with_return when_blank at_end).each do |when_affix| + yield ["#{count_affix}_#{when_affix}", "handle_#{name}#{count_affix}_#{when_affix}", index.zero?] + end + + %w(destroy save postprocess save_fail).each do |event_affix| + %w(before after).each do |when_affix| + yield ["#{count_affix}_#{when_affix}_#{event_affix}", "handle_#{name}#{count_affix}_#{when_affix}_#{event_affix}", index.zero?] + end + end + end + end + + # Expands a statement to a handler method call, arguments, and template UUID for each handler affix. + # Mutates statement queues hash! + def add_statement_handlers_for!(statement_array, statement_queues = {}) + statement_uuid = SecureRandom.uuid + + potential_handlers_for(statement_array[0]) do |when_affix, handler, once| + if !(once && @run_once.include?(handler)) && respond_to?(handler) + statement_queues[when_affix] ||= [] + statement_queues[when_affix] << [handler, statement_array[1..-1], statement_uuid] + @run_once << handler if once + end + end + + # Template for statement return value. + "%% statement:#{statement_uuid} all %%" + end + + # Calls all handlers for a queue of statements in order. + def execute_statements(event, with_return = false, statements: nil) + statements = @statements if statements.blank? + + ["_#{event}", "_once_#{event}"].each do |when_affix| + next if statements[when_affix].blank? + + statements[when_affix].each do |handler, arguments, uuid| + @vars['statement_uuid'][0] = uuid + if with_return + @vars["statement:#{uuid}"] = [public_send(handler, arguments)] + else + public_send(handler, arguments) + end + end + end + end + + # Expire cached statuses after potentially updating them. + def reset_status_caches(statuses = nil) + statuses = [@status, @parent] if statuses.blank? + statuses.each do |status| + next unless @account.id == status&.account_id + + Rails.cache.delete_matched("statuses/#{status.id}-*") + Rails.cache.delete("statuses/#{status.id}") + Rails.cache.delete(status) + Rails.cache.delete_matched("format:#{status.id}:*") + redis.zremrangebyscore("spam_check:#{status.account.id}", status.id, status.id) + end + end + + def author_of_status? + @account.id == @status.account_id + end + + def author_of_parent? + @account.id == @parent&.account_id + end + + def status_text_blank? + @text.blank? || @text.gsub(MENTIONS_OR_HASHTAGS_RE, '').strip.blank? + end + + def destroy_status! + return if @status.destroyed? + + @status.update(published: false) + @status.destroy + end + + def replace_status!(new_status) + return if new_status.blank? + + destroy_status! + @status = new_status + end + + def normalize(text) + text.to_s.strip.downcase + end + + def to_integer(text) + text&.strip.to_i + end + + def unescape_literals(text) + ESCAPE_MAP.each { |escaped, unescaped| text.gsub!(escaped, unescaped) } + text + end + + def html_encode(text) + (@html_entities ||= HTMLEntities.new).encode(text) + end + + def var(name) + @vars[name].presence || [] + end + + def read_visibility_from(arg) + return if arg.strip.blank? + + arg = case arg.strip + when 'p', 'pu', 'all', 'world' + 'public' + when 'u', 'ul' + 'unlisted' + when 'f', 'follower', 'followers', 'packmates', 'follower-only', 'followers-only', 'packmates-only' + 'private' + when 'd', 'dm', 'pm', 'directmessage' + 'direct' + when 'default', 'reset' + @account.user.setting_default_privacy + when 'to', 'allow', 'allow-from', 'from' + 'cc' + when 'm', 'l', 'mp', 'monsterpit', 'local' + 'community' + else + arg.strip + end + + %w(public unlisted private limited direct cc community).include?(arg) ? arg : nil + end + + def read_falsy_from(arg) + %w(f n false no off disable).include?(arg) + end + + def read_truthy_from(arg) + %w(t y true yes on enable).include?(arg) + end + + def read_boolean_from(arg) + arg.present? && (read_truthy_from(arg) || !read_falsy_from(arg)) + end + + def normalize_domain(domain) + return if domain&.strip.blank? || !domain.include?('.') + + domain.split('.').map(&:strip).reject(&:blank?).join('.').downcase + end + + def federating_with_domain?(domain) + return false if domain.blank? + + DomainAllow.where(domain: domain).exists? || Account.where(domain: domain, suspended_at: nil).exists? + end +end diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb index 3c1f8d6e2..665869b26 100644 --- a/app/lib/feed_manager.rb +++ b/app/lib/feed_manager.rb @@ -1,18 +1,17 @@ # frozen_string_literal: true require 'singleton' - class FeedManager include Singleton include Redisable # Maximum number of items stored in a single feed - MAX_ITEMS = 400 + MAX_ITEMS = 1000 # Number of items in the feed since last reblog of status # before the new reblog will be inserted. Must be <= MAX_ITEMS # or the tracking sets will grow forever - REBLOG_FALLOFF = 40 + REBLOG_FALLOFF = 50 # Execute block for every active account # @yield [Account] @@ -40,9 +39,9 @@ class FeedManager def filter?(timeline_type, status, receiver) case timeline_type when :home - filter_from_home?(status, receiver.id, build_crutches(receiver.id, [status])) + filter_from_home?(status, receiver.id, build_crutches(receiver.id, [status]), receiver.user&.filters_unknown?) when :list - filter_from_list?(status, receiver) || filter_from_home?(status, receiver.account_id, build_crutches(receiver.account_id, [status])) + filter_from_list?(status, receiver) || filter_from_home?(status, receiver.account_id, build_crutches(receiver.account_id, [status]), receiver.account.user&.filters_unknown?) when :mentions filter_from_mentions?(status, receiver.id) when :direct @@ -57,7 +56,7 @@ class FeedManager # @param [Status] status # @return [Boolean] def push_to_home(account, status) - return false unless add_to_feed(:home, account.id, status, account.user&.aggregates_reblogs?) + return false unless add_to_feed(:home, account.id, status, account.user&.home_reblogs?) trim(:home, account.id) PushUpdateWorker.perform_async(account.id, status.id, "timeline:#{account.id}") if push_update_required?("timeline:#{account.id}") @@ -68,8 +67,8 @@ class FeedManager # @param [Account] account # @param [Status] status # @return [Boolean] - def unpush_from_home(account, status) - return false unless remove_from_feed(:home, account.id, status, account.user&.aggregates_reblogs?) + def unpush_from_home(account, status, include_reblogs_list = true) + return false unless remove_from_feed(:home, account.id, status, include_reblogs_list) redis.publish("timeline:#{account.id}", Oj.dump(event: :delete, payload: status.id.to_s)) true @@ -80,7 +79,8 @@ class FeedManager # @param [Status] status # @return [Boolean] def push_to_list(list, status) - return false if filter_from_list?(status, list) || !add_to_feed(:list, list.id, status, list.account.user&.aggregates_reblogs?) + return false if filter_from_list?(status, list) + return false unless add_to_feed(:list, list.id, status, list.reblogs?) trim(:list, list.id) PushUpdateWorker.perform_async(list.account_id, status.id, "timeline:list:#{list.id}") if push_update_required?("timeline:list:#{list.id}") @@ -92,7 +92,7 @@ class FeedManager # @param [Status] status # @return [Boolean] def unpush_from_list(list, status) - return false unless remove_from_feed(:list, list.id, status, list.account.user&.aggregates_reblogs?) + return false unless remove_from_feed(:list, list.id, status) redis.publish("timeline:list:#{list.id}", Oj.dump(event: :delete, payload: status.id.to_s)) true @@ -121,13 +121,33 @@ class FeedManager true end + def unpush_status(account, status) + return if account.blank? || status.blank? + + unpush_from_home(account, status) + unpush_from_direct(account, status) if status.direct_visibility? + + account.lists_for_local_distribution.select(:id, :account_id).each do |list| + unpush_from_list(list, status) + end + end + + def unpush_conversation(account, conversation) + return if account.blank? || conversation.blank? + + conversation.statuses.reorder(nil).find_each do |status| + unpush_status(account, status) + end + end + # Fill a home feed with an account's statuses # @param [Account] from_account # @param [Account] into_account # @return [void] def merge_into_home(from_account, into_account) timeline_key = key(:home, into_account.id) - aggregate = into_account.user&.aggregates_reblogs? + reblogs = into_account.user&.home_reblogs? + no_unknown = into_account.user&.filters_unknown? query = from_account.statuses.where(visibility: [:public, :unlisted, :private]).includes(:preloadable_poll, reblog: :account).limit(FeedManager::MAX_ITEMS / 4) if redis.zcard(timeline_key) >= FeedManager::MAX_ITEMS / 4 @@ -139,9 +159,9 @@ class FeedManager crutches = build_crutches(into_account.id, statuses) statuses.each do |status| - next if filter_from_home?(status, into_account.id, crutches) + next if filter_from_home?(status, into_account.id, crutches, no_unknown) - add_to_feed(:home, into_account.id, status, aggregate) + add_to_feed(:home, into_account.id, status, reblogs) end trim(:home, into_account.id) @@ -153,7 +173,8 @@ class FeedManager # @return [void] def merge_into_list(from_account, list) timeline_key = key(:list, list.id) - aggregate = list.account.user&.aggregates_reblogs? + reblogs = list.account.user&.home_reblogs? + no_unknown = list.account.user&.filters_unknown? query = from_account.statuses.where(visibility: [:public, :unlisted, :private]).includes(:preloadable_poll, reblog: :account).limit(FeedManager::MAX_ITEMS / 4) if redis.zcard(timeline_key) >= FeedManager::MAX_ITEMS / 4 @@ -165,9 +186,9 @@ class FeedManager crutches = build_crutches(list.account_id, statuses) statuses.each do |status| - next if filter_from_home?(status, list.account_id, crutches) || filter_from_list?(status, list) + next if filter_from_home?(status, list.account_id, crutches, no_unknown) || filter_from_list?(status, list) - add_to_feed(:list, list.id, status, aggregate) + add_to_feed(:list, list.id, status, reblogs) end trim(:list, list.id) @@ -182,7 +203,7 @@ class FeedManager oldest_home_score = redis.zrange(timeline_key, 0, 0, with_scores: true)&.first&.last&.to_i || 0 from_account.statuses.select('id, reblog_of_id').where('id > ?', oldest_home_score).reorder(nil).find_each do |status| - remove_from_feed(:home, into_account.id, status, into_account.user&.aggregates_reblogs?) + remove_from_feed(:home, into_account.id, status) end end @@ -195,7 +216,7 @@ class FeedManager oldest_list_score = redis.zrange(timeline_key, 0, 0, with_scores: true)&.first&.last&.to_i || 0 from_account.statuses.select('id, reblog_of_id').where('id > ?', oldest_list_score).reorder(nil).find_each do |status| - remove_from_feed(:list, list.id, status, list.account.user&.aggregates_reblogs?) + remove_from_feed(:list, list.id, status, !list.reblogs?) end end @@ -219,16 +240,63 @@ class FeedManager end end + # Clear all reblogs from a home feed + # @param [Account] account + # @return [void] + def clear_reblogs_from_home(account) + timeline_key = key(:home, account.id) + timeline_status_ids = redis.zrange(timeline_key, 0, -1) + + Status.reblogs.joins(:reblog).where(reblogs_statuses: { local: false }).where(id: timeline_status_ids).find_each do |status| + unpush_from_home(account, status, false) + end + end + + # Populate list feeds of account from scratch + # @param [Account] account + # @return [void] + def populate_lists(account) + limit = FeedManager::MAX_ITEMS / 2 + + account.owned_lists.includes(:accounts) do |list| + timeline_key = key(:list, list.id) + + list.accounts.includes(:account_stat).find_each do |target_account| + if redis.zcard(timeline_key) >= limit + oldest_home_score = redis.zrange(timeline_key, 0, 0, with_scores: true).first.last.to_i + last_status_score = Mastodon::Snowflake.id_at(account.last_status_at) + + # If the feed is full and this account has not posted more recently + # than the last item on the feed, then we can skip the whole account + # because none of its statuses would stay on the feed anyway + next if last_status_score < oldest_home_score + end + + statuses = target_account.statuses.published.without_reblogs.where(visibility: [:public, :unlisted, :private]).includes(:mentions, :preloadable_poll).limit(limit) + crutches = build_crutches(account.id, statuses) + + statuses.each do |status| + next if filter_from_list?(status, account.id) || filter_from_home?(status, account.id, crutches, account.user&.filters_unknown?) + + add_to_feed(:list, list.id, status, list.reblogs?) + end + + trim(:list, list.id) + end + end + end + # Populate home feed of account from scratch # @param [Account] account # @return [void] def populate_home(account) limit = FeedManager::MAX_ITEMS / 2 - aggregate = account.user&.aggregates_reblogs? + reblogs = account.user&.home_reblogs? + no_unknown = account.user&.filters_unknown? timeline_key = key(:home, account.id) account.statuses.limit(limit).each do |status| - add_to_feed(:home, account.id, status, aggregate) + add_to_feed(:home, account.id, status, reblogs) end account.following.includes(:account_stat).find_each do |target_account| @@ -242,13 +310,13 @@ class FeedManager next if last_status_score < oldest_home_score end - statuses = target_account.statuses.where(visibility: [:public, :unlisted, :private]).includes(:preloadable_poll, reblog: :account).limit(limit) + statuses = target_account.statuses.published.where(visibility: [:public, :unlisted, :private]).includes(:mentions, :preloadable_poll, reblog: [:account, :mentions]).limit(limit) crutches = build_crutches(account.id, statuses) statuses.each do |status| - next if filter_from_home?(status, account.id, crutches) + next if filter_from_home?(status, account.id, crutches, no_unknown) - add_to_feed(:home, account.id, status, aggregate) + add_to_feed(:home, account.id, status, reblogs, false) end trim(:home, account.id) @@ -270,6 +338,7 @@ class FeedManager statuses.each do |status| next if filter_from_direct?(status, account) + added += 1 if add_to_feed(:direct, account.id, status) end @@ -334,36 +403,55 @@ class FeedManager # @param [Integer] receiver_id # @param [Hash] crutches # @return [Boolean] - def filter_from_home?(status, receiver_id, crutches) + def filter_from_home?(status, receiver_id, crutches, followed_only = false) return false if receiver_id == status.account_id + return true if !status.published? || crutches[:hiding_thread][status.conversation_id] return true if status.reply? && (status.in_reply_to_id.nil? || status.in_reply_to_account_id.nil?) return true if phrase_filtered?(status, receiver_id, :home) check_for_blocks = crutches[:active_mentions][status.id] || [] check_for_blocks.concat([status.account_id]) + check_for_blocks.concat([status.in_reply_to_account_id]) if status.reply? if status.reblog? check_for_blocks.concat([status.reblog.account_id]) check_for_blocks.concat(crutches[:active_mentions][status.reblog_of_id] || []) + check_for_blocks.concat([status.reblog.in_reply_to_account_id]) if status.reblog.reply? end return true if check_for_blocks.any? { |target_account_id| crutches[:blocking][target_account_id] || crutches[:muting][target_account_id] } - if status.reply? && !status.in_reply_to_account_id.nil? # Filter out if it's a reply - should_filter = !crutches[:following][status.in_reply_to_account_id] # and I'm not following the person it's a reply to - should_filter &&= receiver_id != status.in_reply_to_account_id # and it's not a reply to me - should_filter &&= status.account_id != status.in_reply_to_account_id # and it's not a self-reply + # Filter if... + if status.reply? # ...it's a reply and... + # ...you're not following the participants... + should_filter = (status.mentions.pluck(:account_id) - crutches[:following].keys).present? + # ...and the author isn't replying to you... + should_filter &&= receiver_id != status.in_reply_to_account_id return !!should_filter - elsif status.reblog? # Filter out a reblog - should_filter = crutches[:hiding_reblogs][status.account_id] # if the reblogger's reblogs are suppressed - should_filter ||= crutches[:blocked_by][status.reblog.account_id] # or if the author of the reblogged status is blocking me - should_filter ||= crutches[:domain_blocking][status.reblog.account.domain] # or the author's domain is blocked + elsif status.reblog? # ...it's a boost and... + # ...you don't follow the OP and they're non-local or they're silenced... + should_filter = (followed_only || status.reblog.account.silenced?) && !crutches[:following][status.reblog.account_id] + + # ..or you're hiding boosts from them... + should_filter ||= crutches[:hiding_reblogs][status.account_id] + # ...or they're blocking you... + should_filter ||= crutches[:blocked_by][status.reblog.account_id] + # ...or you're blocking their domain... + should_filter ||= crutches[:domain_blocking][status.reblog.account.domain] + + # ...or it's a reply... + if !(should_filter || status.reblog.in_reply_to_account_id.nil?) && status.reblog.reply? + # ...and you don't follow the participants... + should_filter ||= (status.reblog.mentions.pluck(:account_id) - crutches[:following].keys).present? + # ...and the author isn't replying to you... + should_filter &&= receiver_id != status.in_reply_to_account_id + end return !!should_filter end - false + !crutches[:following][status.account_id] end # Check if status should not be added to the mentions feed @@ -381,18 +469,29 @@ class FeedManager check_for_blocks = status.active_mentions.pluck(:account_id) check_for_blocks.concat([status.in_reply_to_account]) if status.reply? && !status.in_reply_to_account_id.nil? - should_filter = blocks_or_mutes?(receiver_id, check_for_blocks, :mentions) # Filter if it's from someone I blocked, in reply to someone I blocked, or mentioning someone I blocked (or muted) - should_filter ||= (status.account.silenced? && !Follow.where(account_id: receiver_id, target_account_id: status.account_id).exists?) # of if the account is silenced and I'm not following them + should_filter = blocks_or_mutes?(receiver_id, check_for_blocks, :mentions) + should_filter ||= (status.account.silenced? && !relationship_exists?(receiver_id, status.account_id)) should_filter end + def following?(account_id, target_account_id) + Follow.where(account_id: account_id, target_account_id: target_account_id).exists? + end + + def relationship_exists?(account_id, target_account_id) + Follow.where(account_id: account_id, target_account_id: target_account_id) + .or(Follow.where(account_id: target_account_id, target_account_id: account_id)) + .exists? + end + # Check if status should not be added to the linear direct message feed # @param [Status] status # @param [Integer] receiver_id # @return [Boolean] def filter_from_direct?(status, receiver_id) return false if receiver_id == status.account_id + filter_from_mentions?(status, receiver_id) end @@ -401,6 +500,9 @@ class FeedManager # @param [List] list # @return [Boolean] def filter_from_list?(status, list) + return true if (list.reblogs? && !status.reblog?) || (!list.reblogs? && status.reblog?) + return true if status.reblog? ? status.reblog.account_id == list.account_id : status.account_id == list.account_id + if status.reply? && status.in_reply_to_account_id != status.account_id should_filter = status.in_reply_to_account_id != list.account_id should_filter &&= !list.show_all_replies? @@ -455,13 +557,19 @@ class FeedManager # @param [Symbol] timeline_type # @param [Integer] account_id # @param [Status] status - # @param [Boolean] aggregate_reblogs + # @param [Boolean] home_reblogs + # @param [Boolean] stream # @return [Boolean] - def add_to_feed(timeline_type, account_id, status, aggregate_reblogs = true) + def add_to_feed(timeline_type, account_id, status, home_reblogs = true, stream = true) timeline_key = key(timeline_type, account_id) reblog_key = key(timeline_type, account_id, 'reblogs') - if status.reblog? && (aggregate_reblogs.nil? || aggregate_reblogs) + if status.reblog? + add_to_reblogs(account_id, status, stream) if timeline_type == :home + return false unless home_reblogs || (timeline_type == :home && (status.reblog.local? || following?(account_id, status.reblog.account_id))) + end + + if status.reblog? # If the original status or a reblog of it is within # REBLOG_FALLOFF statuses from the top, do not re-insert it into # the feed @@ -493,6 +601,8 @@ class FeedManager redis.zadd(timeline_key, status.id, status.id) end + add_to_reblogs(account_id, status, stream) if timeline_type == :home && status.reblog? + true end @@ -503,13 +613,15 @@ class FeedManager # @param [Symbol] timeline_type # @param [Integer] account_id # @param [Status] status - # @param [Boolean] aggregate_reblogs + # @param [Boolean] include_reblogs_list # @return [Boolean] - def remove_from_feed(timeline_type, account_id, status, aggregate_reblogs = true) + def remove_from_feed(timeline_type, account_id, status, include_reblogs_list = true) timeline_key = key(timeline_type, account_id) reblog_key = key(timeline_type, account_id, 'reblogs') - if status.reblog? && (aggregate_reblogs.nil? || aggregate_reblogs) + remove_from_reblogs(account_id, status) if include_reblogs_list && timeline_type == :home && status.reblog? + + if status.reblog? # 1. If the reblogging status is not in the feed, stop. status_rank = redis.zrevrank(timeline_key, status.id) return false if status_rank.nil? @@ -547,27 +659,43 @@ class FeedManager def build_crutches(receiver_id, statuses) crutches = {} - crutches[:active_mentions] = Mention.active.where(status_id: statuses.flat_map { |s| [s.id, s.reblog_of_id] }.compact).pluck(:status_id, :account_id).each_with_object({}) { |(id, account_id), mapping| (mapping[id] ||= []).push(account_id) } - - check_for_blocks = statuses.flat_map do |s| - arr = crutches[:active_mentions][s.id] || [] - arr.concat([s.account_id]) - - if s.reblog? - arr.concat([s.reblog.account_id]) - arr.concat(crutches[:active_mentions][s.reblog_of_id] || []) - end + mentions = Mention.active.where(status_id: statuses.flat_map { |s| [s.id, s.reblog_of_id] }.compact).pluck(:status_id, :account_id) + participants = statuses.flat_map { |s| [s.account_id, s.in_reply_to_account_id, s.reblog&.account_id, s.reblog&.in_reply_to_account_id].compact } | mentions.map { |m| m[1] } - arr - end + crutches[:active_mentions] = mentions.each_with_object({}) { |(id, account_id), mapping| (mapping[id] ||= []).push(account_id) } - crutches[:following] = Follow.where(account_id: receiver_id, target_account_id: statuses.map(&:in_reply_to_account_id).compact).pluck(:target_account_id).each_with_object({}) { |id, mapping| mapping[id] = true } + crutches[:following] = Follow.where(account_id: receiver_id, target_account_id: participants).pluck(:target_account_id).each_with_object({}) { |id, mapping| mapping[id] = true } crutches[:hiding_reblogs] = Follow.where(account_id: receiver_id, target_account_id: statuses.map { |s| s.account_id if s.reblog? }.compact, show_reblogs: false).pluck(:target_account_id).each_with_object({}) { |id, mapping| mapping[id] = true } - crutches[:blocking] = Block.where(account_id: receiver_id, target_account_id: check_for_blocks).pluck(:target_account_id).each_with_object({}) { |id, mapping| mapping[id] = true } - crutches[:muting] = Mute.where(account_id: receiver_id, target_account_id: check_for_blocks).pluck(:target_account_id).each_with_object({}) { |id, mapping| mapping[id] = true } + crutches[:blocking] = Block.where(account_id: receiver_id, target_account_id: participants).pluck(:target_account_id).each_with_object({}) { |id, mapping| mapping[id] = true } + crutches[:muting] = Mute.where(account_id: receiver_id, target_account_id: participants).pluck(:target_account_id).each_with_object({}) { |id, mapping| mapping[id] = true } crutches[:domain_blocking] = AccountDomainBlock.where(account_id: receiver_id, domain: statuses.map { |s| s.reblog&.account&.domain }.compact).pluck(:domain).each_with_object({}) { |domain, mapping| mapping[domain] = true } crutches[:blocked_by] = Block.where(target_account_id: receiver_id, account_id: statuses.map { |s| s.reblog&.account_id }.compact).pluck(:account_id).each_with_object({}) { |id, mapping| mapping[id] = true } + crutches[:hiding_thread] = ConversationMute.where(account_id: receiver_id, conversation_id: statuses.map(&:conversation_id).compact).pluck(:conversation_id).each_with_object({}) { |id, mapping| mapping[id] = true } crutches end + + def find_or_create_reblogs_list(account_id) + List.find_or_create_by!(account_id: account_id, reblogs: true) do |list| + list.title = I18n.t('accounts.reblogs') + list.replies_policy = :no_replies + end + end + + def add_to_reblogs(account_id, status, stream = true) + reblogs_list_id = find_or_create_reblogs_list(account_id).id + return unless add_to_feed(:list, reblogs_list_id, status) + + trim(:list, reblogs_list_id) + return unless stream && push_update_required?("timeline:list:#{reblogs_list_id}") + + PushUpdateWorker.perform_async(account_id, status.id, "timeline:list:#{reblogs_list_id}") + end + + def remove_from_reblogs(account_id, status) + reblogs_list_id = find_or_create_reblogs_list(account_id).id + return unless remove_from_feed(:list, reblogs_list_id, status) + + redis.publish("timeline:list:#{reblogs_list_id}", Oj.dump(event: :delete, payload: status.id.to_s)) + end end diff --git a/app/lib/formatter.rb b/app/lib/formatter.rb index e7bb0743d..e0c1e94db 100644 --- a/app/lib/formatter.rb +++ b/app/lib/formatter.rb @@ -26,6 +26,7 @@ class HTMLRenderer < Redcarpet::Render::HTML end end +# rubocop:disable Metrics/ClassLength class Formatter include Singleton include RoutingHelper @@ -33,50 +34,95 @@ class Formatter include ActionView::Helpers::TextHelper def format(status, **options) - if status.reblog? - prepend_reblog = status.reblog.account.acct - status = status.proper - else - prepend_reblog = false + Rails.cache.fetch(formatter_cache_key(status, options), expires_in: 1.hour) do + uncached_format(status, options) end + end - raw_content = status.text + def uncached_format(status, options) + summary = nil + raw_content = status.proper.text + summary_mode = false + + if status.title.present? + summary = status.spoiler_text.presence || status.text + summary_mode = !options[:article_content] + raw_content = summary_mode ? summary : status.text + end if options[:inline_poll_options] && status.preloadable_poll raw_content = raw_content + "\n\n" + status.preloadable_poll.options.map { |title| "[ ] #{title}" }.join("\n") end return '' if raw_content.blank? + return format_remote_content(raw_content, status.emojis, summary: summary, **options) unless status.local? - unless status.local? - html = reformat(raw_content) - html = encode_custom_emojis(html, status.emojis, options[:autoplay]) if options[:custom_emojify] - return html.html_safe # rubocop:disable Rails/OutputSafety + if status.reblog? + html = "🔁 @#{status.reblog.account.acct}\n🔗 #{ActivityPub::TagManager.instance.url_for(status.reblog)}" + html += "\nℹ️ #{status.reblog.spoiler_text}" if status.reblog.spoiler_text.present? + else + html = raw_content end - linkable_accounts = status.active_mentions.map(&:account) + html = "📄 #{html}" if summary_mode + return html if options[:plaintext] + + linkable_accounts = status.mentions.map(&:account) linkable_accounts << status.account - html = raw_content - html = "RT @#{prepend_reblog} #{html}" if prepend_reblog - html = format_markdown(html) if status.content_type == 'text/markdown' - html = encode_and_link_urls(html, linkable_accounts, keep_html: %w(text/markdown text/html).include?(status.content_type)) - html = reformat(html, true) if %w(text/markdown text/html).include?(status.content_type) + keep_html = !summary_mode && %w(text/markdown text/html).include?(status.content_type) + + html = format_markdown(html) if !summary_mode && status.content_type == 'text/markdown' + html = encode_and_link_urls(html, linkable_accounts, keep_html: keep_html) + html = reformat(html, true) if keep_html html = encode_custom_emojis(html, status.emojis, options[:autoplay]) if options[:custom_emojify] - unless %w(text/markdown text/html).include?(status.content_type) + unless keep_html html = simple_format(html, {}, sanitize: false) - html = html.delete("\n") + html.delete!("\n") end + html = summary_mode ? format_article_summary(html, status) : format_article_content(summary, html) if summary.present? + html = format_footer(html, status.footer, linkable_accounts, status.emojis, **options) if status.footer.present? + html.html_safe # rubocop:disable Rails/OutputSafety + end + + def format_remote_content(html, emojis, **options) + html = reformat(html, options[:outgoing]) + html = encode_custom_emojis(html, emojis, options[:autoplay]) if options[:custom_emojify] + html = format_article_content(options[:summary], html) if options[:article_content] && options[:summary].present? html.html_safe # rubocop:disable Rails/OutputSafety end + def format_footer(html, footer, linkable_accounts, emojis, **options) + footer = encode_and_link_urls(footer, linkable_accounts) + footer = encode_custom_emojis(footer, emojis, options[:autoplay]) if options[:custom_emojify] + footer = "<span class=\"invisible\">– </span>#{footer}" + footer = simple_format(footer, { 'data-name': 'footer' }, sanitize: false) + footer.delete!("\n") + + "#{html}#{footer}" + end + def format_markdown(html) html = markdown_formatter.render(html) html.delete("\r").delete("\n") end + def format_article(text) + text = text.gsub(/>[\r\n]+</, '><') + text.html_safe # rubocop:disable Rails/OutputSafety + end + + def format_article_summary(html, status) + status_url = ActivityPub::TagManager.instance.url_for(status) + "#{html}\n<p data-name=\"permalink\">#{link_url(status_url)}</p>" + end + + def format_article_content(summary, html) + "<blockquote data-name=\"summary\">#{format_summary(summary, html)}</blockquote>#{html}" + end + def reformat(html, outgoing = false) sanitize(html, Sanitize::Config::MASTODON_STRICT.merge(outgoing: outgoing)) rescue ArgumentError @@ -91,7 +137,11 @@ class Formatter end def simplified_format(account, **options) - html = account.local? ? linkify(account.note) : reformat(account.note) + return reformat(account.note) unless account.local? + + html = format_markdown(account.note) + html = encode_and_link_urls(html, keep_html: true) + html = reformat(html, true) html = encode_custom_emojis(html, account.emojis, options[:autoplay]) if options[:custom_emojify] html.html_safe # rubocop:disable Rails/OutputSafety end @@ -100,8 +150,12 @@ class Formatter Sanitize.fragment(html, config) end + def format_summary(summary, fallback) + summary&.strip.presence || fallback[/(?:<p>.*?<\/p>)/im].presence || '🗎❓' + end + def format_spoiler(status, **options) - html = encode(status.spoiler_text) + html = encode(status.title.presence || status.spoiler_text) html = encode_custom_emojis(html, status.emojis, options[:autoplay]) html.html_safe # rubocop:disable Rails/OutputSafety end @@ -124,8 +178,8 @@ class Formatter html.html_safe # rubocop:disable Rails/OutputSafety end - def linkify(text) - html = encode_and_link_urls(text) + def linkify(text, accounts = nil, options = {}) + html = encode_and_link_urls(text, accounts, options) html = simple_format(html, {}, sanitize: false) html = html.delete("\n") @@ -156,7 +210,7 @@ class Formatter renderer = HTMLRenderer.new({ filter_html: false, escape_html: false, - no_images: true, + no_images: false, no_styles: true, safe_links_only: true, hard_wrap: true, @@ -392,4 +446,17 @@ class Formatter def mention_html(account) "<span class=\"h-card\"><a href=\"#{encode(ActivityPub::TagManager.instance.url_for(account))}\" class=\"u-url mention\">@<span>#{encode(account.username)}</span></a></span>" end + + def formatter_cache_key(status, options) + [ + 'format', + status.id.to_s, + options[:article_content] ? '1' : '0', + options[:inline_poll_options] ? '1' : '0', + options[:plaintext] ? '1' : '0', + options[:autoplay] ? '1' : '0', + options[:custom_emojify] ? '1' : '0', + ].join(':') + end end +# rubocop:enable Metrics/ClassLength diff --git a/app/lib/img_tag_handler.rb b/app/lib/img_tag_handler.rb new file mode 100644 index 000000000..0263e1cbd --- /dev/null +++ b/app/lib/img_tag_handler.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +class ImgTagHandler < ::Ox::Sax + attr_reader :srcs + attr_reader :alts + + def initialize + @stack = [] + @srcs = [] + @alts = {} + end + + def start_element(element_name) + @stack << [element_name, {}] + end + + def end_element(_) + self_name, self_attributes = @stack[-1] + if self_name == :img && !self_attributes[:src].nil? + @srcs << self_attributes[:src] + @alts[self_attributes[:src]] = self_attributes[:alt]&.strip + end + @stack.pop + end + + def attr(attribute_name, attribute_value) + _name, attributes = @stack.last + attributes[attribute_name] = attribute_value&.strip + end +end diff --git a/app/lib/rss/serializer.rb b/app/lib/rss/serializer.rb index fd56c568c..334726885 100644 --- a/app/lib/rss/serializer.rb +++ b/app/lib/rss/serializer.rb @@ -10,6 +10,7 @@ class RSS::Serializer .link(ActivityPub::TagManager.instance.url_for(status)) .pub_date(status.created_at) .description(status.spoiler_text.presence || Formatter.instance.format(status, inline_poll_options: true).to_str) + .content(Formatter.instance.format(status, inline_poll_options: true, article_content: true).to_str) status.media_attachments.each do |media| item.enclosure(full_asset_url(media.file.url(:original, false)), media.file.content_type, media.file.size) diff --git a/app/lib/rss_builder.rb b/app/lib/rss_builder.rb index 63ddba2e8..a74b4e035 100644 --- a/app/lib/rss_builder.rb +++ b/app/lib/rss_builder.rb @@ -35,6 +35,12 @@ class RSSBuilder self end + def content(str) + @item << (Ox::Element.new('content:encoded') << str) + + self + end + def enclosure(url, type, size) @item << Ox::Element.new('enclosure').tap do |enclosure| enclosure['url'] = url diff --git a/app/lib/sanitize_config.rb b/app/lib/sanitize_config.rb index 0fb415bd1..3bc25fe9f 100644 --- a/app/lib/sanitize_config.rb +++ b/app/lib/sanitize_config.rb @@ -31,28 +31,23 @@ class Sanitize next true if e =~ /^(h|p|u|dt|e)-/ # microformats classes next true if e =~ /^(mention|hashtag)$/ # semantic classes next true if e =~ /^(ellipsis|invisible)$/ # link formatting classes + next true if %w(center centered abstract).include?(e) end node['class'] = class_list.join(' ') end - IMG_TAG_TRANSFORMER = lambda do |env| + DATA_NAME_ALLOWLIST_TRANSFORMER = lambda do |env| node = env[:node] + name_list = node['data-name']&.split(/[\t\n\f\r ]/) - return unless env[:node_name] == 'img' + return unless name_list - node.name = 'a' - - node['href'] = node['src'] - if node['alt'].present? - node.content = "[🖼 #{node['alt']}]" - else - url = node['href'] - prefix = url.match(/\Ahttps?:\/\/(www\.)?/).to_s - text = url[prefix.length, 30] - text = text + "…" if url[prefix.length..-1].length > 30 - node.content = "[🖼 #{text}]" + name_list.keep_if do |name| + next true if %w(summary abstract permalink footer).include?(name) end + + node['data-name'] = name_list.join(' ') end LINK_REL_TRANSFORMER = lambda do |env| @@ -84,15 +79,17 @@ class Sanitize end MASTODON_STRICT ||= freeze_config( - elements: %w(p br span a abbr del pre blockquote code b strong u sub sup i em h1 h2 h3 h4 h5 ul ol li), + elements: %w(p br span a abbr del pre blockquote code b strong u sub sup i em h1 h2 h3 h4 h5 ul ol li img h6 s center details summary), attributes: { 'a' => %w(href rel class title), 'span' => %w(class), 'abbr' => %w(title), - 'blockquote' => %w(cite), + 'blockquote' => %w(cite data-name), 'ol' => %w(start reversed), 'li' => %w(value), + 'img' => %w(src alt title), + 'p' => %w(data-name), }, add_attributes: { @@ -108,7 +105,7 @@ class Sanitize transformers: [ CLASS_WHITELIST_TRANSFORMER, - IMG_TAG_TRANSFORMER, + DATA_NAME_ALLOWLIST_TRANSFORMER, UNSUPPORTED_HREF_TRANSFORMER, LINK_REL_TRANSFORMER, ] diff --git a/app/lib/status_filter.rb b/app/lib/status_filter.rb index b6c80b801..bd3e5245e 100644 --- a/app/lib/status_filter.rb +++ b/app/lib/status_filter.rb @@ -3,15 +3,17 @@ class StatusFilter attr_reader :status, :account - def initialize(status, account, preloaded_relations = {}) + def initialize(status, account, filter_silenced, preloaded_relations = {}) @status = status @account = account @preloaded_relations = preloaded_relations + @filter_silenced = filter_silenced end def filtered? return false if !account.nil? && account.id == status.account_id - blocked_by_policy? || (account_present? && filtered_status?) || silenced_account? + + blocked_by_policy? || (account_present? && filtered_status?) || (@filter_silenced && silenced_account?) end private diff --git a/app/lib/user_settings_decorator.rb b/app/lib/user_settings_decorator.rb index 581101782..0f927d5d0 100644 --- a/app/lib/user_settings_decorator.rb +++ b/app/lib/user_settings_decorator.rb @@ -1,6 +1,10 @@ # frozen_string_literal: true +require 'w3c_validators' + class UserSettingsDecorator + include W3CValidators + attr_reader :user, :settings def initialize(user) @@ -32,18 +36,38 @@ class UserSettingsDecorator user.settings['system_font_ui'] = system_font_ui_preference if change?('setting_system_font_ui') user.settings['system_emoji_font'] = system_emoji_font_preference if change?('setting_system_emoji_font') user.settings['noindex'] = noindex_preference if change?('setting_noindex') - user.settings['hide_followers_count']= hide_followers_count_preference if change?('setting_hide_followers_count') + user.settings['hide_followers_count'] = hide_followers_count_preference if change?('setting_hide_followers_count') user.settings['flavour'] = flavour_preference if change?('setting_flavour') user.settings['skin'] = skin_preference if change?('setting_skin') user.settings['hide_network'] = hide_network_preference if change?('setting_hide_network') - user.settings['aggregate_reblogs'] = aggregate_reblogs_preference if change?('setting_aggregate_reblogs') user.settings['show_application'] = show_application_preference if change?('setting_show_application') user.settings['advanced_layout'] = advanced_layout_preference if change?('setting_advanced_layout') - user.settings['default_content_type']= default_content_type_preference if change?('setting_default_content_type') + user.settings['default_content_type'] = default_content_type_preference if change?('setting_default_content_type') user.settings['use_blurhash'] = use_blurhash_preference if change?('setting_use_blurhash') user.settings['use_pending_items'] = use_pending_items_preference if change?('setting_use_pending_items') user.settings['trends'] = trends_preference if change?('setting_trends') user.settings['crop_images'] = crop_images_preference if change?('setting_crop_images') + + user.settings['manual_publish'] = manual_publish_preference if change?('setting_manual_publish') + user.settings['style_dashed_nest'] = style_dashed_nest_preference if change?('setting_style_dashed_nest') + user.settings['style_underline_a'] = style_underline_a_preference if change?('setting_style_underline_a') + user.settings['style_css_profile'] = style_css_profile_preference if change?('setting_style_css_profile') + user.settings['style_css_webapp'] = style_css_webapp_preference if change?('setting_style_css_webapp') + user.settings['style_wide_media'] = style_wide_media_preference if change?('setting_style_wide_media') + user.settings['style_lowercase'] = style_lowercase_preference if change?('setting_style_lowercase') + user.settings['publish_in'] = publish_in_preference if change?('setting_publish_in') + user.settings['unpublish_in'] = unpublish_in_preference if change?('setting_unpublish_in') + user.settings['unpublish_delete'] = unpublish_delete_preference if change?('setting_unpublish_delete') + user.settings['boost_every'] = boost_every_preference if change?('setting_boost_every') + user.settings['boost_jitter'] = boost_jitter_preference if change?('setting_boost_jitter') + user.settings['boost_random'] = boost_random_preference if change?('setting_boost_random') + user.settings['filter_unknown'] = filter_unknown_preference if change?('setting_filter_unknown') + user.settings['unpublish_on_delete'] = unpublish_on_delete_preference if change?('setting_unpublish_on_delete') + user.settings['rss_disabled'] = rss_disabled_preference if change?('setting_rss_disabled') + user.settings['home_reblogs'] = home_reblogs_preference if change?('setting_home_reblogs') + user.settings['max_history_public'] = max_history_public_preference if change?('setting_max_history_public') + user.settings['max_history_private'] = max_history_private_preference if change?('setting_max_history_private') + user.settings['web_push'] = web_push_preferences if change?('setting_web_push') end def merged_notification_emails @@ -134,10 +158,6 @@ class UserSettingsDecorator settings['setting_default_language'] end - def aggregate_reblogs_preference - boolean_cast_setting 'setting_aggregate_reblogs' - end - def advanced_layout_preference boolean_cast_setting 'setting_advanced_layout' end @@ -162,6 +182,90 @@ class UserSettingsDecorator boolean_cast_setting 'setting_crop_images' end + def manual_publish_preference + boolean_cast_setting 'setting_manual_publish' + end + + def style_dashed_nest_preference + boolean_cast_setting 'setting_style_dashed_nest' + end + + def style_underline_a_preference + boolean_cast_setting 'setting_style_underline_a' + end + + def style_css_profile_preference + css = settings['setting_style_css_profile'].to_s.strip.delete("\r").gsub(/\n\n\n+/, "\n\n") + user.settings['style_css_profile_errors'] = validate_css(css) + css + end + + def style_css_webapp_preference + css = settings['setting_style_css_webapp'].to_s.strip.delete("\r").gsub(/\n\n\n+/, "\n\n") + user.settings['style_css_webapp_errors'] = validate_css(css) + css + end + + def style_wide_media_preference + boolean_cast_setting 'setting_style_wide_media' + end + + def style_lowercase_preference + boolean_cast_setting 'setting_style_lowercase' + end + + def publish_in_preference + settings['setting_publish_in'].to_i + end + + def unpublish_in_preference + settings['setting_unpublish_in'].to_i + end + + def unpublish_delete_preference + boolean_cast_setting 'setting_unpublish_delete' + end + + def boost_every_preference + settings['setting_boost_every'].to_i + end + + def boost_jitter_preference + settings['setting_boost_jitter'].to_i + end + + def boost_random_preference + boolean_cast_setting 'setting_boost_random' + end + + def filter_unknown_preference + boolean_cast_setting 'setting_filter_unknown' + end + + def unpublish_on_delete_preference + boolean_cast_setting 'setting_unpublish_on_delete' + end + + def rss_disabled_preference + boolean_cast_setting 'setting_rss_disabled' + end + + def home_reblogs_preference + boolean_cast_setting 'setting_home_reblogs' + end + + def max_history_public_preference + settings['setting_max_history_public'].to_i + end + + def max_history_private_preference + settings['setting_max_history_private'].to_i + end + + def web_push_preferences + boolean_cast_setting 'setting_web_push' + end + def boolean_cast_setting(key) ActiveModel::Type::Boolean.new.cast(settings[key]) end @@ -177,4 +281,10 @@ class UserSettingsDecorator def change?(key) !settings[key].nil? end + + def validate_css(css) + @validator ||= CSSValidator.new + results = @validator.validate_text(css) + results.errors.map { |e| e.to_s.strip } + end end |