diff options
Diffstat (limited to 'app/lib')
-rw-r--r-- | app/lib/activitypub/activity.rb | 6 | ||||
-rw-r--r-- | app/lib/activitypub/activity/announce.rb | 3 | ||||
-rw-r--r-- | app/lib/activitypub/activity/create.rb | 14 | ||||
-rw-r--r-- | app/lib/activitypub/activity/update.rb | 7 | ||||
-rw-r--r-- | app/lib/entity_cache.rb | 34 | ||||
-rw-r--r-- | app/lib/exceptions.rb | 1 | ||||
-rw-r--r-- | app/lib/feed_manager.rb | 17 | ||||
-rw-r--r-- | app/lib/formatter.rb | 10 | ||||
-rw-r--r-- | app/lib/ostatus/activity/creation.rb | 12 | ||||
-rw-r--r-- | app/lib/ostatus/atom_serializer.rb | 2 | ||||
-rw-r--r-- | app/lib/provider_discovery.rb | 47 | ||||
-rw-r--r-- | app/lib/request.rb | 19 | ||||
-rw-r--r-- | app/lib/rss_builder.rb | 130 | ||||
-rw-r--r-- | app/lib/status_filter.rb | 17 |
14 files changed, 233 insertions, 86 deletions
diff --git a/app/lib/activitypub/activity.rb b/app/lib/activitypub/activity.rb index 9b00f0f52..84d4b1752 100644 --- a/app/lib/activitypub/activity.rb +++ b/app/lib/activitypub/activity.rb @@ -78,8 +78,10 @@ class ActivityPub::Activity notify_about_reblog(status) if reblog_of_local_account?(status) notify_about_mentions(status) - # Only continue if the status is supposed to have - # arrived in real-time + # Only continue if the status is supposed to have arrived in real-time. + # Note that if @options[:override_timestamps] isn't set, the status + # may have a lower snowflake id than other existing statuses, potentially + # "hiding" it from paginated API calls return unless @options[:override_timestamps] || status.within_realtime_window? distribute_to_followers(status) diff --git a/app/lib/activitypub/activity/announce.rb b/app/lib/activitypub/activity/announce.rb index c8a358195..7e146ea8c 100644 --- a/app/lib/activitypub/activity/announce.rb +++ b/app/lib/activitypub/activity/announce.rb @@ -15,7 +15,8 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity account: @account, reblog: original_status, uri: @json['id'], - created_at: @options[:override_timestamps] ? nil : @json['published'], + created_at: @json['published'], + override_timestamps: @options[:override_timestamps], visibility: original_status.visibility ) diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index 45c0e91cb..8d17a4ebe 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -47,7 +47,8 @@ class ActivityPub::Activity::Create < ActivityPub::Activity text: text_from_content || '', language: detected_language, spoiler_text: @object['summary'] || '', - created_at: @options[:override_timestamps] ? nil : @object['published'], + created_at: @object['published'], + override_timestamps: @options[:override_timestamps], reply: @object['inReplyTo'].present?, sensitive: @object['sensitive'] || false, visibility: visibility_from_audience, @@ -61,12 +62,11 @@ class ActivityPub::Activity::Create < ActivityPub::Activity return if @object['tag'].nil? as_array(@object['tag']).each do |tag| - case tag['type'] - when 'Hashtag' + if equals_or_includes?(tag['type'], 'Hashtag') process_hashtag tag, status - when 'Mention' + elsif equals_or_includes?(tag['type'], 'Mention') process_mention tag, status - when 'Emoji' + elsif equals_or_includes?(tag['type'], 'Emoji') process_emoji tag, status end end @@ -235,11 +235,11 @@ class ActivityPub::Activity::Create < ActivityPub::Activity end def supported_object_type? - SUPPORTED_TYPES.include?(@object['type']) + equals_or_includes_any?(@object['type'], SUPPORTED_TYPES) end def converted_object_type? - CONVERTED_TYPES.include?(@object['type']) + equals_or_includes_any?(@object['type'], CONVERTED_TYPES) end def skip_download? diff --git a/app/lib/activitypub/activity/update.rb b/app/lib/activitypub/activity/update.rb index 0134b4015..aa5907f03 100644 --- a/app/lib/activitypub/activity/update.rb +++ b/app/lib/activitypub/activity/update.rb @@ -1,11 +1,10 @@ # frozen_string_literal: true class ActivityPub::Activity::Update < ActivityPub::Activity + SUPPORTED_TYPES = %w(Application Group Organization Person Service).freeze + def perform - case @object['type'] - when 'Person' - update_account - end + update_account if equals_or_includes_any?(@object['type'], SUPPORTED_TYPES) end private diff --git a/app/lib/entity_cache.rb b/app/lib/entity_cache.rb new file mode 100644 index 000000000..2aa37389c --- /dev/null +++ b/app/lib/entity_cache.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +require 'singleton' + +class EntityCache + include Singleton + + MAX_EXPIRATION = 7.days.freeze + + def mention(username, domain) + Rails.cache.fetch(to_key(:mention, username, domain), expires_in: MAX_EXPIRATION) { Account.select(:username, :domain, :url).find_remote(username, domain) } + end + + def emoji(shortcodes, domain) + shortcodes = [shortcodes] unless shortcodes.is_a?(Array) + cached = Rails.cache.read_multi(*shortcodes.map { |shortcode| to_key(:emoji, shortcode, domain) }) + uncached_ids = [] + + shortcodes.each do |shortcode| + uncached_ids << shortcode unless cached.key?(to_key(:emoji, shortcode, domain)) + end + + unless uncached_ids.empty? + uncached = CustomEmoji.where(shortcode: shortcodes, domain: domain, disabled: false).map { |item| [item.shortcode, item] }.to_h + uncached.each_value { |item| Rails.cache.write(to_key(:emoji, item.shortcode, domain), item, expires_in: MAX_EXPIRATION) } + end + + shortcodes.map { |shortcode| cached[to_key(:emoji, shortcode, domain)] || uncached[shortcode] }.compact + end + + def to_key(type, *ids) + "#{type}:#{ids.compact.map(&:downcase).join(':')}" + end +end diff --git a/app/lib/exceptions.rb b/app/lib/exceptions.rb index e88e98eae..01346bfe5 100644 --- a/app/lib/exceptions.rb +++ b/app/lib/exceptions.rb @@ -6,6 +6,7 @@ module Mastodon class ValidationError < Error; end class HostValidationError < ValidationError; end class LengthValidationError < ValidationError; end + class DimensionsValidationError < ValidationError; end class RaceConditionError < Error; end class UnexpectedResponseError < Error diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb index 700fd61c4..3a2dcac68 100644 --- a/app/lib/feed_manager.rb +++ b/app/lib/feed_manager.rb @@ -145,10 +145,14 @@ class FeedManager redis.exists("subscribed:#{timeline_id}") end + def blocks_or_mutes?(receiver_id, account_ids, context) + Block.where(account_id: receiver_id, target_account_id: account_ids).any? || + (context == :home ? Mute.where(account_id: receiver_id, target_account_id: account_ids).any? : Mute.where(account_id: receiver_id, target_account_id: account_ids, hide_notifications: true).any?) + end + def filter_from_home?(status, receiver_id) return false if receiver_id == status.account_id return true if status.reply? && (status.in_reply_to_id.nil? || status.in_reply_to_account_id.nil?) - return true if keyword_filter?(status, receiver_id) check_for_mutes = [status.account_id] @@ -158,9 +162,10 @@ class FeedManager return true if Mute.where(account_id: receiver_id, target_account_id: check_for_mutes).any? check_for_blocks = status.mentions.pluck(:account_id) + check_for_blocks.concat([status.account_id]) check_for_blocks.concat([status.reblog.account_id]) if status.reblog? - return true if Block.where(account_id: receiver_id, target_account_id: check_for_blocks).any? + return true if blocks_or_mutes?(receiver_id, check_for_blocks, :home) if status.reply? && !status.in_reply_to_account_id.nil? # Filter out if it's a reply should_filter = !Follow.where(account_id: receiver_id, target_account_id: status.in_reply_to_account_id).exists? # and I'm not following the person it's a reply to @@ -184,11 +189,13 @@ class FeedManager def filter_from_mentions?(status, receiver_id) return true if receiver_id == status.account_id - check_for_blocks = [status.account_id] - check_for_blocks.concat(status.mentions.pluck(:account_id)) + # This filter is called from NotifyService, but already after the sender of + # the notification has been checked for mute/block. Therefore, it's not + # necessary to check the author of the toot for mute/block again + check_for_blocks = status.mentions.pluck(:account_id) check_for_blocks.concat([status.in_reply_to_account]) if status.reply? && !status.in_reply_to_account_id.nil? - should_filter = Block.where(account_id: receiver_id, target_account_id: check_for_blocks).any? # Filter if it's from someone I blocked, in reply to someone I blocked, or mentioning someone I blocked + should_filter = blocks_or_mutes?(receiver_id, check_for_blocks, :mentions) # Filter if it's from someone I blocked, in reply to someone I blocked, or mentioning someone I blocked (or muted) should_filter ||= (status.account.silenced? && !Follow.where(account_id: receiver_id, target_account_id: status.account_id).exists?) # of if the account is silenced and I'm not following them should_filter ||= keyword_filter?(status, receiver_id) # or if the mention contains a muted keyword diff --git a/app/lib/formatter.rb b/app/lib/formatter.rb index 4124f1660..050c651ee 100644 --- a/app/lib/formatter.rb +++ b/app/lib/formatter.rb @@ -52,12 +52,8 @@ class Formatter end def simplified_format(account, **options) - html = if account.local? - linkify(account.note) - else - reformat(account.note) - end - html = encode_custom_emojis(html, CustomEmoji.from_text(account.note, account.domain)) if options[:custom_emojify] + html = account.local? ? linkify(account.note) : reformat(account.note) + html = encode_custom_emojis(html, account.emojis) if options[:custom_emojify] html.html_safe # rubocop:disable Rails/OutputSafety end @@ -211,7 +207,7 @@ class Formatter username, domain = acct.split('@') domain = nil if TagManager.instance.local_domain?(domain) - account = Account.find_remote(username, domain) + account = EntityCache.instance.mention(username, domain) account ? mention_html(account) : "@#{acct}" end diff --git a/app/lib/ostatus/activity/creation.rb b/app/lib/ostatus/activity/creation.rb index 6235127b2..1e7f47029 100644 --- a/app/lib/ostatus/activity/creation.rb +++ b/app/lib/ostatus/activity/creation.rb @@ -39,7 +39,8 @@ class OStatus::Activity::Creation < OStatus::Activity::Base reblog: cached_reblog, text: content, spoiler_text: content_warning, - created_at: @options[:override_timestamps] ? nil : published, + created_at: published, + override_timestamps: @options[:override_timestamps], reply: thread?, language: content_language, visibility: visibility_scope, @@ -61,7 +62,14 @@ class OStatus::Activity::Creation < OStatus::Activity::Base Rails.logger.debug "Queuing remote status #{status.id} (#{id}) for distribution" LinkCrawlWorker.perform_async(status.id) unless status.spoiler_text? - DistributionWorker.perform_async(status.id) if @options[:override_timestamps] || status.within_realtime_window? + + # Only continue if the status is supposed to have arrived in real-time. + # Note that if @options[:override_timestamps] isn't set, the status + # may have a lower snowflake id than other existing statuses, potentially + # "hiding" it from paginated API calls + return status unless @options[:override_timestamps] || status.within_realtime_window? + + DistributionWorker.perform_async(status.id) status end diff --git a/app/lib/ostatus/atom_serializer.rb b/app/lib/ostatus/atom_serializer.rb index 055b4649c..7c66f2066 100644 --- a/app/lib/ostatus/atom_serializer.rb +++ b/app/lib/ostatus/atom_serializer.rb @@ -364,8 +364,6 @@ class OStatus::AtomSerializer append_element(entry, 'category', nil, term: tag.name) end - append_element(entry, 'category', nil, term: 'nsfw') if status.sensitive? - status.media_attachments.each do |media| append_element(entry, 'link', nil, rel: :enclosure, type: media.file_content_type, length: media.file_file_size, href: full_asset_url(media.file.url(:original, false))) end diff --git a/app/lib/provider_discovery.rb b/app/lib/provider_discovery.rb deleted file mode 100644 index 3bec7211b..000000000 --- a/app/lib/provider_discovery.rb +++ /dev/null @@ -1,47 +0,0 @@ -# frozen_string_literal: true - -class ProviderDiscovery < OEmbed::ProviderDiscovery - class << self - def get(url, **options) - provider = discover_provider(url, options) - - options.delete(:html) - - provider.get(url, options) - end - - def discover_provider(url, **options) - format = options[:format] - - html = if options[:html] - Nokogiri::HTML(options[:html]) - else - Request.new(:get, url).perform do |res| - raise OEmbed::NotFound, url if res.code != 200 || res.mime_type != 'text/html' - Nokogiri::HTML(res.body_with_limit) - end - end - - if format.nil? || format == :json - provider_endpoint ||= html.at_xpath('//link[@type="application/json+oembed"]')&.attribute('href')&.value - format ||= :json if provider_endpoint - end - - if format.nil? || format == :xml - provider_endpoint ||= html.at_xpath('//link[@type="text/xml+oembed"]')&.attribute('href')&.value - format ||= :xml if provider_endpoint - end - - raise OEmbed::NotFound, url if provider_endpoint.nil? - begin - provider_endpoint = Addressable::URI.parse(provider_endpoint) - provider_endpoint.query = nil - provider_endpoint = provider_endpoint.to_s - rescue Addressable::URI::InvalidURIError - raise OEmbed::NotFound, url - end - - OEmbed::Provider.new(provider_endpoint, format) - end - end -end diff --git a/app/lib/request.rb b/app/lib/request.rb index dca93a6e9..00f94dacf 100644 --- a/app/lib/request.rb +++ b/app/lib/request.rb @@ -9,11 +9,15 @@ class Request include RoutingHelper def initialize(verb, url, **options) + raise ArgumentError if url.blank? + @verb = verb @url = Addressable::URI.parse(url).normalize - @options = options.merge(socket_class: Socket) + @options = options.merge(use_proxy? ? Rails.configuration.x.http_client_proxy : { socket_class: Socket }) @headers = {} + raise Mastodon::HostValidationError, 'Instance does not support hidden service connections' if block_hidden_service? + set_common_headers! set_digest! if options.key?(:body) end @@ -99,6 +103,14 @@ class Request @http_client ||= HTTP.timeout(:per_operation, timeout).follow(max_hops: 2) end + def use_proxy? + Rails.configuration.x.http_client_proxy.present? + end + + def block_hidden_service? + !Rails.configuration.x.access_to_hidden_service && /\.(onion|i2p)$/.match(@url.host) + end + module ClientLimit def body_with_limit(limit = 1.megabyte) raise Mastodon::LengthValidationError if content_length.present? && content_length > limit @@ -129,6 +141,7 @@ class Request class Socket < TCPSocket class << self def open(host, *args) + return super host, *args if thru_hidden_service? host outer_e = nil Addrinfo.foreach(host, nil, nil, :SOCK_STREAM) do |address| begin @@ -142,6 +155,10 @@ class Request end alias new open + + def thru_hidden_service?(host) + Rails.configuration.x.hidden_service_via_transparent_proxy && /\.(onion|i2p)$/.match(host) + end end end diff --git a/app/lib/rss_builder.rb b/app/lib/rss_builder.rb new file mode 100644 index 000000000..63ddba2e8 --- /dev/null +++ b/app/lib/rss_builder.rb @@ -0,0 +1,130 @@ +# frozen_string_literal: true + +class RSSBuilder + class ItemBuilder + def initialize + @item = Ox::Element.new('item') + end + + def title(str) + @item << (Ox::Element.new('title') << str) + + self + end + + def link(str) + @item << Ox::Element.new('guid').tap do |guid| + guid['isPermalink'] = 'true' + guid << str + end + + @item << (Ox::Element.new('link') << str) + + self + end + + def pub_date(date) + @item << (Ox::Element.new('pubDate') << date.to_formatted_s(:rfc822)) + + self + end + + def description(str) + @item << (Ox::Element.new('description') << str) + + self + end + + def enclosure(url, type, size) + @item << Ox::Element.new('enclosure').tap do |enclosure| + enclosure['url'] = url + enclosure['length'] = size + enclosure['type'] = type + end + + self + end + + def to_element + @item + end + end + + def initialize + @document = Ox::Document.new(version: '1.0') + @channel = Ox::Element.new('channel') + + @document << (rss << @channel) + end + + def title(str) + @channel << (Ox::Element.new('title') << str) + + self + end + + def link(str) + @channel << (Ox::Element.new('link') << str) + + self + end + + def image(str) + @channel << Ox::Element.new('image').tap do |image| + image << (Ox::Element.new('url') << str) + image << (Ox::Element.new('title') << '') + image << (Ox::Element.new('link') << '') + end + + @channel << (Ox::Element.new('webfeeds:icon') << str) + + self + end + + def cover(str) + @channel << Ox::Element.new('webfeeds:cover').tap do |cover| + cover['image'] = str + end + + self + end + + def logo(str) + @channel << (Ox::Element.new('webfeeds:logo') << str) + + self + end + + def accent_color(str) + @channel << (Ox::Element.new('webfeeds:accentColor') << str) + + self + end + + def description(str) + @channel << (Ox::Element.new('description') << str) + + self + end + + def item + @channel << ItemBuilder.new.tap do |item| + yield item + end.to_element + + self + end + + def to_xml + ('<?xml version="1.0" encoding="UTF-8"?>' + Ox.dump(@document, effort: :tolerant)).force_encoding('UTF-8') + end + + private + + def rss + Ox::Element.new('rss').tap do |rss| + rss['version'] = '2.0' + rss['xmlns:webfeeds'] = 'http://webfeeds.org/rss/1.0' + end + end +end diff --git a/app/lib/status_filter.rb b/app/lib/status_filter.rb index 41d4381e5..b6c80b801 100644 --- a/app/lib/status_filter.rb +++ b/app/lib/status_filter.rb @@ -3,9 +3,10 @@ class StatusFilter attr_reader :status, :account - def initialize(status, account) - @status = status - @account = account + def initialize(status, account, preloaded_relations = {}) + @status = status + @account = account + @preloaded_relations = preloaded_relations end def filtered? @@ -24,15 +25,15 @@ class StatusFilter end def blocking_account? - account.blocking? status.account_id + @preloaded_relations[:blocking] ? @preloaded_relations[:blocking][status.account_id] : account.blocking?(status.account_id) end def blocking_domain? - account.domain_blocking? status.account_domain + @preloaded_relations[:domain_blocking_by_domain] ? @preloaded_relations[:domain_blocking_by_domain][status.account_domain] : account.domain_blocking?(status.account_domain) end def muting_account? - account.muting? status.account_id + @preloaded_relations[:muting] ? @preloaded_relations[:muting][status.account_id] : account.muting?(status.account_id) end def silenced_account? @@ -44,7 +45,7 @@ class StatusFilter end def account_following_status_account? - account&.following? status.account_id + @preloaded_relations[:following] ? @preloaded_relations[:following][status.account_id] : account&.following?(status.account_id) end def blocked_by_policy? @@ -52,6 +53,6 @@ class StatusFilter end def policy_allows_show? - StatusPolicy.new(account, status).show? + StatusPolicy.new(account, status, @preloaded_relations).show? end end |