From 75c4ab9d12d3a2f3de52c51b5006fe9d5d9afae4 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sun, 22 Apr 2018 22:09:03 +0200 Subject: Remove "nsfw" category for sensitive statuses in OStatus serializer (#7048) Fix #7011 --- app/lib/ostatus/atom_serializer.rb | 2 -- 1 file changed, 2 deletions(-) (limited to 'app/lib') diff --git a/app/lib/ostatus/atom_serializer.rb b/app/lib/ostatus/atom_serializer.rb index 055b4649c..7c66f2066 100644 --- a/app/lib/ostatus/atom_serializer.rb +++ b/app/lib/ostatus/atom_serializer.rb @@ -364,8 +364,6 @@ class OStatus::AtomSerializer append_element(entry, 'category', nil, term: tag.name) end - append_element(entry, 'category', nil, term: 'nsfw') if status.sensitive? - status.media_attachments.each do |media| append_element(entry, 'link', nil, rel: :enclosure, type: media.file_content_type, length: media.file_file_size, href: full_asset_url(media.file.url(:original, false))) end -- cgit From 7db7d68136d8c58c6d354e85096137c39d421671 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Mon, 23 Apr 2018 09:16:38 +0200 Subject: Detect and prevent image bombs, max. processable dimension 4096^2 (#7229) --- app/lib/exceptions.rb | 1 + app/models/concerns/attachmentable.rb | 32 +++++++++++++++++++++++++++++--- app/models/custom_emoji.rb | 2 ++ app/models/media_attachment.rb | 16 ++-------------- 4 files changed, 34 insertions(+), 17 deletions(-) (limited to 'app/lib') diff --git a/app/lib/exceptions.rb b/app/lib/exceptions.rb index e88e98eae..01346bfe5 100644 --- a/app/lib/exceptions.rb +++ b/app/lib/exceptions.rb @@ -6,6 +6,7 @@ module Mastodon class ValidationError < Error; end class HostValidationError < ValidationError; end class LengthValidationError < ValidationError; end + class DimensionsValidationError < ValidationError; end class RaceConditionError < Error; end class UnexpectedResponseError < Error diff --git a/app/models/concerns/attachmentable.rb b/app/models/concerns/attachmentable.rb index 90ce88463..6f8489b89 100644 --- a/app/models/concerns/attachmentable.rb +++ b/app/models/concerns/attachmentable.rb @@ -1,10 +1,15 @@ # frozen_string_literal: true +require 'mime/types' + module Attachmentable extend ActiveSupport::Concern + MAX_MATRIX_LIMIT = 16_777_216 # 4096x4096px or approx. 16MB + included do before_post_process :set_file_extensions + before_post_process :check_image_dimensions end private @@ -12,10 +17,31 @@ module Attachmentable def set_file_extensions self.class.attachment_definitions.each_key do |attachment_name| attachment = send(attachment_name) + next if attachment.blank? - extension = Paperclip::Interpolations.content_type_extension(attachment, :original) - basename = Paperclip::Interpolations.basename(attachment, :original) - attachment.instance_write :file_name, [basename, extension].delete_if(&:blank?).join('.') + + attachment.instance_write :file_name, [Paperclip::Interpolations.basename(attachment, :original), appropriate_extension(attachment)].delete_if(&:blank?).join('.') + end + end + + def check_image_dimensions + self.class.attachment_definitions.each_key do |attachment_name| + attachment = send(attachment_name) + + next if attachment.blank? || !attachment.content_type.match?(/image.*/) || attachment.queued_for_write[:original].blank? + + width, height = FastImage.size(attachment.queued_for_write[:original].path) + + raise Mastodon::DimensionsValidationError, "#{width}x#{height} images are not supported" if width.present? && height.present? && (width * height >= MAX_MATRIX_LIMIT) end end + + def appropriate_extension(attachment) + mime_type = MIME::Types[attachment.content_type] + + extensions_for_mime_type = mime_type.empty? ? [] : mime_type.first.extensions + original_extension = Paperclip::Interpolations.extension(attachment, :original) + + extensions_for_mime_type.include?(original_extension) ? original_extension : extensions_for_mime_type.first + end end diff --git a/app/models/custom_emoji.rb b/app/models/custom_emoji.rb index 1ec21d1a0..2dd3cac61 100644 --- a/app/models/custom_emoji.rb +++ b/app/models/custom_emoji.rb @@ -40,6 +40,8 @@ class CustomEmoji < ApplicationRecord remotable_attachment :image, LIMIT + include Attachmentable + def local? domain.nil? end diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb index 8fd9ac09f..c9abab9e2 100644 --- a/app/models/media_attachment.rb +++ b/app/models/media_attachment.rb @@ -19,8 +19,6 @@ # description :text # -require 'mime/types' - class MediaAttachment < ApplicationRecord self.inheritance_column = nil @@ -70,6 +68,8 @@ class MediaAttachment < ApplicationRecord validates_attachment_size :file, less_than: LIMIT remotable_attachment :file, LIMIT + include Attachmentable + validates :account, presence: true validates :description, length: { maximum: 420 }, if: :local? @@ -176,9 +176,6 @@ class MediaAttachment < ApplicationRecord def set_type_and_extension self.type = VIDEO_MIME_TYPES.include?(file_content_type) ? :video : :image - extension = appropriate_extension - basename = Paperclip::Interpolations.basename(file, :original) - file.instance_write :file_name, [basename, extension].delete_if(&:blank?).join('.') end def set_meta @@ -223,13 +220,4 @@ class MediaAttachment < ApplicationRecord bitrate: movie.bitrate, } end - - def appropriate_extension - mime_type = MIME::Types[file.content_type] - - extensions_for_mime_type = mime_type.empty? ? [] : mime_type.first.extensions - original_extension = Paperclip::Interpolations.extension(file, :original) - - extensions_for_mime_type.include?(original_extension) ? original_extension : extensions_for_mime_type.first - end end -- cgit From 9d4710ed0059b2f789e6b32b9f81d4ce90b98907 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Wed, 25 Apr 2018 02:10:02 +0200 Subject: Add RSS feeds for end-users (#7259) * Add RSS feed for accounts * Add RSS feeds for hashtags * Fix code style issues * Fix code style issues --- app/controllers/accounts_controller.rb | 10 ++- app/controllers/tags_controller.rb | 11 ++- app/helpers/stream_entries_helper.rb | 12 +-- app/lib/rss_builder.rb | 130 ++++++++++++++++++++++++++++++ app/serializers/rss/account_serializer.rb | 39 +++++++++ app/serializers/rss/tag_serializer.rb | 37 +++++++++ 6 files changed, 230 insertions(+), 9 deletions(-) create mode 100644 app/lib/rss_builder.rb create mode 100644 app/serializers/rss/account_serializer.rb create mode 100644 app/serializers/rss/tag_serializer.rb (limited to 'app/lib') diff --git a/app/controllers/accounts_controller.rb b/app/controllers/accounts_controller.rb index 7bf35825f..1152d4aca 100644 --- a/app/controllers/accounts_controller.rb +++ b/app/controllers/accounts_controller.rb @@ -20,9 +20,10 @@ class AccountsController < ApplicationController @pinned_statuses = cache_collection(@account.pinned_statuses, Status) if show_pinned_statuses? @statuses = filtered_status_page(params) @statuses = cache_collection(@statuses, Status) + unless @statuses.empty? - @older_url = older_url if @statuses.last.id > filtered_statuses.last.id - @newer_url = newer_url if @statuses.first.id < filtered_statuses.first.id + @older_url = older_url if @statuses.last.id > filtered_statuses.last.id + @newer_url = newer_url if @statuses.first.id < filtered_statuses.first.id end end @@ -31,6 +32,11 @@ class AccountsController < ApplicationController render xml: OStatus::AtomSerializer.render(OStatus::AtomSerializer.new.feed(@account, @entries.reject { |entry| entry.status.nil? })) end + format.rss do + @statuses = cache_collection(default_statuses.without_reblogs.without_replies.limit(PAGE_SIZE), Status) + render xml: RSS::AccountSerializer.render(@account, @statuses) + end + format.json do skip_session! diff --git a/app/controllers/tags_controller.rb b/app/controllers/tags_controller.rb index 9f3090e37..014a5c9b8 100644 --- a/app/controllers/tags_controller.rb +++ b/app/controllers/tags_controller.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class TagsController < ApplicationController + PAGE_SIZE = 20 + before_action :set_body_classes before_action :set_instance_presenter @@ -13,8 +15,15 @@ class TagsController < ApplicationController @initial_state_json = serializable_resource.to_json end + format.rss do + @statuses = Status.as_tag_timeline(@tag).limit(PAGE_SIZE) + @statuses = cache_collection(@statuses, Status) + + render xml: RSS::TagSerializer.render(@tag, @statuses) + end + format.json do - @statuses = Status.as_tag_timeline(@tag, current_account, params[:local]).paginate_by_max_id(20, params[:max_id]) + @statuses = Status.as_tag_timeline(@tag, current_account, params[:local]).paginate_by_max_id(PAGE_SIZE, params[:max_id]) @statuses = cache_collection(@statuses, Status) render json: collection_presenter, diff --git a/app/helpers/stream_entries_helper.rb b/app/helpers/stream_entries_helper.rb index 8254ef4dc..c6f12ecd4 100644 --- a/app/helpers/stream_entries_helper.rb +++ b/app/helpers/stream_entries_helper.rb @@ -12,17 +12,17 @@ module StreamEntriesHelper prepend_str = [ [ number_to_human(account.statuses_count, strip_insignificant_zeros: true), - t('accounts.posts'), + I18n.t('accounts.posts'), ].join(' '), [ number_to_human(account.following_count, strip_insignificant_zeros: true), - t('accounts.following'), + I18n.t('accounts.following'), ].join(' '), [ number_to_human(account.followers_count, strip_insignificant_zeros: true), - t('accounts.followers'), + I18n.t('accounts.followers'), ].join(' '), ].join(', ') @@ -40,16 +40,16 @@ module StreamEntriesHelper end end - text = attachments.to_a.reject { |_, value| value.zero? }.map { |key, value| t("statuses.attached.#{key}", count: value) }.join(' · ') + text = attachments.to_a.reject { |_, value| value.zero? }.map { |key, value| I18n.t("statuses.attached.#{key}", count: value) }.join(' · ') return if text.blank? - t('statuses.attached.description', attached: text) + I18n.t('statuses.attached.description', attached: text) end def status_text_summary(status) return if status.spoiler_text.blank? - t('statuses.content_warning', warning: status.spoiler_text) + I18n.t('statuses.content_warning', warning: status.spoiler_text) end def status_description(status) diff --git a/app/lib/rss_builder.rb b/app/lib/rss_builder.rb new file mode 100644 index 000000000..63ddba2e8 --- /dev/null +++ b/app/lib/rss_builder.rb @@ -0,0 +1,130 @@ +# frozen_string_literal: true + +class RSSBuilder + class ItemBuilder + def initialize + @item = Ox::Element.new('item') + end + + def title(str) + @item << (Ox::Element.new('title') << str) + + self + end + + def link(str) + @item << Ox::Element.new('guid').tap do |guid| + guid['isPermalink'] = 'true' + guid << str + end + + @item << (Ox::Element.new('link') << str) + + self + end + + def pub_date(date) + @item << (Ox::Element.new('pubDate') << date.to_formatted_s(:rfc822)) + + self + end + + def description(str) + @item << (Ox::Element.new('description') << str) + + self + end + + def enclosure(url, type, size) + @item << Ox::Element.new('enclosure').tap do |enclosure| + enclosure['url'] = url + enclosure['length'] = size + enclosure['type'] = type + end + + self + end + + def to_element + @item + end + end + + def initialize + @document = Ox::Document.new(version: '1.0') + @channel = Ox::Element.new('channel') + + @document << (rss << @channel) + end + + def title(str) + @channel << (Ox::Element.new('title') << str) + + self + end + + def link(str) + @channel << (Ox::Element.new('link') << str) + + self + end + + def image(str) + @channel << Ox::Element.new('image').tap do |image| + image << (Ox::Element.new('url') << str) + image << (Ox::Element.new('title') << '') + image << (Ox::Element.new('link') << '') + end + + @channel << (Ox::Element.new('webfeeds:icon') << str) + + self + end + + def cover(str) + @channel << Ox::Element.new('webfeeds:cover').tap do |cover| + cover['image'] = str + end + + self + end + + def logo(str) + @channel << (Ox::Element.new('webfeeds:logo') << str) + + self + end + + def accent_color(str) + @channel << (Ox::Element.new('webfeeds:accentColor') << str) + + self + end + + def description(str) + @channel << (Ox::Element.new('description') << str) + + self + end + + def item + @channel << ItemBuilder.new.tap do |item| + yield item + end.to_element + + self + end + + def to_xml + ('' + Ox.dump(@document, effort: :tolerant)).force_encoding('UTF-8') + end + + private + + def rss + Ox::Element.new('rss').tap do |rss| + rss['version'] = '2.0' + rss['xmlns:webfeeds'] = 'http://webfeeds.org/rss/1.0' + end + end +end diff --git a/app/serializers/rss/account_serializer.rb b/app/serializers/rss/account_serializer.rb new file mode 100644 index 000000000..bde360a41 --- /dev/null +++ b/app/serializers/rss/account_serializer.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +class RSS::AccountSerializer + include ActionView::Helpers::NumberHelper + include StreamEntriesHelper + include RoutingHelper + + def render(account, statuses) + builder = RSSBuilder.new + + builder.title("#{display_name(account)} (@#{account.local_username_and_domain})") + .description(account_description(account)) + .link(TagManager.instance.url_for(account)) + .logo(full_asset_url(asset_pack_path('logo.svg'))) + .accent_color('2b90d9') + + builder.image(full_asset_url(account.avatar.url(:original))) if account.avatar? + builder.cover(full_asset_url(account.header.url(:original))) if account.header? + + statuses.each do |status| + builder.item do |item| + item.title(status.title) + .link(TagManager.instance.url_for(status)) + .pub_date(status.created_at) + .description(status.spoiler_text.presence || Formatter.instance.format(status).to_str) + + status.media_attachments.each do |media| + item.enclosure(full_asset_url(media.file.url(:original, false)), media.file.content_type, length: media.file.size) + end + end + end + + builder.to_xml + end + + def self.render(account, statuses) + new.render(account, statuses) + end +end diff --git a/app/serializers/rss/tag_serializer.rb b/app/serializers/rss/tag_serializer.rb new file mode 100644 index 000000000..7680a8da5 --- /dev/null +++ b/app/serializers/rss/tag_serializer.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +class RSS::TagSerializer + include ActionView::Helpers::NumberHelper + include ActionView::Helpers::SanitizeHelper + include StreamEntriesHelper + include RoutingHelper + + def render(tag, statuses) + builder = RSSBuilder.new + + builder.title("##{tag.name}") + .description(strip_tags(I18n.t('about.about_hashtag_html', hashtag: tag.name))) + .link(tag_url(tag)) + .logo(full_asset_url(asset_pack_path('logo.svg'))) + .accent_color('2b90d9') + + statuses.each do |status| + builder.item do |item| + item.title(status.title) + .link(TagManager.instance.url_for(status)) + .pub_date(status.created_at) + .description(status.spoiler_text.presence || Formatter.instance.format(status).to_str) + + status.media_attachments.each do |media| + item.enclosure(full_asset_url(media.file.url(:original, false)), media.file.content_type, length: media.file.size) + end + end + end + + builder.to_xml + end + + def self.render(tag, statuses) + new.render(tag, statuses) + end +end -- cgit From f58dcbc9814b5ba2fd4f7d7af643aa25dcf40594 Mon Sep 17 00:00:00 2001 From: MIYAGI Hikaru Date: Wed, 25 Apr 2018 09:14:49 +0900 Subject: HTTP proxy support for outgoing request, manage access to hidden service (#7134) * Add support for HTTP client proxy * Add access control for darknet Supress error when access to darknet via transparent proxy * Fix the codes pointed out * Lint * Fix an omission + lint * any? -> include? * Change detection method to regexp to avoid test fail --- .env.production.sample | 7 +++++++ app/lib/request.rb | 16 +++++++++++++++- config/initializers/http_client_proxy.rb | 24 ++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 config/initializers/http_client_proxy.rb (limited to 'app/lib') diff --git a/.env.production.sample b/.env.production.sample index 9de2c0650..c936546da 100644 --- a/.env.production.sample +++ b/.env.production.sample @@ -214,3 +214,10 @@ STREAMING_CLUSTER_NUM=1 # SAML_UID_ATTRIBUTE="urn:oid:0.9.2342.19200300.100.1.1" # SAML_ATTRIBUTES_STATEMENTS_VERIFIED= # SAML_ATTRIBUTES_STATEMENTS_VERIFIED_EMAIL= + +# Use HTTP proxy for outgoing request (optional) +# http_proxy=http://gateway.local:8118 +# Access control for hidden service. +# ALLOW_ACCESS_TO_HIDDEN_SERVICE=true +# If you use transparent proxy to access to hidden service, uncomment following for skipping private address check. +# HIDDEN_SERVICE_VIA_TRANSPARENT_PROXY=true diff --git a/app/lib/request.rb b/app/lib/request.rb index dca93a6e9..0acd654da 100644 --- a/app/lib/request.rb +++ b/app/lib/request.rb @@ -11,9 +11,10 @@ class Request def initialize(verb, url, **options) @verb = verb @url = Addressable::URI.parse(url).normalize - @options = options.merge(socket_class: Socket) + @options = options.merge(use_proxy? ? Rails.configuration.x.http_client_proxy : { socket_class: Socket }) @headers = {} + raise Mastodon::HostValidationError, 'Instance does not support hidden service connections' if block_hidden_service? set_common_headers! set_digest! if options.key?(:body) end @@ -99,6 +100,14 @@ class Request @http_client ||= HTTP.timeout(:per_operation, timeout).follow(max_hops: 2) end + def use_proxy? + Rails.configuration.x.http_client_proxy.present? + end + + def block_hidden_service? + !Rails.configuration.x.access_to_hidden_service && /\.(onion|i2p)$/.match(@url.host) + end + module ClientLimit def body_with_limit(limit = 1.megabyte) raise Mastodon::LengthValidationError if content_length.present? && content_length > limit @@ -129,6 +138,7 @@ class Request class Socket < TCPSocket class << self def open(host, *args) + return super host, *args if thru_hidden_service? host outer_e = nil Addrinfo.foreach(host, nil, nil, :SOCK_STREAM) do |address| begin @@ -142,6 +152,10 @@ class Request end alias new open + + def thru_hidden_service?(host) + Rails.configuration.x.hidden_service_via_transparent_proxy && /\.(onion|i2p)$/.match(host) + end end end diff --git a/config/initializers/http_client_proxy.rb b/config/initializers/http_client_proxy.rb new file mode 100644 index 000000000..f5026d59e --- /dev/null +++ b/config/initializers/http_client_proxy.rb @@ -0,0 +1,24 @@ +Rails.application.configure do + config.x.http_client_proxy = {} + if ENV['http_proxy'].present? + proxy = URI.parse(ENV['http_proxy']) + raise "Unsupported proxy type: #{proxy.scheme}" unless %w(http https).include? proxy.scheme + raise "No proxy host" unless proxy.host + + host = proxy.host + host = host[1...-1] if host[0] == '[' #for IPv6 address + config.x.http_client_proxy[:proxy] = { proxy_address: host, proxy_port: proxy.port, proxy_username: proxy.user, proxy_password: proxy.password }.compact + end + + config.x.access_to_hidden_service = ENV['ALLOW_ACCESS_TO_HIDDEN_SERVICE'] == 'true' + config.x.hidden_service_via_transparent_proxy = ENV['HIDDEN_SERVICE_VIA_TRANSPARENT_PROXY'] == 'true' +end + +module Goldfinger + def self.finger(uri, opts = {}) + to_hidden = /\.(onion|i2p)(:\d+)?$/.match(uri) + raise Mastodon::HostValidationError, 'Instance does not support hidden service connections' if !Rails.configuration.x.access_to_hidden_service && to_hidden + opts = opts.merge(Rails.configuration.x.http_client_proxy).merge(ssl: !to_hidden) + Goldfinger::Client.new(uri, opts).finger + end +end -- cgit From a872392cd958167d5d9dd3fef613415cc9068774 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Fri, 27 Apr 2018 01:38:10 +0200 Subject: Add entity cache (#7271) * Add entity cache Use a caching layer for mentions and custom emojis that are dynamically extracted from text. Reduce duplicate text extractions * Fix code style issue --- app/lib/entity_cache.rb | 34 ++++++++++++++++++++++++++++++++++ app/lib/formatter.rb | 10 +++------- app/models/account.rb | 2 +- app/models/custom_emoji.rb | 10 +++++++++- app/models/status.rb | 2 +- 5 files changed, 48 insertions(+), 10 deletions(-) create mode 100644 app/lib/entity_cache.rb (limited to 'app/lib') diff --git a/app/lib/entity_cache.rb b/app/lib/entity_cache.rb new file mode 100644 index 000000000..0c4edbfab --- /dev/null +++ b/app/lib/entity_cache.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +require 'singleton' + +class EntityCache + include Singleton + + MAX_EXPIRATION = 7.days.freeze + + def mention(username, domain) + Rails.cache.fetch(to_key(:mention, username, domain), expires_in: MAX_EXPIRATION) { Account.select(:username, :domain, :url).find_remote(username, domain) } + end + + def emoji(shortcodes, domain) + shortcodes = [shortcodes] unless shortcodes.is_a?(Array) + cached = Rails.cache.read_multi(*shortcodes.map { |shortcode| to_key(:emoji, shortcode, domain) }) + uncached_ids = [] + + shortcodes.each do |shortcode| + uncached_ids << shortcode unless cached.key?(to_key(:emoji, shortcode, domain)) + end + + unless uncached_ids.empty? + uncached = CustomEmoji.where(shortcode: shortcodes, domain: domain, disabled: false).select(:shortcode, :id, :image_file_name, :visible_in_picker).map { |item| [item.shortcode, item] }.to_h + uncached.each_value { |item| Rails.cache.write(to_key(:emoji, item.shortcode, domain), item, expires_in: MAX_EXPIRATION) } + end + + shortcodes.map { |shortcode| cached[to_key(:emoji, shortcode, domain)] || uncached[shortcode] }.compact + end + + def to_key(type, *ids) + "#{type}:#{ids.compact.map(&:downcase).join(':')}" + end +end diff --git a/app/lib/formatter.rb b/app/lib/formatter.rb index 4124f1660..050c651ee 100644 --- a/app/lib/formatter.rb +++ b/app/lib/formatter.rb @@ -52,12 +52,8 @@ class Formatter end def simplified_format(account, **options) - html = if account.local? - linkify(account.note) - else - reformat(account.note) - end - html = encode_custom_emojis(html, CustomEmoji.from_text(account.note, account.domain)) if options[:custom_emojify] + html = account.local? ? linkify(account.note) : reformat(account.note) + html = encode_custom_emojis(html, account.emojis) if options[:custom_emojify] html.html_safe # rubocop:disable Rails/OutputSafety end @@ -211,7 +207,7 @@ class Formatter username, domain = acct.split('@') domain = nil if TagManager.instance.local_domain?(domain) - account = Account.find_remote(username, domain) + account = EntityCache.instance.mention(username, domain) account ? mention_html(account) : "@#{acct}" end diff --git a/app/models/account.rb b/app/models/account.rb index ee47f04af..647b5c358 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -391,7 +391,7 @@ class Account < ApplicationRecord end def emojis - CustomEmoji.from_text(note, domain) + @emojis ||= CustomEmoji.from_text(note, domain) end before_create :generate_keys diff --git a/app/models/custom_emoji.rb b/app/models/custom_emoji.rb index 8235332f1..b99ed01f0 100644 --- a/app/models/custom_emoji.rb +++ b/app/models/custom_emoji.rb @@ -42,6 +42,8 @@ class CustomEmoji < ApplicationRecord include Attachmentable + after_commit :remove_entity_cache + def local? domain.nil? end @@ -58,11 +60,17 @@ class CustomEmoji < ApplicationRecord return [] if shortcodes.empty? - where(shortcode: shortcodes, domain: domain, disabled: false) + EntityCache.instance.emoji(shortcodes, domain) end def search(shortcode) where('"custom_emojis"."shortcode" ILIKE ?', "%#{shortcode}%") end end + + private + + def remove_entity_cache + Rails.cache.delete(EntityCache.instance.to_key(:emoji, shortcode, domain)) + end end diff --git a/app/models/status.rb b/app/models/status.rb index 37f2db562..fbb1f89aa 100644 --- a/app/models/status.rb +++ b/app/models/status.rb @@ -160,7 +160,7 @@ class Status < ApplicationRecord end def emojis - CustomEmoji.from_text([spoiler_text, text].join(' '), account.domain) + @emojis ||= CustomEmoji.from_text([spoiler_text, text].join(' '), account.domain) end after_create_commit :store_uri, if: :local? -- cgit From 295e3ef02bb3fcdd4d8992ad6105c0ada2b3db0c Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Mon, 30 Apr 2018 09:12:55 +0200 Subject: Fix missing domain attribute in EntityCache for emoji (#7290) --- app/lib/entity_cache.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/lib') diff --git a/app/lib/entity_cache.rb b/app/lib/entity_cache.rb index 0c4edbfab..03bfb7cf0 100644 --- a/app/lib/entity_cache.rb +++ b/app/lib/entity_cache.rb @@ -21,7 +21,7 @@ class EntityCache end unless uncached_ids.empty? - uncached = CustomEmoji.where(shortcode: shortcodes, domain: domain, disabled: false).select(:shortcode, :id, :image_file_name, :visible_in_picker).map { |item| [item.shortcode, item] }.to_h + uncached = CustomEmoji.where(shortcode: shortcodes, domain: domain, disabled: false).select(:shortcode, :id, :domain, :image_file_name, :visible_in_picker).map { |item| [item.shortcode, item] }.to_h uncached.each_value { |item| Rails.cache.write(to_key(:emoji, item.shortcode, domain), item, expires_in: MAX_EXPIRATION) } end -- cgit From 705f1d7bf15b7dc46256ab4a3bfff4075c79a8e7 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Mon, 30 Apr 2018 22:49:33 +0200 Subject: Fix missing updated_at attribute on emoji EntityCache (#7297) Just don't try to save space by only selecting few attributes. If anyone is wondering, this is needed because the emoji entity cache is not really only used for entities, it's accessed again to generate Emoji tags in ActivityPub/OStatus, so a lot more properties are used than what is needed in HTML alone... --- app/lib/entity_cache.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/lib') diff --git a/app/lib/entity_cache.rb b/app/lib/entity_cache.rb index 03bfb7cf0..2aa37389c 100644 --- a/app/lib/entity_cache.rb +++ b/app/lib/entity_cache.rb @@ -21,7 +21,7 @@ class EntityCache end unless uncached_ids.empty? - uncached = CustomEmoji.where(shortcode: shortcodes, domain: domain, disabled: false).select(:shortcode, :id, :domain, :image_file_name, :visible_in_picker).map { |item| [item.shortcode, item] }.to_h + uncached = CustomEmoji.where(shortcode: shortcodes, domain: domain, disabled: false).map { |item| [item.shortcode, item] }.to_h uncached.each_value { |item| Rails.cache.write(to_key(:emoji, item.shortcode, domain), item, expires_in: MAX_EXPIRATION) } end -- cgit From dc786c0cf4467ade8db7d8b17e09f16923bfc1e8 Mon Sep 17 00:00:00 2001 From: Surinna Curtis Date: Wed, 2 May 2018 05:40:24 -0500 Subject: Support Actors/Statuses with multiple types (#7305) * Add equals_or_includes_any? helper in JsonLdHelper * Support arrays in JSON-LD type fields for actors/tags/objects. * Spec for resolving accounts with extension types * Style tweaks for codeclimate --- app/helpers/jsonld_helper.rb | 4 ++++ app/lib/activitypub/activity/create.rb | 11 +++++------ app/lib/activitypub/activity/update.rb | 5 +---- app/services/activitypub/fetch_remote_account_service.rb | 2 +- app/services/activitypub/fetch_remote_key_service.rb | 4 ++-- app/services/activitypub/fetch_remote_status_service.rb | 2 +- app/services/activitypub/process_account_service.rb | 5 +---- app/services/fetch_atom_service.rb | 4 ++-- app/services/resolve_account_service.rb | 2 +- app/services/resolve_url_service.rb | 5 ++--- spec/fixtures/requests/activitypub-actor-individual.txt | 9 +++++++++ spec/services/resolve_account_service_spec.rb | 14 ++++++++++++++ 12 files changed, 43 insertions(+), 24 deletions(-) create mode 100644 spec/fixtures/requests/activitypub-actor-individual.txt (limited to 'app/lib') diff --git a/app/helpers/jsonld_helper.rb b/app/helpers/jsonld_helper.rb index dfb8fcb8b..a3cfdadb8 100644 --- a/app/helpers/jsonld_helper.rb +++ b/app/helpers/jsonld_helper.rb @@ -5,6 +5,10 @@ module JsonLdHelper haystack.is_a?(Array) ? haystack.include?(needle) : haystack == needle end + def equals_or_includes_any?(haystack, needles) + needles.any? { |needle| equals_or_includes?(haystack, needle) } + end + def first_of_value(value) value.is_a?(Array) ? value.first : value end diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index 45c0e91cb..411286fa5 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -61,12 +61,11 @@ class ActivityPub::Activity::Create < ActivityPub::Activity return if @object['tag'].nil? as_array(@object['tag']).each do |tag| - case tag['type'] - when 'Hashtag' + if equals_or_includes?(tag['type'], 'Hashtag') process_hashtag tag, status - when 'Mention' + elsif equals_or_includes?(tag['type'], 'Mention') process_mention tag, status - when 'Emoji' + elsif equals_or_includes?(tag['type'], 'Emoji') process_emoji tag, status end end @@ -235,11 +234,11 @@ class ActivityPub::Activity::Create < ActivityPub::Activity end def supported_object_type? - SUPPORTED_TYPES.include?(@object['type']) + equals_or_includes_any?(@object['type'], SUPPORTED_TYPES) end def converted_object_type? - CONVERTED_TYPES.include?(@object['type']) + equals_or_includes_any?(@object['type'], CONVERTED_TYPES) end def skip_download? diff --git a/app/lib/activitypub/activity/update.rb b/app/lib/activitypub/activity/update.rb index 0134b4015..47e98e041 100644 --- a/app/lib/activitypub/activity/update.rb +++ b/app/lib/activitypub/activity/update.rb @@ -2,10 +2,7 @@ class ActivityPub::Activity::Update < ActivityPub::Activity def perform - case @object['type'] - when 'Person' - update_account - end + update_account if equals_or_includes?(@object['type'], 'Person') end private diff --git a/app/services/activitypub/fetch_remote_account_service.rb b/app/services/activitypub/fetch_remote_account_service.rb index 5024853ca..867e70876 100644 --- a/app/services/activitypub/fetch_remote_account_service.rb +++ b/app/services/activitypub/fetch_remote_account_service.rb @@ -56,6 +56,6 @@ class ActivityPub::FetchRemoteAccountService < BaseService end def expected_type? - SUPPORTED_TYPES.include?(@json['type']) + equals_or_includes_any?(@json['type'], SUPPORTED_TYPES) end end diff --git a/app/services/activitypub/fetch_remote_key_service.rb b/app/services/activitypub/fetch_remote_key_service.rb index 41837d462..505baccd4 100644 --- a/app/services/activitypub/fetch_remote_key_service.rb +++ b/app/services/activitypub/fetch_remote_key_service.rb @@ -43,7 +43,7 @@ class ActivityPub::FetchRemoteKeyService < BaseService end def person? - ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES.include?(@json['type']) + equals_or_includes_any?(@json['type'], ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES) end def public_key? @@ -55,6 +55,6 @@ class ActivityPub::FetchRemoteKeyService < BaseService end def confirmed_owner? - ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES.include?(@owner['type']) && value_or_id(@owner['publicKey']) == @json['id'] + equals_or_includes_any?(@owner['type'], ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES) && value_or_id(@owner['publicKey']) == @json['id'] end end diff --git a/app/services/activitypub/fetch_remote_status_service.rb b/app/services/activitypub/fetch_remote_status_service.rb index 503c175d8..930fbad1f 100644 --- a/app/services/activitypub/fetch_remote_status_service.rb +++ b/app/services/activitypub/fetch_remote_status_service.rb @@ -42,7 +42,7 @@ class ActivityPub::FetchRemoteStatusService < BaseService end def expected_type? - (ActivityPub::Activity::Create::SUPPORTED_TYPES + ActivityPub::Activity::Create::CONVERTED_TYPES).include? @json['type'] + equals_or_includes_any?(@json['type'], ActivityPub::Activity::Create::SUPPORTED_TYPES + ActivityPub::Activity::Create::CONVERTED_TYPES) end def needs_update(actor) diff --git a/app/services/activitypub/process_account_service.rb b/app/services/activitypub/process_account_service.rb index da32f9615..f67ebb443 100644 --- a/app/services/activitypub/process_account_service.rb +++ b/app/services/activitypub/process_account_service.rb @@ -201,10 +201,7 @@ class ActivityPub::ProcessAccountService < BaseService return if @json['tag'].blank? as_array(@json['tag']).each do |tag| - case tag['type'] - when 'Emoji' - process_emoji tag - end + process_emoji tag if equals_or_includes?(tag['type'], 'Emoji') end end diff --git a/app/services/fetch_atom_service.rb b/app/services/fetch_atom_service.rb index 0444baf74..550e75f33 100644 --- a/app/services/fetch_atom_service.rb +++ b/app/services/fetch_atom_service.rb @@ -42,7 +42,7 @@ class FetchAtomService < BaseService elsif ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(response.mime_type) body = response.body_with_limit json = body_to_json(body) - if supported_context?(json) && ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES.include?(json['type']) && json['inbox'].present? + if supported_context?(json) && equals_or_includes_any?(json['type'], ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES) && json['inbox'].present? [json['id'], { prefetched_body: body, id: true }, :activitypub] elsif supported_context?(json) && expected_type?(json) [json['id'], { prefetched_body: body, id: true }, :activitypub] @@ -62,7 +62,7 @@ class FetchAtomService < BaseService end def expected_type?(json) - (ActivityPub::Activity::Create::SUPPORTED_TYPES + ActivityPub::Activity::Create::CONVERTED_TYPES).include? json['type'] + equals_or_includes_any?(json['type'], ActivityPub::Activity::Create::SUPPORTED_TYPES + ActivityPub::Activity::Create::CONVERTED_TYPES) end def process_html(response) diff --git a/app/services/resolve_account_service.rb b/app/services/resolve_account_service.rb index 8cba88f01..de8d1151d 100644 --- a/app/services/resolve_account_service.rb +++ b/app/services/resolve_account_service.rb @@ -189,7 +189,7 @@ class ResolveAccountService < BaseService return @actor_json if defined?(@actor_json) json = fetch_resource(actor_url, false) - @actor_json = supported_context?(json) && ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES.include?(json['type']) ? json : nil + @actor_json = supported_context?(json) && equals_or_includes_any?(json['type'], ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES) ? json : nil end def atom diff --git a/app/services/resolve_url_service.rb b/app/services/resolve_url_service.rb index c19b568cb..a068c1ed8 100644 --- a/app/services/resolve_url_service.rb +++ b/app/services/resolve_url_service.rb @@ -16,10 +16,9 @@ class ResolveURLService < BaseService private def process_url - case type - when 'Application', 'Group', 'Organization', 'Person', 'Service' + if equals_or_includes_any?(type, %w(Application Group Organization Person Service)) FetchRemoteAccountService.new.call(atom_url, body, protocol) - when 'Note', 'Article', 'Image', 'Video' + elsif equals_or_includes_any?(type, %w(Note Article Image Video)) FetchRemoteStatusService.new.call(atom_url, body, protocol) end end diff --git a/spec/fixtures/requests/activitypub-actor-individual.txt b/spec/fixtures/requests/activitypub-actor-individual.txt new file mode 100644 index 000000000..74411e544 --- /dev/null +++ b/spec/fixtures/requests/activitypub-actor-individual.txt @@ -0,0 +1,9 @@ +HTTP/1.1 200 OK +Cache-Control: max-age=0, private, must-revalidate +Content-Type: application/activity+json; charset=utf-8 +Link: ; rel="lrdd"; type="application/xrd+xml", ; rel="alternate"; type="application/atom+xml", ; rel="alternate"; type="application/activity+json" +Vary: Accept-Encoding +X-Content-Type-Options: nosniff +X-Xss-Protection: 1; mode=block + +{"@context":["https://www.w3.org/ns/activitystreams","https://w3id.org/security/v1",{"vcard": "http://www.w3.org/2006/vcard/ns#"},{"manuallyApprovesFollowers":"as:manuallyApprovesFollowers","sensitive":"as:sensitive","Hashtag":"as:Hashtag","ostatus":"http://ostatus.org#","atomUri":"ostatus:atomUri","inReplyToAtomUri":"ostatus:inReplyToAtomUri","conversation":"ostatus:conversation"}],"id":"https://ap.example.com/users/foo","type":["Person","vcard:individual"],"following":"https://ap.example.com/users/foo/following","followers":"https://ap.example.com/users/foo/followers","inbox":"https://ap.example.com/users/foo/inbox","outbox":"https://ap.example.com/users/foo/outbox","preferredUsername":"foo","vcard:fn":"foo","name":"","summary":"\u003cp\u003etest\u003c/p\u003e","url":"https://ap.example.com/@foo","manuallyApprovesFollowers":false,"publicKey":{"id":"https://ap.example.com/users/foo#main-key","owner":"https://ap.example.com/users/foo","publicKeyPem":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAu3L4vnpNLzVH31MeWI39\n4F0wKeJFsLDAsNXGeOu0QF2x+h1zLWZw/agqD2R3JPU9/kaDJGPIV2Sn5zLyUA9S\n6swCCMOtn7BBR9g9sucgXJmUFB0tACH2QSgHywMAybGfmSb3LsEMNKsGJ9VsvYoh\n8lDET6X4Pyw+ZJU0/OLo/41q9w+OrGtlsTm/PuPIeXnxa6BLqnDaxC+4IcjG/FiP\nahNCTINl/1F/TgSSDZ4Taf4U9XFEIFw8wmgploELozzIzKq+t8nhQYkgAkt64euW\npva3qL5KD1mTIZQEP+LZvh3s2WHrLi3fhbdRuwQ2c0KkJA2oSTFPDpqqbPGZ3Qvu\nHQIDAQAB\n-----END PUBLIC KEY-----\n"},"endpoints":{"sharedInbox":"https://ap.example.com/inbox"},"icon":{"type":"Image","url":"https://quitter.no/avatar/7477-300-20160211190340.png"}} \ No newline at end of file diff --git a/spec/services/resolve_account_service_spec.rb b/spec/services/resolve_account_service_spec.rb index 5f1b4467b..84dfe578a 100644 --- a/spec/services/resolve_account_service_spec.rb +++ b/spec/services/resolve_account_service_spec.rb @@ -105,6 +105,20 @@ RSpec.describe ResolveAccountService do expect(account.inbox_url).to eq 'https://ap.example.com/users/foo/inbox' end + context 'with multiple types' do + before do + stub_request(:get, "https://ap.example.com/users/foo").to_return(request_fixture('activitypub-actor-individual.txt')) + end + + it 'returns new remote account' do + account = subject.call('foo@ap.example.com') + + expect(account.activitypub?).to eq true + expect(account.domain).to eq 'ap.example.com' + expect(account.inbox_url).to eq 'https://ap.example.com/users/foo/inbox' + end + end + pending end -- cgit From 965345316fb3fef640a6bcc463d09d4a38b28608 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Wed, 2 May 2018 15:44:22 +0200 Subject: Guard against nil URLs in Request class (#7284) Fix #7265 --- app/lib/request.rb | 3 +++ app/services/activitypub/fetch_featured_collection_service.rb | 2 ++ 2 files changed, 5 insertions(+) (limited to 'app/lib') diff --git a/app/lib/request.rb b/app/lib/request.rb index 0acd654da..00f94dacf 100644 --- a/app/lib/request.rb +++ b/app/lib/request.rb @@ -9,12 +9,15 @@ class Request include RoutingHelper def initialize(verb, url, **options) + raise ArgumentError if url.blank? + @verb = verb @url = Addressable::URI.parse(url).normalize @options = options.merge(use_proxy? ? Rails.configuration.x.http_client_proxy : { socket_class: Socket }) @headers = {} raise Mastodon::HostValidationError, 'Instance does not support hidden service connections' if block_hidden_service? + set_common_headers! set_digest! if options.key?(:body) end diff --git a/app/services/activitypub/fetch_featured_collection_service.rb b/app/services/activitypub/fetch_featured_collection_service.rb index 40714e980..6a137b520 100644 --- a/app/services/activitypub/fetch_featured_collection_service.rb +++ b/app/services/activitypub/fetch_featured_collection_service.rb @@ -4,6 +4,8 @@ class ActivityPub::FetchFeaturedCollectionService < BaseService include JsonLdHelper def call(account) + return if account.featured_collection_url.blank? + @account = account @json = fetch_resource(@account.featured_collection_url, true) -- cgit From f62539ce5c106e27a371702d499ec4df52eccde6 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Wed, 2 May 2018 15:50:20 +0200 Subject: Remove most behaviour disparities between blocks and mutes (#7231) * Remove most behaviour disparities between blocks and mutes The only differences between block and mute should be: - Mutes can optionally NOT affect notifications - Mutes should not be visible to the muted Fix #7230 Fix #5713 * Do not allow boosting someone you blocked Fix #7248 * Do not allow favouriting someone you blocked * Fix nil error in StatusPolicy --- app/lib/feed_manager.rb | 21 ++++++++++++--------- app/policies/status_policy.rb | 6 +++++- app/services/favourite_service.rb | 2 +- 3 files changed, 18 insertions(+), 11 deletions(-) (limited to 'app/lib') diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb index dd78e543f..c18c07b33 100644 --- a/app/lib/feed_manager.rb +++ b/app/lib/feed_manager.rb @@ -145,19 +145,20 @@ class FeedManager redis.exists("subscribed:#{timeline_id}") end + def blocks_or_mutes?(receiver_id, account_ids, context) + Block.where(account_id: receiver_id, target_account_id: account_ids).any? || + (context == :home ? Mute.where(account_id: receiver_id, target_account_id: account_ids).any? : Mute.where(account_id: receiver_id, target_account_id: account_ids, hide_notifications: true).any?) + end + def filter_from_home?(status, receiver_id) return false if receiver_id == status.account_id return true if status.reply? && (status.in_reply_to_id.nil? || status.in_reply_to_account_id.nil?) - check_for_mutes = [status.account_id] - check_for_mutes.concat([status.reblog.account_id]) if status.reblog? - - return true if Mute.where(account_id: receiver_id, target_account_id: check_for_mutes).any? - check_for_blocks = status.mentions.pluck(:account_id) + check_for_blocks.concat([status.account_id]) check_for_blocks.concat([status.reblog.account_id]) if status.reblog? - return true if Block.where(account_id: receiver_id, target_account_id: check_for_blocks).any? + return true if blocks_or_mutes?(receiver_id, check_for_blocks, :home) if status.reply? && !status.in_reply_to_account_id.nil? # Filter out if it's a reply should_filter = !Follow.where(account_id: receiver_id, target_account_id: status.in_reply_to_account_id).exists? # and I'm not following the person it's a reply to @@ -177,11 +178,13 @@ class FeedManager def filter_from_mentions?(status, receiver_id) return true if receiver_id == status.account_id - check_for_blocks = [status.account_id] - check_for_blocks.concat(status.mentions.pluck(:account_id)) + # This filter is called from NotifyService, but already after the sender of + # the notification has been checked for mute/block. Therefore, it's not + # necessary to check the author of the toot for mute/block again + check_for_blocks = status.mentions.pluck(:account_id) check_for_blocks.concat([status.in_reply_to_account]) if status.reply? && !status.in_reply_to_account_id.nil? - should_filter = Block.where(account_id: receiver_id, target_account_id: check_for_blocks).any? # Filter if it's from someone I blocked, in reply to someone I blocked, or mentioning someone I blocked + should_filter = blocks_or_mutes?(receiver_id, check_for_blocks, :mentions) # Filter if it's from someone I blocked, in reply to someone I blocked, or mentioning someone I blocked (or muted) should_filter ||= (status.account.silenced? && !Follow.where(account_id: receiver_id, target_account_id: status.account_id).exists?) # of if the account is silenced and I'm not following them should_filter diff --git a/app/policies/status_policy.rb b/app/policies/status_policy.rb index 5573289b6..4145d7e9c 100644 --- a/app/policies/status_policy.rb +++ b/app/policies/status_policy.rb @@ -16,7 +16,11 @@ class StatusPolicy < ApplicationPolicy end def reblog? - !direct? && (!private? || owned?) && show? + !direct? && (!private? || owned?) && show? && !current_account&.blocking?(author) + end + + def favourite? + show? && !current_account&.blocking?(author) end def destroy? diff --git a/app/services/favourite_service.rb b/app/services/favourite_service.rb index 44df3ed13..bc2d1547a 100644 --- a/app/services/favourite_service.rb +++ b/app/services/favourite_service.rb @@ -8,7 +8,7 @@ class FavouriteService < BaseService # @param [Status] status # @return [Favourite] def call(account, status) - authorize_with account, status, :show? + authorize_with account, status, :favourite? favourite = Favourite.find_by(account: account, status: status) -- cgit From d0cdd5cf94ff479e4037dc47539f7f9c408831b3 Mon Sep 17 00:00:00 2001 From: ThibG Date: Wed, 2 May 2018 16:08:16 +0200 Subject: Accept actor object updates from all supported actor types (#7312) --- app/lib/activitypub/activity/update.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'app/lib') diff --git a/app/lib/activitypub/activity/update.rb b/app/lib/activitypub/activity/update.rb index 47e98e041..aa5907f03 100644 --- a/app/lib/activitypub/activity/update.rb +++ b/app/lib/activitypub/activity/update.rb @@ -1,8 +1,10 @@ # frozen_string_literal: true class ActivityPub::Activity::Update < ActivityPub::Activity + SUPPORTED_TYPES = %w(Application Group Organization Person Service).freeze + def perform - update_account if equals_or_includes?(@object['type'], 'Person') + update_account if equals_or_includes_any?(@object['type'], SUPPORTED_TYPES) end private -- cgit From cb5b5cb5f79bb2187d8124df91af4c8e1bfd7256 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Wed, 2 May 2018 18:58:48 +0200 Subject: Slightly reduce RAM usage (#7301) * No need to re-require sidekiq plugins, they are required via Gemfile * Add derailed_benchmarks tool, no need to require TTY gems in Gemfile * Replace ruby-oembed with FetchOEmbedService Reduce startup by 45382 allocated objects * Remove preloaded JSON-LD in favour of caching HTTP responses Reduce boot RAM by about 6 MiB * Fix tests * Fix test suite by stubbing out JSON-LD contexts --- Gemfile | 12 +- Gemfile.lock | 22 +- app/controllers/api/web/embeds_controller.rb | 11 +- .../settings/follower_domains_controller.rb | 2 - app/helpers/jsonld_helper.rb | 17 +- app/lib/provider_discovery.rb | 47 --- app/services/fan_out_on_write_service.rb | 2 - app/services/fetch_link_card_service.rb | 38 +- app/services/fetch_oembed_service.rb | 71 ++++ app/workers/scheduler/backup_cleanup_scheduler.rb | 1 - .../scheduler/doorkeeper_cleanup_scheduler.rb | 1 - app/workers/scheduler/email_scheduler.rb | 1 - app/workers/scheduler/feed_cleanup_scheduler.rb | 1 - app/workers/scheduler/ip_cleanup_scheduler.rb | 1 - app/workers/scheduler/media_cleanup_scheduler.rb | 1 - .../scheduler/subscriptions_cleanup_scheduler.rb | 2 - app/workers/scheduler/subscriptions_scheduler.rb | 3 - app/workers/scheduler/user_cleanup_scheduler.rb | 1 - app/workers/soft_block_domain_followers_worker.rb | 2 - config/initializers/json_ld.rb | 5 - config/initializers/oembed.rb | 4 - lib/json_ld/activitystreams.rb | 153 -------- lib/json_ld/identity.rb | 86 ----- lib/json_ld/security.rb | 50 --- lib/tasks/mastodon.rake | 2 + spec/fixtures/requests/json-ld.activitystreams.txt | 391 +++++++++++++++++++++ spec/fixtures/requests/json-ld.identity.txt | 100 ++++++ spec/fixtures/requests/json-ld.security.txt | 61 ++++ spec/lib/activitypub/linked_data_signature_spec.rb | 4 + spec/lib/provider_discovery_spec.rb | 118 ------- spec/rails_helper.rb | 14 + spec/services/account_search_service_spec.rb | 2 +- .../fetch_remote_account_service_spec.rb | 2 +- .../fetch_remote_status_service_spec.rb | 2 +- .../activitypub/process_account_service_spec.rb | 2 +- .../activitypub/process_collection_service_spec.rb | 2 +- spec/services/after_block_service_spec.rb | 2 +- spec/services/authorize_follow_service_spec.rb | 2 +- .../services/batched_remove_status_service_spec.rb | 2 +- .../block_domain_from_account_service_spec.rb | 2 +- spec/services/block_domain_service_spec.rb | 2 +- spec/services/block_service_spec.rb | 2 +- spec/services/bootstrap_timeline_service_spec.rb | 2 +- spec/services/fan_out_on_write_service_spec.rb | 2 +- spec/services/favourite_service_spec.rb | 2 +- spec/services/fetch_atom_service_spec.rb | 2 +- spec/services/fetch_link_card_service_spec.rb | 2 +- spec/services/fetch_oembed_service_spec.rb | 125 +++++++ spec/services/fetch_remote_account_service_spec.rb | 2 +- spec/services/fetch_remote_status_service_spec.rb | 2 +- spec/services/follow_service_spec.rb | 2 +- spec/services/mute_service_spec.rb | 2 +- spec/services/notify_service_spec.rb | 2 +- spec/services/post_status_service_spec.rb | 2 +- spec/services/precompute_feed_service_spec.rb | 2 +- spec/services/process_feed_service_spec.rb | 2 +- spec/services/process_interaction_service_spec.rb | 2 +- spec/services/process_mentions_service_spec.rb | 2 +- .../pubsubhubbub/subscribe_service_spec.rb | 2 +- .../pubsubhubbub/unsubscribe_service_spec.rb | 2 +- spec/services/reblog_service_spec.rb | 2 +- spec/services/reject_follow_service_spec.rb | 2 +- spec/services/remove_status_service_spec.rb | 2 +- spec/services/report_service_spec.rb | 2 +- spec/services/resolve_account_service_spec.rb | 2 +- spec/services/resolve_url_service_spec.rb | 2 +- spec/services/search_service_spec.rb | 2 +- spec/services/send_interaction_service_spec.rb | 2 +- spec/services/subscribe_service_spec.rb | 2 +- spec/services/suspend_account_service_spec.rb | 2 +- spec/services/unblock_domain_service_spec.rb | 2 +- spec/services/unblock_service_spec.rb | 2 +- spec/services/unfollow_service_spec.rb | 2 +- spec/services/unmute_service_spec.rb | 2 +- spec/services/unsubscribe_service_spec.rb | 2 +- .../services/update_remote_profile_service_spec.rb | 2 +- spec/spec_helper.rb | 12 +- 77 files changed, 881 insertions(+), 568 deletions(-) delete mode 100644 app/lib/provider_discovery.rb create mode 100644 app/services/fetch_oembed_service.rb delete mode 100644 config/initializers/json_ld.rb delete mode 100644 config/initializers/oembed.rb delete mode 100644 lib/json_ld/activitystreams.rb delete mode 100644 lib/json_ld/identity.rb delete mode 100644 lib/json_ld/security.rb create mode 100644 spec/fixtures/requests/json-ld.activitystreams.txt create mode 100644 spec/fixtures/requests/json-ld.identity.txt create mode 100644 spec/fixtures/requests/json-ld.security.txt delete mode 100644 spec/lib/provider_discovery_spec.rb create mode 100644 spec/services/fetch_oembed_service_spec.rb (limited to 'app/lib') diff --git a/Gemfile b/Gemfile index a33748568..f1665ce95 100644 --- a/Gemfile +++ b/Gemfile @@ -54,7 +54,7 @@ gem 'httplog', '~> 1.0' gem 'idn-ruby', require: 'idn' gem 'kaminari', '~> 1.1' gem 'link_header', '~> 0.0' -gem 'mime-types', '~> 3.1' +gem 'mime-types', '~> 3.1', require: 'mime/types/columnar' gem 'nokogiri', '~> 1.8' gem 'nsa', '~> 0.2' gem 'oj', '~> 3.5' @@ -70,7 +70,6 @@ gem 'rails-settings-cached', '~> 0.6' gem 'redis', '~> 4.0', require: ['redis', 'redis/connection/hiredis'] gem 'mario-redis-lock', '~> 1.2', require: 'redis_lock' gem 'rqrcode', '~> 0.10' -gem 'ruby-oembed', '~> 0.12', require: 'oembed' gem 'ruby-progressbar', '~> 1.4' gem 'sanitize', '~> 4.6' gem 'sidekiq', '~> 5.1' @@ -82,14 +81,14 @@ gem 'simple_form', '~> 4.0' gem 'sprockets-rails', '~> 3.2', require: 'sprockets/railtie' gem 'stoplight', '~> 2.1.3' gem 'strong_migrations', '~> 0.2' -gem 'tty-command', '~> 0.8' -gem 'tty-prompt', '~> 0.16' +gem 'tty-command', '~> 0.8', require: false +gem 'tty-prompt', '~> 0.16', require: false gem 'twitter-text', '~> 1.14' gem 'tzinfo-data', '~> 1.2018' gem 'webpacker', '~> 3.4' gem 'webpush' -gem 'json-ld-preloaded', '~> 2.2' +gem 'json-ld', '~> 2.2' gem 'rdf-normalize', '~> 0.3' group :development, :test do @@ -135,6 +134,9 @@ group :development do gem 'capistrano-rails', '~> 1.3' gem 'capistrano-rbenv', '~> 2.1' gem 'capistrano-yarn', '~> 2.0' + + gem 'derailed_benchmarks' + gem 'stackprof' end group :production do diff --git a/Gemfile.lock b/Gemfile.lock index d96165dcf..94ab0b7ca 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -75,6 +75,7 @@ GEM aws-sigv4 (~> 1.0) aws-sigv4 (1.0.2) bcrypt (3.1.11) + benchmark-ips (2.7.2) better_errors (2.4.0) coderay (>= 1.0.0) erubi (>= 1.0.0) @@ -138,6 +139,14 @@ GEM css_parser (1.6.0) addressable debug_inspector (0.0.3) + derailed_benchmarks (1.3.4) + benchmark-ips (~> 2) + get_process_mem (~> 0) + heapy (~> 0) + memory_profiler (~> 0) + rack (>= 1) + rake (> 10, < 13) + thor (~> 0.19) devise (4.4.3) bcrypt (~> 3.0) orm_adapter (~> 0.1) @@ -206,6 +215,7 @@ GEM fuubar (2.3.1) rspec-core (~> 3.0) ruby-progressbar (~> 1.4) + get_process_mem (0.2.1) globalid (0.4.1) activesupport (>= 4.2.0) goldfinger (2.1.0) @@ -226,6 +236,7 @@ GEM concurrent-ruby (~> 1.0) hashdiff (0.3.7) hashie (3.5.7) + heapy (0.1.3) highline (1.7.10) hiredis (0.6.1) hitimes (1.2.6) @@ -264,10 +275,6 @@ GEM json-ld (2.2.1) multi_json (~> 1.12) rdf (>= 2.2.8, < 4.0) - json-ld-preloaded (2.2.3) - json-ld (>= 2.2, < 4.0) - multi_json (~> 1.12) - rdf (>= 2.2, < 4.0) jsonapi-renderer (0.2.0) jwt (2.1.0) kaminari (1.1.1) @@ -502,7 +509,6 @@ GEM rainbow (>= 2.2.2, < 4.0) ruby-progressbar (~> 1.7) unicode-display_width (~> 1.0, >= 1.0.1) - ruby-oembed (0.12.0) ruby-progressbar (1.9.0) ruby-saml (1.7.2) nokogiri (>= 1.5.10) @@ -557,6 +563,7 @@ GEM sshkit (1.16.0) net-scp (>= 1.1.2) net-ssh (>= 2.8.0) + stackprof (0.2.11) statsd-ruby (1.2.1) stoplight (2.1.3) streamio-ffmpeg (3.0.2) @@ -645,6 +652,7 @@ DEPENDENCIES chewy (~> 5.0) cld3 (~> 3.2.0) climate_control (~> 0.2) + derailed_benchmarks devise (~> 4.4) devise-two-factor (~> 3.0) devise_pam_authenticatable2 (~> 9.1) @@ -668,7 +676,7 @@ DEPENDENCIES i18n-tasks (~> 0.9) idn-ruby iso-639 - json-ld-preloaded (~> 2.2) + json-ld (~> 2.2) kaminari (~> 1.1) letter_opener (~> 1.4) letter_opener_web (~> 1.3) @@ -714,7 +722,6 @@ DEPENDENCIES rspec-retry (~> 0.5) rspec-sidekiq (~> 3.0) rubocop (~> 0.55) - ruby-oembed (~> 0.12) ruby-progressbar (~> 1.4) sanitize (~> 4.6) scss_lint (~> 0.57) @@ -726,6 +733,7 @@ DEPENDENCIES simple_form (~> 4.0) simplecov (~> 0.16) sprockets-rails (~> 3.2) + stackprof stoplight (~> 2.1.3) streamio-ffmpeg (~> 3.0) strong_migrations (~> 0.2) diff --git a/app/controllers/api/web/embeds_controller.rb b/app/controllers/api/web/embeds_controller.rb index f2fe74b17..987290a14 100644 --- a/app/controllers/api/web/embeds_controller.rb +++ b/app/controllers/api/web/embeds_controller.rb @@ -9,9 +9,12 @@ class Api::Web::EmbedsController < Api::Web::BaseController status = StatusFinder.new(params[:url]).status render json: status, serializer: OEmbedSerializer, width: 400 rescue ActiveRecord::RecordNotFound - oembed = OEmbed::Providers.get(params[:url]) - render json: Oj.dump(oembed.fields) - rescue OEmbed::NotFound - render json: {}, status: :not_found + oembed = FetchOEmbedService.new.call(params[:url]) + + if oembed + render json: oembed + else + render json: {}, status: :not_found + end end end diff --git a/app/controllers/settings/follower_domains_controller.rb b/app/controllers/settings/follower_domains_controller.rb index 213d9e96d..91b521e7f 100644 --- a/app/controllers/settings/follower_domains_controller.rb +++ b/app/controllers/settings/follower_domains_controller.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require 'sidekiq-bulk' - class Settings::FollowerDomainsController < ApplicationController layout 'admin' diff --git a/app/helpers/jsonld_helper.rb b/app/helpers/jsonld_helper.rb index a3cfdadb8..e9056166c 100644 --- a/app/helpers/jsonld_helper.rb +++ b/app/helpers/jsonld_helper.rb @@ -48,7 +48,7 @@ module JsonLdHelper end def canonicalize(json) - graph = RDF::Graph.new << JSON::LD::API.toRdf(json) + graph = RDF::Graph.new << JSON::LD::API.toRdf(json, documentLoader: method(:load_jsonld_context)) graph.dump(:normalize) end @@ -90,4 +90,19 @@ module JsonLdHelper request.add_headers('Accept' => 'application/activity+json, application/ld+json') request end + + def load_jsonld_context(url, _options = {}, &_block) + json = Rails.cache.fetch("jsonld:context:#{url}", expires_in: 30.days, raw: true) do + request = Request.new(:get, url) + request.add_headers('Accept' => 'application/ld+json') + + request.perform do |res| + raise JSON::LD::JsonLdError::LoadingDocumentFailed unless res.code == 200 && res.mime_type == 'application/ld+json' + res.body_with_limit + end + end + + doc = JSON::LD::API::RemoteDocument.new(url, json) + block_given? ? yield(doc) : doc + end end diff --git a/app/lib/provider_discovery.rb b/app/lib/provider_discovery.rb deleted file mode 100644 index 3bec7211b..000000000 --- a/app/lib/provider_discovery.rb +++ /dev/null @@ -1,47 +0,0 @@ -# frozen_string_literal: true - -class ProviderDiscovery < OEmbed::ProviderDiscovery - class << self - def get(url, **options) - provider = discover_provider(url, options) - - options.delete(:html) - - provider.get(url, options) - end - - def discover_provider(url, **options) - format = options[:format] - - html = if options[:html] - Nokogiri::HTML(options[:html]) - else - Request.new(:get, url).perform do |res| - raise OEmbed::NotFound, url if res.code != 200 || res.mime_type != 'text/html' - Nokogiri::HTML(res.body_with_limit) - end - end - - if format.nil? || format == :json - provider_endpoint ||= html.at_xpath('//link[@type="application/json+oembed"]')&.attribute('href')&.value - format ||= :json if provider_endpoint - end - - if format.nil? || format == :xml - provider_endpoint ||= html.at_xpath('//link[@type="text/xml+oembed"]')&.attribute('href')&.value - format ||= :xml if provider_endpoint - end - - raise OEmbed::NotFound, url if provider_endpoint.nil? - begin - provider_endpoint = Addressable::URI.parse(provider_endpoint) - provider_endpoint.query = nil - provider_endpoint = provider_endpoint.to_s - rescue Addressable::URI::InvalidURIError - raise OEmbed::NotFound, url - end - - OEmbed::Provider.new(provider_endpoint, format) - end - end -end diff --git a/app/services/fan_out_on_write_service.rb b/app/services/fan_out_on_write_service.rb index 0f77556dc..510b80c82 100644 --- a/app/services/fan_out_on_write_service.rb +++ b/app/services/fan_out_on_write_service.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require 'sidekiq-bulk' - class FanOutOnWriteService < BaseService # Push a status into home and mentions feeds # @param [Status] status diff --git a/app/services/fetch_link_card_service.rb b/app/services/fetch_link_card_service.rb index d5920a417..77d4aa538 100644 --- a/app/services/fetch_link_card_service.rb +++ b/app/services/fetch_link_card_service.rb @@ -85,42 +85,40 @@ class FetchLinkCardService < BaseService end def attempt_oembed - embed = OEmbed::Providers.get(@url, html: @html) + embed = FetchOEmbedService.new.call(@url, html: @html) - return false unless embed.respond_to?(:type) + return false if embed.nil? - @card.type = embed.type - @card.title = embed.respond_to?(:title) ? embed.title : '' - @card.author_name = embed.respond_to?(:author_name) ? embed.author_name : '' - @card.author_url = embed.respond_to?(:author_url) ? embed.author_url : '' - @card.provider_name = embed.respond_to?(:provider_name) ? embed.provider_name : '' - @card.provider_url = embed.respond_to?(:provider_url) ? embed.provider_url : '' + @card.type = embed[:type] + @card.title = embed[:title] || '' + @card.author_name = embed[:author_name] || '' + @card.author_url = embed[:author_url] || '' + @card.provider_name = embed[:provider_name] || '' + @card.provider_url = embed[:provider_url] || '' @card.width = 0 @card.height = 0 case @card.type when 'link' - @card.image_remote_url = embed.thumbnail_url if embed.respond_to?(:thumbnail_url) + @card.image_remote_url = embed[:thumbnail_url] if embed[:thumbnail_url].present? when 'photo' - return false unless embed.respond_to?(:url) + return false if embed[:url].blank? - @card.embed_url = embed.url - @card.image_remote_url = embed.url - @card.width = embed.width.presence || 0 - @card.height = embed.height.presence || 0 + @card.embed_url = embed[:url] + @card.image_remote_url = embed[:url] + @card.width = embed[:width].presence || 0 + @card.height = embed[:height].presence || 0 when 'video' - @card.width = embed.width.presence || 0 - @card.height = embed.height.presence || 0 - @card.html = Formatter.instance.sanitize(embed.html, Sanitize::Config::MASTODON_OEMBED) - @card.image_remote_url = embed.thumbnail_url if embed.respond_to?(:thumbnail_url) + @card.width = embed[:width].presence || 0 + @card.height = embed[:height].presence || 0 + @card.html = Formatter.instance.sanitize(embed[:html], Sanitize::Config::MASTODON_OEMBED) + @card.image_remote_url = embed[:thumbnail_url] if embed[:thumbnail_url].present? when 'rich' # Most providers rely on