From a9c440637ca9f36bcf051094abe3bcba1da63166 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Fri, 20 Apr 2018 02:28:48 +0200 Subject: Improve report layout (#7188) * Use table for statuses in report * Display reported account and reporter in the same table * Split accounts and general report info into two tables again * Redesign report statuses table, notes, merge notes and action log * Remove unused translations * Fix code style issue * Fix code style issue * Fix code style issue --- app/helpers/admin/account_moderation_notes_helper.rb | 16 ++++++++++++++++ app/helpers/application_helper.rb | 4 ++++ app/helpers/stream_entries_helper.rb | 13 +++++++++++++ 3 files changed, 33 insertions(+) (limited to 'app/helpers') diff --git a/app/helpers/admin/account_moderation_notes_helper.rb b/app/helpers/admin/account_moderation_notes_helper.rb index b17c52264..fdfadef08 100644 --- a/app/helpers/admin/account_moderation_notes_helper.rb +++ b/app/helpers/admin/account_moderation_notes_helper.rb @@ -1,4 +1,20 @@ # frozen_string_literal: true module Admin::AccountModerationNotesHelper + def admin_account_link_to(account) + link_to admin_account_path(account.id), class: name_tag_classes(account) do + safe_join([ + image_tag(account.avatar.url, width: 15, height: 15, alt: display_name(account), class: 'avatar'), + content_tag(:span, account.acct, class: 'username'), + ], ' ') + end + end + + private + + def name_tag_classes(account) + classes = ['name-tag'] + classes << 'suspended' if account.suspended? + classes.join(' ') + end end diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb index bab4615a1..95863ab1f 100644 --- a/app/helpers/application_helper.rb +++ b/app/helpers/application_helper.rb @@ -63,4 +63,8 @@ module ApplicationHelper def opengraph(property, content) tag(:meta, content: content, property: property) end + + def react_component(name, props = {}) + content_tag(:div, nil, data: { component: name.to_s.camelcase, props: Oj.dump(props) }) + end end diff --git a/app/helpers/stream_entries_helper.rb b/app/helpers/stream_entries_helper.rb index 3992432db..8254ef4dc 100644 --- a/app/helpers/stream_entries_helper.rb +++ b/app/helpers/stream_entries_helper.rb @@ -113,6 +113,19 @@ module StreamEntriesHelper end end + def fa_visibility_icon(status) + case status.visibility + when 'public' + fa_icon 'globe fw' + when 'unlisted' + fa_icon 'unlock-alt fw' + when 'private' + fa_icon 'lock fw' + when 'direct' + fa_icon 'envelope fw' + end + end + private def simplified_text(text) -- cgit From 9d4710ed0059b2f789e6b32b9f81d4ce90b98907 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Wed, 25 Apr 2018 02:10:02 +0200 Subject: Add RSS feeds for end-users (#7259) * Add RSS feed for accounts * Add RSS feeds for hashtags * Fix code style issues * Fix code style issues --- app/controllers/accounts_controller.rb | 10 ++- app/controllers/tags_controller.rb | 11 ++- app/helpers/stream_entries_helper.rb | 12 +-- app/lib/rss_builder.rb | 130 ++++++++++++++++++++++++++++++ app/serializers/rss/account_serializer.rb | 39 +++++++++ app/serializers/rss/tag_serializer.rb | 37 +++++++++ 6 files changed, 230 insertions(+), 9 deletions(-) create mode 100644 app/lib/rss_builder.rb create mode 100644 app/serializers/rss/account_serializer.rb create mode 100644 app/serializers/rss/tag_serializer.rb (limited to 'app/helpers') diff --git a/app/controllers/accounts_controller.rb b/app/controllers/accounts_controller.rb index 7bf35825f..1152d4aca 100644 --- a/app/controllers/accounts_controller.rb +++ b/app/controllers/accounts_controller.rb @@ -20,9 +20,10 @@ class AccountsController < ApplicationController @pinned_statuses = cache_collection(@account.pinned_statuses, Status) if show_pinned_statuses? @statuses = filtered_status_page(params) @statuses = cache_collection(@statuses, Status) + unless @statuses.empty? - @older_url = older_url if @statuses.last.id > filtered_statuses.last.id - @newer_url = newer_url if @statuses.first.id < filtered_statuses.first.id + @older_url = older_url if @statuses.last.id > filtered_statuses.last.id + @newer_url = newer_url if @statuses.first.id < filtered_statuses.first.id end end @@ -31,6 +32,11 @@ class AccountsController < ApplicationController render xml: OStatus::AtomSerializer.render(OStatus::AtomSerializer.new.feed(@account, @entries.reject { |entry| entry.status.nil? })) end + format.rss do + @statuses = cache_collection(default_statuses.without_reblogs.without_replies.limit(PAGE_SIZE), Status) + render xml: RSS::AccountSerializer.render(@account, @statuses) + end + format.json do skip_session! diff --git a/app/controllers/tags_controller.rb b/app/controllers/tags_controller.rb index 9f3090e37..014a5c9b8 100644 --- a/app/controllers/tags_controller.rb +++ b/app/controllers/tags_controller.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class TagsController < ApplicationController + PAGE_SIZE = 20 + before_action :set_body_classes before_action :set_instance_presenter @@ -13,8 +15,15 @@ class TagsController < ApplicationController @initial_state_json = serializable_resource.to_json end + format.rss do + @statuses = Status.as_tag_timeline(@tag).limit(PAGE_SIZE) + @statuses = cache_collection(@statuses, Status) + + render xml: RSS::TagSerializer.render(@tag, @statuses) + end + format.json do - @statuses = Status.as_tag_timeline(@tag, current_account, params[:local]).paginate_by_max_id(20, params[:max_id]) + @statuses = Status.as_tag_timeline(@tag, current_account, params[:local]).paginate_by_max_id(PAGE_SIZE, params[:max_id]) @statuses = cache_collection(@statuses, Status) render json: collection_presenter, diff --git a/app/helpers/stream_entries_helper.rb b/app/helpers/stream_entries_helper.rb index 8254ef4dc..c6f12ecd4 100644 --- a/app/helpers/stream_entries_helper.rb +++ b/app/helpers/stream_entries_helper.rb @@ -12,17 +12,17 @@ module StreamEntriesHelper prepend_str = [ [ number_to_human(account.statuses_count, strip_insignificant_zeros: true), - t('accounts.posts'), + I18n.t('accounts.posts'), ].join(' '), [ number_to_human(account.following_count, strip_insignificant_zeros: true), - t('accounts.following'), + I18n.t('accounts.following'), ].join(' '), [ number_to_human(account.followers_count, strip_insignificant_zeros: true), - t('accounts.followers'), + I18n.t('accounts.followers'), ].join(' '), ].join(', ') @@ -40,16 +40,16 @@ module StreamEntriesHelper end end - text = attachments.to_a.reject { |_, value| value.zero? }.map { |key, value| t("statuses.attached.#{key}", count: value) }.join(' · ') + text = attachments.to_a.reject { |_, value| value.zero? }.map { |key, value| I18n.t("statuses.attached.#{key}", count: value) }.join(' · ') return if text.blank? - t('statuses.attached.description', attached: text) + I18n.t('statuses.attached.description', attached: text) end def status_text_summary(status) return if status.spoiler_text.blank? - t('statuses.content_warning', warning: status.spoiler_text) + I18n.t('statuses.content_warning', warning: status.spoiler_text) end def status_description(status) diff --git a/app/lib/rss_builder.rb b/app/lib/rss_builder.rb new file mode 100644 index 000000000..63ddba2e8 --- /dev/null +++ b/app/lib/rss_builder.rb @@ -0,0 +1,130 @@ +# frozen_string_literal: true + +class RSSBuilder + class ItemBuilder + def initialize + @item = Ox::Element.new('item') + end + + def title(str) + @item << (Ox::Element.new('title') << str) + + self + end + + def link(str) + @item << Ox::Element.new('guid').tap do |guid| + guid['isPermalink'] = 'true' + guid << str + end + + @item << (Ox::Element.new('link') << str) + + self + end + + def pub_date(date) + @item << (Ox::Element.new('pubDate') << date.to_formatted_s(:rfc822)) + + self + end + + def description(str) + @item << (Ox::Element.new('description') << str) + + self + end + + def enclosure(url, type, size) + @item << Ox::Element.new('enclosure').tap do |enclosure| + enclosure['url'] = url + enclosure['length'] = size + enclosure['type'] = type + end + + self + end + + def to_element + @item + end + end + + def initialize + @document = Ox::Document.new(version: '1.0') + @channel = Ox::Element.new('channel') + + @document << (rss << @channel) + end + + def title(str) + @channel << (Ox::Element.new('title') << str) + + self + end + + def link(str) + @channel << (Ox::Element.new('link') << str) + + self + end + + def image(str) + @channel << Ox::Element.new('image').tap do |image| + image << (Ox::Element.new('url') << str) + image << (Ox::Element.new('title') << '') + image << (Ox::Element.new('link') << '') + end + + @channel << (Ox::Element.new('webfeeds:icon') << str) + + self + end + + def cover(str) + @channel << Ox::Element.new('webfeeds:cover').tap do |cover| + cover['image'] = str + end + + self + end + + def logo(str) + @channel << (Ox::Element.new('webfeeds:logo') << str) + + self + end + + def accent_color(str) + @channel << (Ox::Element.new('webfeeds:accentColor') << str) + + self + end + + def description(str) + @channel << (Ox::Element.new('description') << str) + + self + end + + def item + @channel << ItemBuilder.new.tap do |item| + yield item + end.to_element + + self + end + + def to_xml + ('' + Ox.dump(@document, effort: :tolerant)).force_encoding('UTF-8') + end + + private + + def rss + Ox::Element.new('rss').tap do |rss| + rss['version'] = '2.0' + rss['xmlns:webfeeds'] = 'http://webfeeds.org/rss/1.0' + end + end +end diff --git a/app/serializers/rss/account_serializer.rb b/app/serializers/rss/account_serializer.rb new file mode 100644 index 000000000..bde360a41 --- /dev/null +++ b/app/serializers/rss/account_serializer.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +class RSS::AccountSerializer + include ActionView::Helpers::NumberHelper + include StreamEntriesHelper + include RoutingHelper + + def render(account, statuses) + builder = RSSBuilder.new + + builder.title("#{display_name(account)} (@#{account.local_username_and_domain})") + .description(account_description(account)) + .link(TagManager.instance.url_for(account)) + .logo(full_asset_url(asset_pack_path('logo.svg'))) + .accent_color('2b90d9') + + builder.image(full_asset_url(account.avatar.url(:original))) if account.avatar? + builder.cover(full_asset_url(account.header.url(:original))) if account.header? + + statuses.each do |status| + builder.item do |item| + item.title(status.title) + .link(TagManager.instance.url_for(status)) + .pub_date(status.created_at) + .description(status.spoiler_text.presence || Formatter.instance.format(status).to_str) + + status.media_attachments.each do |media| + item.enclosure(full_asset_url(media.file.url(:original, false)), media.file.content_type, length: media.file.size) + end + end + end + + builder.to_xml + end + + def self.render(account, statuses) + new.render(account, statuses) + end +end diff --git a/app/serializers/rss/tag_serializer.rb b/app/serializers/rss/tag_serializer.rb new file mode 100644 index 000000000..7680a8da5 --- /dev/null +++ b/app/serializers/rss/tag_serializer.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +class RSS::TagSerializer + include ActionView::Helpers::NumberHelper + include ActionView::Helpers::SanitizeHelper + include StreamEntriesHelper + include RoutingHelper + + def render(tag, statuses) + builder = RSSBuilder.new + + builder.title("##{tag.name}") + .description(strip_tags(I18n.t('about.about_hashtag_html', hashtag: tag.name))) + .link(tag_url(tag)) + .logo(full_asset_url(asset_pack_path('logo.svg'))) + .accent_color('2b90d9') + + statuses.each do |status| + builder.item do |item| + item.title(status.title) + .link(TagManager.instance.url_for(status)) + .pub_date(status.created_at) + .description(status.spoiler_text.presence || Formatter.instance.format(status).to_str) + + status.media_attachments.each do |media| + item.enclosure(full_asset_url(media.file.url(:original, false)), media.file.content_type, length: media.file.size) + end + end + end + + builder.to_xml + end + + def self.render(tag, statuses) + new.render(tag, statuses) + end +end -- cgit From 36b6631c1261f35ca22b9d2f9e99b72f83ed3492 Mon Sep 17 00:00:00 2001 From: Yamagishi Kazutoshi Date: Thu, 26 Apr 2018 20:56:45 +0900 Subject: Add Basque language support (#7267) --- app/helpers/settings_helper.rb | 1 + config/application.rb | 1 + 2 files changed, 2 insertions(+) (limited to 'app/helpers') diff --git a/app/helpers/settings_helper.rb b/app/helpers/settings_helper.rb index a2f5917f9..2252aab2f 100644 --- a/app/helpers/settings_helper.rb +++ b/app/helpers/settings_helper.rb @@ -9,6 +9,7 @@ module SettingsHelper de: 'Deutsch', eo: 'Esperanto', es: 'Español', + eu: 'Euskara', fa: 'فارسی', gl: 'Galego', fi: 'Suomi', diff --git a/config/application.rb b/config/application.rb index e989e2333..ec3ff47a4 100644 --- a/config/application.rb +++ b/config/application.rb @@ -43,6 +43,7 @@ module Mastodon :de, :eo, :es, + :eu, :fa, :fi, :fr, -- cgit From dc786c0cf4467ade8db7d8b17e09f16923bfc1e8 Mon Sep 17 00:00:00 2001 From: Surinna Curtis Date: Wed, 2 May 2018 05:40:24 -0500 Subject: Support Actors/Statuses with multiple types (#7305) * Add equals_or_includes_any? helper in JsonLdHelper * Support arrays in JSON-LD type fields for actors/tags/objects. * Spec for resolving accounts with extension types * Style tweaks for codeclimate --- app/helpers/jsonld_helper.rb | 4 ++++ app/lib/activitypub/activity/create.rb | 11 +++++------ app/lib/activitypub/activity/update.rb | 5 +---- app/services/activitypub/fetch_remote_account_service.rb | 2 +- app/services/activitypub/fetch_remote_key_service.rb | 4 ++-- app/services/activitypub/fetch_remote_status_service.rb | 2 +- app/services/activitypub/process_account_service.rb | 5 +---- app/services/fetch_atom_service.rb | 4 ++-- app/services/resolve_account_service.rb | 2 +- app/services/resolve_url_service.rb | 5 ++--- spec/fixtures/requests/activitypub-actor-individual.txt | 9 +++++++++ spec/services/resolve_account_service_spec.rb | 14 ++++++++++++++ 12 files changed, 43 insertions(+), 24 deletions(-) create mode 100644 spec/fixtures/requests/activitypub-actor-individual.txt (limited to 'app/helpers') diff --git a/app/helpers/jsonld_helper.rb b/app/helpers/jsonld_helper.rb index dfb8fcb8b..a3cfdadb8 100644 --- a/app/helpers/jsonld_helper.rb +++ b/app/helpers/jsonld_helper.rb @@ -5,6 +5,10 @@ module JsonLdHelper haystack.is_a?(Array) ? haystack.include?(needle) : haystack == needle end + def equals_or_includes_any?(haystack, needles) + needles.any? { |needle| equals_or_includes?(haystack, needle) } + end + def first_of_value(value) value.is_a?(Array) ? value.first : value end diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index 45c0e91cb..411286fa5 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -61,12 +61,11 @@ class ActivityPub::Activity::Create < ActivityPub::Activity return if @object['tag'].nil? as_array(@object['tag']).each do |tag| - case tag['type'] - when 'Hashtag' + if equals_or_includes?(tag['type'], 'Hashtag') process_hashtag tag, status - when 'Mention' + elsif equals_or_includes?(tag['type'], 'Mention') process_mention tag, status - when 'Emoji' + elsif equals_or_includes?(tag['type'], 'Emoji') process_emoji tag, status end end @@ -235,11 +234,11 @@ class ActivityPub::Activity::Create < ActivityPub::Activity end def supported_object_type? - SUPPORTED_TYPES.include?(@object['type']) + equals_or_includes_any?(@object['type'], SUPPORTED_TYPES) end def converted_object_type? - CONVERTED_TYPES.include?(@object['type']) + equals_or_includes_any?(@object['type'], CONVERTED_TYPES) end def skip_download? diff --git a/app/lib/activitypub/activity/update.rb b/app/lib/activitypub/activity/update.rb index 0134b4015..47e98e041 100644 --- a/app/lib/activitypub/activity/update.rb +++ b/app/lib/activitypub/activity/update.rb @@ -2,10 +2,7 @@ class ActivityPub::Activity::Update < ActivityPub::Activity def perform - case @object['type'] - when 'Person' - update_account - end + update_account if equals_or_includes?(@object['type'], 'Person') end private diff --git a/app/services/activitypub/fetch_remote_account_service.rb b/app/services/activitypub/fetch_remote_account_service.rb index 5024853ca..867e70876 100644 --- a/app/services/activitypub/fetch_remote_account_service.rb +++ b/app/services/activitypub/fetch_remote_account_service.rb @@ -56,6 +56,6 @@ class ActivityPub::FetchRemoteAccountService < BaseService end def expected_type? - SUPPORTED_TYPES.include?(@json['type']) + equals_or_includes_any?(@json['type'], SUPPORTED_TYPES) end end diff --git a/app/services/activitypub/fetch_remote_key_service.rb b/app/services/activitypub/fetch_remote_key_service.rb index 41837d462..505baccd4 100644 --- a/app/services/activitypub/fetch_remote_key_service.rb +++ b/app/services/activitypub/fetch_remote_key_service.rb @@ -43,7 +43,7 @@ class ActivityPub::FetchRemoteKeyService < BaseService end def person? - ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES.include?(@json['type']) + equals_or_includes_any?(@json['type'], ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES) end def public_key? @@ -55,6 +55,6 @@ class ActivityPub::FetchRemoteKeyService < BaseService end def confirmed_owner? - ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES.include?(@owner['type']) && value_or_id(@owner['publicKey']) == @json['id'] + equals_or_includes_any?(@owner['type'], ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES) && value_or_id(@owner['publicKey']) == @json['id'] end end diff --git a/app/services/activitypub/fetch_remote_status_service.rb b/app/services/activitypub/fetch_remote_status_service.rb index 503c175d8..930fbad1f 100644 --- a/app/services/activitypub/fetch_remote_status_service.rb +++ b/app/services/activitypub/fetch_remote_status_service.rb @@ -42,7 +42,7 @@ class ActivityPub::FetchRemoteStatusService < BaseService end def expected_type? - (ActivityPub::Activity::Create::SUPPORTED_TYPES + ActivityPub::Activity::Create::CONVERTED_TYPES).include? @json['type'] + equals_or_includes_any?(@json['type'], ActivityPub::Activity::Create::SUPPORTED_TYPES + ActivityPub::Activity::Create::CONVERTED_TYPES) end def needs_update(actor) diff --git a/app/services/activitypub/process_account_service.rb b/app/services/activitypub/process_account_service.rb index da32f9615..f67ebb443 100644 --- a/app/services/activitypub/process_account_service.rb +++ b/app/services/activitypub/process_account_service.rb @@ -201,10 +201,7 @@ class ActivityPub::ProcessAccountService < BaseService return if @json['tag'].blank? as_array(@json['tag']).each do |tag| - case tag['type'] - when 'Emoji' - process_emoji tag - end + process_emoji tag if equals_or_includes?(tag['type'], 'Emoji') end end diff --git a/app/services/fetch_atom_service.rb b/app/services/fetch_atom_service.rb index 0444baf74..550e75f33 100644 --- a/app/services/fetch_atom_service.rb +++ b/app/services/fetch_atom_service.rb @@ -42,7 +42,7 @@ class FetchAtomService < BaseService elsif ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(response.mime_type) body = response.body_with_limit json = body_to_json(body) - if supported_context?(json) && ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES.include?(json['type']) && json['inbox'].present? + if supported_context?(json) && equals_or_includes_any?(json['type'], ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES) && json['inbox'].present? [json['id'], { prefetched_body: body, id: true }, :activitypub] elsif supported_context?(json) && expected_type?(json) [json['id'], { prefetched_body: body, id: true }, :activitypub] @@ -62,7 +62,7 @@ class FetchAtomService < BaseService end def expected_type?(json) - (ActivityPub::Activity::Create::SUPPORTED_TYPES + ActivityPub::Activity::Create::CONVERTED_TYPES).include? json['type'] + equals_or_includes_any?(json['type'], ActivityPub::Activity::Create::SUPPORTED_TYPES + ActivityPub::Activity::Create::CONVERTED_TYPES) end def process_html(response) diff --git a/app/services/resolve_account_service.rb b/app/services/resolve_account_service.rb index 8cba88f01..de8d1151d 100644 --- a/app/services/resolve_account_service.rb +++ b/app/services/resolve_account_service.rb @@ -189,7 +189,7 @@ class ResolveAccountService < BaseService return @actor_json if defined?(@actor_json) json = fetch_resource(actor_url, false) - @actor_json = supported_context?(json) && ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES.include?(json['type']) ? json : nil + @actor_json = supported_context?(json) && equals_or_includes_any?(json['type'], ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES) ? json : nil end def atom diff --git a/app/services/resolve_url_service.rb b/app/services/resolve_url_service.rb index c19b568cb..a068c1ed8 100644 --- a/app/services/resolve_url_service.rb +++ b/app/services/resolve_url_service.rb @@ -16,10 +16,9 @@ class ResolveURLService < BaseService private def process_url - case type - when 'Application', 'Group', 'Organization', 'Person', 'Service' + if equals_or_includes_any?(type, %w(Application Group Organization Person Service)) FetchRemoteAccountService.new.call(atom_url, body, protocol) - when 'Note', 'Article', 'Image', 'Video' + elsif equals_or_includes_any?(type, %w(Note Article Image Video)) FetchRemoteStatusService.new.call(atom_url, body, protocol) end end diff --git a/spec/fixtures/requests/activitypub-actor-individual.txt b/spec/fixtures/requests/activitypub-actor-individual.txt new file mode 100644 index 000000000..74411e544 --- /dev/null +++ b/spec/fixtures/requests/activitypub-actor-individual.txt @@ -0,0 +1,9 @@ +HTTP/1.1 200 OK +Cache-Control: max-age=0, private, must-revalidate +Content-Type: application/activity+json; charset=utf-8 +Link: ; rel="lrdd"; type="application/xrd+xml", ; rel="alternate"; type="application/atom+xml", ; rel="alternate"; type="application/activity+json" +Vary: Accept-Encoding +X-Content-Type-Options: nosniff +X-Xss-Protection: 1; mode=block + +{"@context":["https://www.w3.org/ns/activitystreams","https://w3id.org/security/v1",{"vcard": "http://www.w3.org/2006/vcard/ns#"},{"manuallyApprovesFollowers":"as:manuallyApprovesFollowers","sensitive":"as:sensitive","Hashtag":"as:Hashtag","ostatus":"http://ostatus.org#","atomUri":"ostatus:atomUri","inReplyToAtomUri":"ostatus:inReplyToAtomUri","conversation":"ostatus:conversation"}],"id":"https://ap.example.com/users/foo","type":["Person","vcard:individual"],"following":"https://ap.example.com/users/foo/following","followers":"https://ap.example.com/users/foo/followers","inbox":"https://ap.example.com/users/foo/inbox","outbox":"https://ap.example.com/users/foo/outbox","preferredUsername":"foo","vcard:fn":"foo","name":"","summary":"\u003cp\u003etest\u003c/p\u003e","url":"https://ap.example.com/@foo","manuallyApprovesFollowers":false,"publicKey":{"id":"https://ap.example.com/users/foo#main-key","owner":"https://ap.example.com/users/foo","publicKeyPem":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAu3L4vnpNLzVH31MeWI39\n4F0wKeJFsLDAsNXGeOu0QF2x+h1zLWZw/agqD2R3JPU9/kaDJGPIV2Sn5zLyUA9S\n6swCCMOtn7BBR9g9sucgXJmUFB0tACH2QSgHywMAybGfmSb3LsEMNKsGJ9VsvYoh\n8lDET6X4Pyw+ZJU0/OLo/41q9w+OrGtlsTm/PuPIeXnxa6BLqnDaxC+4IcjG/FiP\nahNCTINl/1F/TgSSDZ4Taf4U9XFEIFw8wmgploELozzIzKq+t8nhQYkgAkt64euW\npva3qL5KD1mTIZQEP+LZvh3s2WHrLi3fhbdRuwQ2c0KkJA2oSTFPDpqqbPGZ3Qvu\nHQIDAQAB\n-----END PUBLIC KEY-----\n"},"endpoints":{"sharedInbox":"https://ap.example.com/inbox"},"icon":{"type":"Image","url":"https://quitter.no/avatar/7477-300-20160211190340.png"}} \ No newline at end of file diff --git a/spec/services/resolve_account_service_spec.rb b/spec/services/resolve_account_service_spec.rb index 5f1b4467b..84dfe578a 100644 --- a/spec/services/resolve_account_service_spec.rb +++ b/spec/services/resolve_account_service_spec.rb @@ -105,6 +105,20 @@ RSpec.describe ResolveAccountService do expect(account.inbox_url).to eq 'https://ap.example.com/users/foo/inbox' end + context 'with multiple types' do + before do + stub_request(:get, "https://ap.example.com/users/foo").to_return(request_fixture('activitypub-actor-individual.txt')) + end + + it 'returns new remote account' do + account = subject.call('foo@ap.example.com') + + expect(account.activitypub?).to eq true + expect(account.domain).to eq 'ap.example.com' + expect(account.inbox_url).to eq 'https://ap.example.com/users/foo/inbox' + end + end + pending end -- cgit From cb5b5cb5f79bb2187d8124df91af4c8e1bfd7256 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Wed, 2 May 2018 18:58:48 +0200 Subject: Slightly reduce RAM usage (#7301) * No need to re-require sidekiq plugins, they are required via Gemfile * Add derailed_benchmarks tool, no need to require TTY gems in Gemfile * Replace ruby-oembed with FetchOEmbedService Reduce startup by 45382 allocated objects * Remove preloaded JSON-LD in favour of caching HTTP responses Reduce boot RAM by about 6 MiB * Fix tests * Fix test suite by stubbing out JSON-LD contexts --- Gemfile | 12 +- Gemfile.lock | 22 +- app/controllers/api/web/embeds_controller.rb | 11 +- .../settings/follower_domains_controller.rb | 2 - app/helpers/jsonld_helper.rb | 17 +- app/lib/provider_discovery.rb | 47 --- app/services/fan_out_on_write_service.rb | 2 - app/services/fetch_link_card_service.rb | 38 +- app/services/fetch_oembed_service.rb | 71 ++++ app/workers/scheduler/backup_cleanup_scheduler.rb | 1 - .../scheduler/doorkeeper_cleanup_scheduler.rb | 1 - app/workers/scheduler/email_scheduler.rb | 1 - app/workers/scheduler/feed_cleanup_scheduler.rb | 1 - app/workers/scheduler/ip_cleanup_scheduler.rb | 1 - app/workers/scheduler/media_cleanup_scheduler.rb | 1 - .../scheduler/subscriptions_cleanup_scheduler.rb | 2 - app/workers/scheduler/subscriptions_scheduler.rb | 3 - app/workers/scheduler/user_cleanup_scheduler.rb | 1 - app/workers/soft_block_domain_followers_worker.rb | 2 - config/initializers/json_ld.rb | 5 - config/initializers/oembed.rb | 4 - lib/json_ld/activitystreams.rb | 153 -------- lib/json_ld/identity.rb | 86 ----- lib/json_ld/security.rb | 50 --- lib/tasks/mastodon.rake | 2 + spec/fixtures/requests/json-ld.activitystreams.txt | 391 +++++++++++++++++++++ spec/fixtures/requests/json-ld.identity.txt | 100 ++++++ spec/fixtures/requests/json-ld.security.txt | 61 ++++ spec/lib/activitypub/linked_data_signature_spec.rb | 4 + spec/lib/provider_discovery_spec.rb | 118 ------- spec/rails_helper.rb | 14 + spec/services/account_search_service_spec.rb | 2 +- .../fetch_remote_account_service_spec.rb | 2 +- .../fetch_remote_status_service_spec.rb | 2 +- .../activitypub/process_account_service_spec.rb | 2 +- .../activitypub/process_collection_service_spec.rb | 2 +- spec/services/after_block_service_spec.rb | 2 +- spec/services/authorize_follow_service_spec.rb | 2 +- .../services/batched_remove_status_service_spec.rb | 2 +- .../block_domain_from_account_service_spec.rb | 2 +- spec/services/block_domain_service_spec.rb | 2 +- spec/services/block_service_spec.rb | 2 +- spec/services/bootstrap_timeline_service_spec.rb | 2 +- spec/services/fan_out_on_write_service_spec.rb | 2 +- spec/services/favourite_service_spec.rb | 2 +- spec/services/fetch_atom_service_spec.rb | 2 +- spec/services/fetch_link_card_service_spec.rb | 2 +- spec/services/fetch_oembed_service_spec.rb | 125 +++++++ spec/services/fetch_remote_account_service_spec.rb | 2 +- spec/services/fetch_remote_status_service_spec.rb | 2 +- spec/services/follow_service_spec.rb | 2 +- spec/services/mute_service_spec.rb | 2 +- spec/services/notify_service_spec.rb | 2 +- spec/services/post_status_service_spec.rb | 2 +- spec/services/precompute_feed_service_spec.rb | 2 +- spec/services/process_feed_service_spec.rb | 2 +- spec/services/process_interaction_service_spec.rb | 2 +- spec/services/process_mentions_service_spec.rb | 2 +- .../pubsubhubbub/subscribe_service_spec.rb | 2 +- .../pubsubhubbub/unsubscribe_service_spec.rb | 2 +- spec/services/reblog_service_spec.rb | 2 +- spec/services/reject_follow_service_spec.rb | 2 +- spec/services/remove_status_service_spec.rb | 2 +- spec/services/report_service_spec.rb | 2 +- spec/services/resolve_account_service_spec.rb | 2 +- spec/services/resolve_url_service_spec.rb | 2 +- spec/services/search_service_spec.rb | 2 +- spec/services/send_interaction_service_spec.rb | 2 +- spec/services/subscribe_service_spec.rb | 2 +- spec/services/suspend_account_service_spec.rb | 2 +- spec/services/unblock_domain_service_spec.rb | 2 +- spec/services/unblock_service_spec.rb | 2 +- spec/services/unfollow_service_spec.rb | 2 +- spec/services/unmute_service_spec.rb | 2 +- spec/services/unsubscribe_service_spec.rb | 2 +- .../services/update_remote_profile_service_spec.rb | 2 +- spec/spec_helper.rb | 12 +- 77 files changed, 881 insertions(+), 568 deletions(-) delete mode 100644 app/lib/provider_discovery.rb create mode 100644 app/services/fetch_oembed_service.rb delete mode 100644 config/initializers/json_ld.rb delete mode 100644 config/initializers/oembed.rb delete mode 100644 lib/json_ld/activitystreams.rb delete mode 100644 lib/json_ld/identity.rb delete mode 100644 lib/json_ld/security.rb create mode 100644 spec/fixtures/requests/json-ld.activitystreams.txt create mode 100644 spec/fixtures/requests/json-ld.identity.txt create mode 100644 spec/fixtures/requests/json-ld.security.txt delete mode 100644 spec/lib/provider_discovery_spec.rb create mode 100644 spec/services/fetch_oembed_service_spec.rb (limited to 'app/helpers') diff --git a/Gemfile b/Gemfile index a33748568..f1665ce95 100644 --- a/Gemfile +++ b/Gemfile @@ -54,7 +54,7 @@ gem 'httplog', '~> 1.0' gem 'idn-ruby', require: 'idn' gem 'kaminari', '~> 1.1' gem 'link_header', '~> 0.0' -gem 'mime-types', '~> 3.1' +gem 'mime-types', '~> 3.1', require: 'mime/types/columnar' gem 'nokogiri', '~> 1.8' gem 'nsa', '~> 0.2' gem 'oj', '~> 3.5' @@ -70,7 +70,6 @@ gem 'rails-settings-cached', '~> 0.6' gem 'redis', '~> 4.0', require: ['redis', 'redis/connection/hiredis'] gem 'mario-redis-lock', '~> 1.2', require: 'redis_lock' gem 'rqrcode', '~> 0.10' -gem 'ruby-oembed', '~> 0.12', require: 'oembed' gem 'ruby-progressbar', '~> 1.4' gem 'sanitize', '~> 4.6' gem 'sidekiq', '~> 5.1' @@ -82,14 +81,14 @@ gem 'simple_form', '~> 4.0' gem 'sprockets-rails', '~> 3.2', require: 'sprockets/railtie' gem 'stoplight', '~> 2.1.3' gem 'strong_migrations', '~> 0.2' -gem 'tty-command', '~> 0.8' -gem 'tty-prompt', '~> 0.16' +gem 'tty-command', '~> 0.8', require: false +gem 'tty-prompt', '~> 0.16', require: false gem 'twitter-text', '~> 1.14' gem 'tzinfo-data', '~> 1.2018' gem 'webpacker', '~> 3.4' gem 'webpush' -gem 'json-ld-preloaded', '~> 2.2' +gem 'json-ld', '~> 2.2' gem 'rdf-normalize', '~> 0.3' group :development, :test do @@ -135,6 +134,9 @@ group :development do gem 'capistrano-rails', '~> 1.3' gem 'capistrano-rbenv', '~> 2.1' gem 'capistrano-yarn', '~> 2.0' + + gem 'derailed_benchmarks' + gem 'stackprof' end group :production do diff --git a/Gemfile.lock b/Gemfile.lock index d96165dcf..94ab0b7ca 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -75,6 +75,7 @@ GEM aws-sigv4 (~> 1.0) aws-sigv4 (1.0.2) bcrypt (3.1.11) + benchmark-ips (2.7.2) better_errors (2.4.0) coderay (>= 1.0.0) erubi (>= 1.0.0) @@ -138,6 +139,14 @@ GEM css_parser (1.6.0) addressable debug_inspector (0.0.3) + derailed_benchmarks (1.3.4) + benchmark-ips (~> 2) + get_process_mem (~> 0) + heapy (~> 0) + memory_profiler (~> 0) + rack (>= 1) + rake (> 10, < 13) + thor (~> 0.19) devise (4.4.3) bcrypt (~> 3.0) orm_adapter (~> 0.1) @@ -206,6 +215,7 @@ GEM fuubar (2.3.1) rspec-core (~> 3.0) ruby-progressbar (~> 1.4) + get_process_mem (0.2.1) globalid (0.4.1) activesupport (>= 4.2.0) goldfinger (2.1.0) @@ -226,6 +236,7 @@ GEM concurrent-ruby (~> 1.0) hashdiff (0.3.7) hashie (3.5.7) + heapy (0.1.3) highline (1.7.10) hiredis (0.6.1) hitimes (1.2.6) @@ -264,10 +275,6 @@ GEM json-ld (2.2.1) multi_json (~> 1.12) rdf (>= 2.2.8, < 4.0) - json-ld-preloaded (2.2.3) - json-ld (>= 2.2, < 4.0) - multi_json (~> 1.12) - rdf (>= 2.2, < 4.0) jsonapi-renderer (0.2.0) jwt (2.1.0) kaminari (1.1.1) @@ -502,7 +509,6 @@ GEM rainbow (>= 2.2.2, < 4.0) ruby-progressbar (~> 1.7) unicode-display_width (~> 1.0, >= 1.0.1) - ruby-oembed (0.12.0) ruby-progressbar (1.9.0) ruby-saml (1.7.2) nokogiri (>= 1.5.10) @@ -557,6 +563,7 @@ GEM sshkit (1.16.0) net-scp (>= 1.1.2) net-ssh (>= 2.8.0) + stackprof (0.2.11) statsd-ruby (1.2.1) stoplight (2.1.3) streamio-ffmpeg (3.0.2) @@ -645,6 +652,7 @@ DEPENDENCIES chewy (~> 5.0) cld3 (~> 3.2.0) climate_control (~> 0.2) + derailed_benchmarks devise (~> 4.4) devise-two-factor (~> 3.0) devise_pam_authenticatable2 (~> 9.1) @@ -668,7 +676,7 @@ DEPENDENCIES i18n-tasks (~> 0.9) idn-ruby iso-639 - json-ld-preloaded (~> 2.2) + json-ld (~> 2.2) kaminari (~> 1.1) letter_opener (~> 1.4) letter_opener_web (~> 1.3) @@ -714,7 +722,6 @@ DEPENDENCIES rspec-retry (~> 0.5) rspec-sidekiq (~> 3.0) rubocop (~> 0.55) - ruby-oembed (~> 0.12) ruby-progressbar (~> 1.4) sanitize (~> 4.6) scss_lint (~> 0.57) @@ -726,6 +733,7 @@ DEPENDENCIES simple_form (~> 4.0) simplecov (~> 0.16) sprockets-rails (~> 3.2) + stackprof stoplight (~> 2.1.3) streamio-ffmpeg (~> 3.0) strong_migrations (~> 0.2) diff --git a/app/controllers/api/web/embeds_controller.rb b/app/controllers/api/web/embeds_controller.rb index f2fe74b17..987290a14 100644 --- a/app/controllers/api/web/embeds_controller.rb +++ b/app/controllers/api/web/embeds_controller.rb @@ -9,9 +9,12 @@ class Api::Web::EmbedsController < Api::Web::BaseController status = StatusFinder.new(params[:url]).status render json: status, serializer: OEmbedSerializer, width: 400 rescue ActiveRecord::RecordNotFound - oembed = OEmbed::Providers.get(params[:url]) - render json: Oj.dump(oembed.fields) - rescue OEmbed::NotFound - render json: {}, status: :not_found + oembed = FetchOEmbedService.new.call(params[:url]) + + if oembed + render json: oembed + else + render json: {}, status: :not_found + end end end diff --git a/app/controllers/settings/follower_domains_controller.rb b/app/controllers/settings/follower_domains_controller.rb index 213d9e96d..91b521e7f 100644 --- a/app/controllers/settings/follower_domains_controller.rb +++ b/app/controllers/settings/follower_domains_controller.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require 'sidekiq-bulk' - class Settings::FollowerDomainsController < ApplicationController layout 'admin' diff --git a/app/helpers/jsonld_helper.rb b/app/helpers/jsonld_helper.rb index a3cfdadb8..e9056166c 100644 --- a/app/helpers/jsonld_helper.rb +++ b/app/helpers/jsonld_helper.rb @@ -48,7 +48,7 @@ module JsonLdHelper end def canonicalize(json) - graph = RDF::Graph.new << JSON::LD::API.toRdf(json) + graph = RDF::Graph.new << JSON::LD::API.toRdf(json, documentLoader: method(:load_jsonld_context)) graph.dump(:normalize) end @@ -90,4 +90,19 @@ module JsonLdHelper request.add_headers('Accept' => 'application/activity+json, application/ld+json') request end + + def load_jsonld_context(url, _options = {}, &_block) + json = Rails.cache.fetch("jsonld:context:#{url}", expires_in: 30.days, raw: true) do + request = Request.new(:get, url) + request.add_headers('Accept' => 'application/ld+json') + + request.perform do |res| + raise JSON::LD::JsonLdError::LoadingDocumentFailed unless res.code == 200 && res.mime_type == 'application/ld+json' + res.body_with_limit + end + end + + doc = JSON::LD::API::RemoteDocument.new(url, json) + block_given? ? yield(doc) : doc + end end diff --git a/app/lib/provider_discovery.rb b/app/lib/provider_discovery.rb deleted file mode 100644 index 3bec7211b..000000000 --- a/app/lib/provider_discovery.rb +++ /dev/null @@ -1,47 +0,0 @@ -# frozen_string_literal: true - -class ProviderDiscovery < OEmbed::ProviderDiscovery - class << self - def get(url, **options) - provider = discover_provider(url, options) - - options.delete(:html) - - provider.get(url, options) - end - - def discover_provider(url, **options) - format = options[:format] - - html = if options[:html] - Nokogiri::HTML(options[:html]) - else - Request.new(:get, url).perform do |res| - raise OEmbed::NotFound, url if res.code != 200 || res.mime_type != 'text/html' - Nokogiri::HTML(res.body_with_limit) - end - end - - if format.nil? || format == :json - provider_endpoint ||= html.at_xpath('//link[@type="application/json+oembed"]')&.attribute('href')&.value - format ||= :json if provider_endpoint - end - - if format.nil? || format == :xml - provider_endpoint ||= html.at_xpath('//link[@type="text/xml+oembed"]')&.attribute('href')&.value - format ||= :xml if provider_endpoint - end - - raise OEmbed::NotFound, url if provider_endpoint.nil? - begin - provider_endpoint = Addressable::URI.parse(provider_endpoint) - provider_endpoint.query = nil - provider_endpoint = provider_endpoint.to_s - rescue Addressable::URI::InvalidURIError - raise OEmbed::NotFound, url - end - - OEmbed::Provider.new(provider_endpoint, format) - end - end -end diff --git a/app/services/fan_out_on_write_service.rb b/app/services/fan_out_on_write_service.rb index 0f77556dc..510b80c82 100644 --- a/app/services/fan_out_on_write_service.rb +++ b/app/services/fan_out_on_write_service.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require 'sidekiq-bulk' - class FanOutOnWriteService < BaseService # Push a status into home and mentions feeds # @param [Status] status diff --git a/app/services/fetch_link_card_service.rb b/app/services/fetch_link_card_service.rb index d5920a417..77d4aa538 100644 --- a/app/services/fetch_link_card_service.rb +++ b/app/services/fetch_link_card_service.rb @@ -85,42 +85,40 @@ class FetchLinkCardService < BaseService end def attempt_oembed - embed = OEmbed::Providers.get(@url, html: @html) + embed = FetchOEmbedService.new.call(@url, html: @html) - return false unless embed.respond_to?(:type) + return false if embed.nil? - @card.type = embed.type - @card.title = embed.respond_to?(:title) ? embed.title : '' - @card.author_name = embed.respond_to?(:author_name) ? embed.author_name : '' - @card.author_url = embed.respond_to?(:author_url) ? embed.author_url : '' - @card.provider_name = embed.respond_to?(:provider_name) ? embed.provider_name : '' - @card.provider_url = embed.respond_to?(:provider_url) ? embed.provider_url : '' + @card.type = embed[:type] + @card.title = embed[:title] || '' + @card.author_name = embed[:author_name] || '' + @card.author_url = embed[:author_url] || '' + @card.provider_name = embed[:provider_name] || '' + @card.provider_url = embed[:provider_url] || '' @card.width = 0 @card.height = 0 case @card.type when 'link' - @card.image_remote_url = embed.thumbnail_url if embed.respond_to?(:thumbnail_url) + @card.image_remote_url = embed[:thumbnail_url] if embed[:thumbnail_url].present? when 'photo' - return false unless embed.respond_to?(:url) + return false if embed[:url].blank? - @card.embed_url = embed.url - @card.image_remote_url = embed.url - @card.width = embed.width.presence || 0 - @card.height = embed.height.presence || 0 + @card.embed_url = embed[:url] + @card.image_remote_url = embed[:url] + @card.width = embed[:width].presence || 0 + @card.height = embed[:height].presence || 0 when 'video' - @card.width = embed.width.presence || 0 - @card.height = embed.height.presence || 0 - @card.html = Formatter.instance.sanitize(embed.html, Sanitize::Config::MASTODON_OEMBED) - @card.image_remote_url = embed.thumbnail_url if embed.respond_to?(:thumbnail_url) + @card.width = embed[:width].presence || 0 + @card.height = embed[:height].presence || 0 + @card.html = Formatter.instance.sanitize(embed[:html], Sanitize::Config::MASTODON_OEMBED) + @card.image_remote_url = embed[:thumbnail_url] if embed[:thumbnail_url].present? when 'rich' # Most providers rely on