From eb593a5a0c9ee9749115f867a5675ceb1f231379 Mon Sep 17 00:00:00 2001 From: MIYAGI Hikaru Date: Wed, 25 Apr 2018 21:12:28 +0900 Subject: Append '.test' to hostname in stub data (#7260) --- spec/fixtures/requests/oembed_json.html | 2 +- spec/fixtures/requests/oembed_json_xml.html | 4 ++-- spec/fixtures/requests/oembed_xml.html | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) (limited to 'spec/fixtures/requests') diff --git a/spec/fixtures/requests/oembed_json.html b/spec/fixtures/requests/oembed_json.html index 773a4f92a..167085871 100644 --- a/spec/fixtures/requests/oembed_json.html +++ b/spec/fixtures/requests/oembed_json.html @@ -1,7 +1,7 @@ - + diff --git a/spec/fixtures/requests/oembed_json_xml.html b/spec/fixtures/requests/oembed_json_xml.html index 8afd8e997..9f5b9e8be 100644 --- a/spec/fixtures/requests/oembed_json_xml.html +++ b/spec/fixtures/requests/oembed_json_xml.html @@ -7,8 +7,8 @@ > The type attribute must contain either application/json+oembed for JSON > responses, or text/xml+oembed for XML. --> - - + + diff --git a/spec/fixtures/requests/oembed_xml.html b/spec/fixtures/requests/oembed_xml.html index bdfcca170..788dfaabd 100644 --- a/spec/fixtures/requests/oembed_xml.html +++ b/spec/fixtures/requests/oembed_xml.html @@ -7,7 +7,7 @@ > The type attribute must contain either application/json+oembed for JSON > responses, or text/xml+oembed for XML. --> - + -- cgit From dc786c0cf4467ade8db7d8b17e09f16923bfc1e8 Mon Sep 17 00:00:00 2001 From: Surinna Curtis Date: Wed, 2 May 2018 05:40:24 -0500 Subject: Support Actors/Statuses with multiple types (#7305) * Add equals_or_includes_any? helper in JsonLdHelper * Support arrays in JSON-LD type fields for actors/tags/objects. * Spec for resolving accounts with extension types * Style tweaks for codeclimate --- app/helpers/jsonld_helper.rb | 4 ++++ app/lib/activitypub/activity/create.rb | 11 +++++------ app/lib/activitypub/activity/update.rb | 5 +---- app/services/activitypub/fetch_remote_account_service.rb | 2 +- app/services/activitypub/fetch_remote_key_service.rb | 4 ++-- app/services/activitypub/fetch_remote_status_service.rb | 2 +- app/services/activitypub/process_account_service.rb | 5 +---- app/services/fetch_atom_service.rb | 4 ++-- app/services/resolve_account_service.rb | 2 +- app/services/resolve_url_service.rb | 5 ++--- spec/fixtures/requests/activitypub-actor-individual.txt | 9 +++++++++ spec/services/resolve_account_service_spec.rb | 14 ++++++++++++++ 12 files changed, 43 insertions(+), 24 deletions(-) create mode 100644 spec/fixtures/requests/activitypub-actor-individual.txt (limited to 'spec/fixtures/requests') diff --git a/app/helpers/jsonld_helper.rb b/app/helpers/jsonld_helper.rb index dfb8fcb8b..a3cfdadb8 100644 --- a/app/helpers/jsonld_helper.rb +++ b/app/helpers/jsonld_helper.rb @@ -5,6 +5,10 @@ module JsonLdHelper haystack.is_a?(Array) ? haystack.include?(needle) : haystack == needle end + def equals_or_includes_any?(haystack, needles) + needles.any? { |needle| equals_or_includes?(haystack, needle) } + end + def first_of_value(value) value.is_a?(Array) ? value.first : value end diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index 45c0e91cb..411286fa5 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -61,12 +61,11 @@ class ActivityPub::Activity::Create < ActivityPub::Activity return if @object['tag'].nil? as_array(@object['tag']).each do |tag| - case tag['type'] - when 'Hashtag' + if equals_or_includes?(tag['type'], 'Hashtag') process_hashtag tag, status - when 'Mention' + elsif equals_or_includes?(tag['type'], 'Mention') process_mention tag, status - when 'Emoji' + elsif equals_or_includes?(tag['type'], 'Emoji') process_emoji tag, status end end @@ -235,11 +234,11 @@ class ActivityPub::Activity::Create < ActivityPub::Activity end def supported_object_type? - SUPPORTED_TYPES.include?(@object['type']) + equals_or_includes_any?(@object['type'], SUPPORTED_TYPES) end def converted_object_type? - CONVERTED_TYPES.include?(@object['type']) + equals_or_includes_any?(@object['type'], CONVERTED_TYPES) end def skip_download? diff --git a/app/lib/activitypub/activity/update.rb b/app/lib/activitypub/activity/update.rb index 0134b4015..47e98e041 100644 --- a/app/lib/activitypub/activity/update.rb +++ b/app/lib/activitypub/activity/update.rb @@ -2,10 +2,7 @@ class ActivityPub::Activity::Update < ActivityPub::Activity def perform - case @object['type'] - when 'Person' - update_account - end + update_account if equals_or_includes?(@object['type'], 'Person') end private diff --git a/app/services/activitypub/fetch_remote_account_service.rb b/app/services/activitypub/fetch_remote_account_service.rb index 5024853ca..867e70876 100644 --- a/app/services/activitypub/fetch_remote_account_service.rb +++ b/app/services/activitypub/fetch_remote_account_service.rb @@ -56,6 +56,6 @@ class ActivityPub::FetchRemoteAccountService < BaseService end def expected_type? - SUPPORTED_TYPES.include?(@json['type']) + equals_or_includes_any?(@json['type'], SUPPORTED_TYPES) end end diff --git a/app/services/activitypub/fetch_remote_key_service.rb b/app/services/activitypub/fetch_remote_key_service.rb index 41837d462..505baccd4 100644 --- a/app/services/activitypub/fetch_remote_key_service.rb +++ b/app/services/activitypub/fetch_remote_key_service.rb @@ -43,7 +43,7 @@ class ActivityPub::FetchRemoteKeyService < BaseService end def person? - ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES.include?(@json['type']) + equals_or_includes_any?(@json['type'], ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES) end def public_key? @@ -55,6 +55,6 @@ class ActivityPub::FetchRemoteKeyService < BaseService end def confirmed_owner? - ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES.include?(@owner['type']) && value_or_id(@owner['publicKey']) == @json['id'] + equals_or_includes_any?(@owner['type'], ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES) && value_or_id(@owner['publicKey']) == @json['id'] end end diff --git a/app/services/activitypub/fetch_remote_status_service.rb b/app/services/activitypub/fetch_remote_status_service.rb index 503c175d8..930fbad1f 100644 --- a/app/services/activitypub/fetch_remote_status_service.rb +++ b/app/services/activitypub/fetch_remote_status_service.rb @@ -42,7 +42,7 @@ class ActivityPub::FetchRemoteStatusService < BaseService end def expected_type? - (ActivityPub::Activity::Create::SUPPORTED_TYPES + ActivityPub::Activity::Create::CONVERTED_TYPES).include? @json['type'] + equals_or_includes_any?(@json['type'], ActivityPub::Activity::Create::SUPPORTED_TYPES + ActivityPub::Activity::Create::CONVERTED_TYPES) end def needs_update(actor) diff --git a/app/services/activitypub/process_account_service.rb b/app/services/activitypub/process_account_service.rb index da32f9615..f67ebb443 100644 --- a/app/services/activitypub/process_account_service.rb +++ b/app/services/activitypub/process_account_service.rb @@ -201,10 +201,7 @@ class ActivityPub::ProcessAccountService < BaseService return if @json['tag'].blank? as_array(@json['tag']).each do |tag| - case tag['type'] - when 'Emoji' - process_emoji tag - end + process_emoji tag if equals_or_includes?(tag['type'], 'Emoji') end end diff --git a/app/services/fetch_atom_service.rb b/app/services/fetch_atom_service.rb index 0444baf74..550e75f33 100644 --- a/app/services/fetch_atom_service.rb +++ b/app/services/fetch_atom_service.rb @@ -42,7 +42,7 @@ class FetchAtomService < BaseService elsif ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(response.mime_type) body = response.body_with_limit json = body_to_json(body) - if supported_context?(json) && ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES.include?(json['type']) && json['inbox'].present? + if supported_context?(json) && equals_or_includes_any?(json['type'], ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES) && json['inbox'].present? [json['id'], { prefetched_body: body, id: true }, :activitypub] elsif supported_context?(json) && expected_type?(json) [json['id'], { prefetched_body: body, id: true }, :activitypub] @@ -62,7 +62,7 @@ class FetchAtomService < BaseService end def expected_type?(json) - (ActivityPub::Activity::Create::SUPPORTED_TYPES + ActivityPub::Activity::Create::CONVERTED_TYPES).include? json['type'] + equals_or_includes_any?(json['type'], ActivityPub::Activity::Create::SUPPORTED_TYPES + ActivityPub::Activity::Create::CONVERTED_TYPES) end def process_html(response) diff --git a/app/services/resolve_account_service.rb b/app/services/resolve_account_service.rb index 8cba88f01..de8d1151d 100644 --- a/app/services/resolve_account_service.rb +++ b/app/services/resolve_account_service.rb @@ -189,7 +189,7 @@ class ResolveAccountService < BaseService return @actor_json if defined?(@actor_json) json = fetch_resource(actor_url, false) - @actor_json = supported_context?(json) && ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES.include?(json['type']) ? json : nil + @actor_json = supported_context?(json) && equals_or_includes_any?(json['type'], ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES) ? json : nil end def atom diff --git a/app/services/resolve_url_service.rb b/app/services/resolve_url_service.rb index c19b568cb..a068c1ed8 100644 --- a/app/services/resolve_url_service.rb +++ b/app/services/resolve_url_service.rb @@ -16,10 +16,9 @@ class ResolveURLService < BaseService private def process_url - case type - when 'Application', 'Group', 'Organization', 'Person', 'Service' + if equals_or_includes_any?(type, %w(Application Group Organization Person Service)) FetchRemoteAccountService.new.call(atom_url, body, protocol) - when 'Note', 'Article', 'Image', 'Video' + elsif equals_or_includes_any?(type, %w(Note Article Image Video)) FetchRemoteStatusService.new.call(atom_url, body, protocol) end end diff --git a/spec/fixtures/requests/activitypub-actor-individual.txt b/spec/fixtures/requests/activitypub-actor-individual.txt new file mode 100644 index 000000000..74411e544 --- /dev/null +++ b/spec/fixtures/requests/activitypub-actor-individual.txt @@ -0,0 +1,9 @@ +HTTP/1.1 200 OK +Cache-Control: max-age=0, private, must-revalidate +Content-Type: application/activity+json; charset=utf-8 +Link: ; rel="lrdd"; type="application/xrd+xml", ; rel="alternate"; type="application/atom+xml", ; rel="alternate"; type="application/activity+json" +Vary: Accept-Encoding +X-Content-Type-Options: nosniff +X-Xss-Protection: 1; mode=block + +{"@context":["https://www.w3.org/ns/activitystreams","https://w3id.org/security/v1",{"vcard": "http://www.w3.org/2006/vcard/ns#"},{"manuallyApprovesFollowers":"as:manuallyApprovesFollowers","sensitive":"as:sensitive","Hashtag":"as:Hashtag","ostatus":"http://ostatus.org#","atomUri":"ostatus:atomUri","inReplyToAtomUri":"ostatus:inReplyToAtomUri","conversation":"ostatus:conversation"}],"id":"https://ap.example.com/users/foo","type":["Person","vcard:individual"],"following":"https://ap.example.com/users/foo/following","followers":"https://ap.example.com/users/foo/followers","inbox":"https://ap.example.com/users/foo/inbox","outbox":"https://ap.example.com/users/foo/outbox","preferredUsername":"foo","vcard:fn":"foo","name":"","summary":"\u003cp\u003etest\u003c/p\u003e","url":"https://ap.example.com/@foo","manuallyApprovesFollowers":false,"publicKey":{"id":"https://ap.example.com/users/foo#main-key","owner":"https://ap.example.com/users/foo","publicKeyPem":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAu3L4vnpNLzVH31MeWI39\n4F0wKeJFsLDAsNXGeOu0QF2x+h1zLWZw/agqD2R3JPU9/kaDJGPIV2Sn5zLyUA9S\n6swCCMOtn7BBR9g9sucgXJmUFB0tACH2QSgHywMAybGfmSb3LsEMNKsGJ9VsvYoh\n8lDET6X4Pyw+ZJU0/OLo/41q9w+OrGtlsTm/PuPIeXnxa6BLqnDaxC+4IcjG/FiP\nahNCTINl/1F/TgSSDZ4Taf4U9XFEIFw8wmgploELozzIzKq+t8nhQYkgAkt64euW\npva3qL5KD1mTIZQEP+LZvh3s2WHrLi3fhbdRuwQ2c0KkJA2oSTFPDpqqbPGZ3Qvu\nHQIDAQAB\n-----END PUBLIC KEY-----\n"},"endpoints":{"sharedInbox":"https://ap.example.com/inbox"},"icon":{"type":"Image","url":"https://quitter.no/avatar/7477-300-20160211190340.png"}} \ No newline at end of file diff --git a/spec/services/resolve_account_service_spec.rb b/spec/services/resolve_account_service_spec.rb index 5f1b4467b..84dfe578a 100644 --- a/spec/services/resolve_account_service_spec.rb +++ b/spec/services/resolve_account_service_spec.rb @@ -105,6 +105,20 @@ RSpec.describe ResolveAccountService do expect(account.inbox_url).to eq 'https://ap.example.com/users/foo/inbox' end + context 'with multiple types' do + before do + stub_request(:get, "https://ap.example.com/users/foo").to_return(request_fixture('activitypub-actor-individual.txt')) + end + + it 'returns new remote account' do + account = subject.call('foo@ap.example.com') + + expect(account.activitypub?).to eq true + expect(account.domain).to eq 'ap.example.com' + expect(account.inbox_url).to eq 'https://ap.example.com/users/foo/inbox' + end + end + pending end -- cgit From cb5b5cb5f79bb2187d8124df91af4c8e1bfd7256 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Wed, 2 May 2018 18:58:48 +0200 Subject: Slightly reduce RAM usage (#7301) * No need to re-require sidekiq plugins, they are required via Gemfile * Add derailed_benchmarks tool, no need to require TTY gems in Gemfile * Replace ruby-oembed with FetchOEmbedService Reduce startup by 45382 allocated objects * Remove preloaded JSON-LD in favour of caching HTTP responses Reduce boot RAM by about 6 MiB * Fix tests * Fix test suite by stubbing out JSON-LD contexts --- Gemfile | 12 +- Gemfile.lock | 22 +- app/controllers/api/web/embeds_controller.rb | 11 +- .../settings/follower_domains_controller.rb | 2 - app/helpers/jsonld_helper.rb | 17 +- app/lib/provider_discovery.rb | 47 --- app/services/fan_out_on_write_service.rb | 2 - app/services/fetch_link_card_service.rb | 38 +- app/services/fetch_oembed_service.rb | 71 ++++ app/workers/scheduler/backup_cleanup_scheduler.rb | 1 - .../scheduler/doorkeeper_cleanup_scheduler.rb | 1 - app/workers/scheduler/email_scheduler.rb | 1 - app/workers/scheduler/feed_cleanup_scheduler.rb | 1 - app/workers/scheduler/ip_cleanup_scheduler.rb | 1 - app/workers/scheduler/media_cleanup_scheduler.rb | 1 - .../scheduler/subscriptions_cleanup_scheduler.rb | 2 - app/workers/scheduler/subscriptions_scheduler.rb | 3 - app/workers/scheduler/user_cleanup_scheduler.rb | 1 - app/workers/soft_block_domain_followers_worker.rb | 2 - config/initializers/json_ld.rb | 5 - config/initializers/oembed.rb | 4 - lib/json_ld/activitystreams.rb | 153 -------- lib/json_ld/identity.rb | 86 ----- lib/json_ld/security.rb | 50 --- lib/tasks/mastodon.rake | 2 + spec/fixtures/requests/json-ld.activitystreams.txt | 391 +++++++++++++++++++++ spec/fixtures/requests/json-ld.identity.txt | 100 ++++++ spec/fixtures/requests/json-ld.security.txt | 61 ++++ spec/lib/activitypub/linked_data_signature_spec.rb | 4 + spec/lib/provider_discovery_spec.rb | 118 ------- spec/rails_helper.rb | 14 + spec/services/account_search_service_spec.rb | 2 +- .../fetch_remote_account_service_spec.rb | 2 +- .../fetch_remote_status_service_spec.rb | 2 +- .../activitypub/process_account_service_spec.rb | 2 +- .../activitypub/process_collection_service_spec.rb | 2 +- spec/services/after_block_service_spec.rb | 2 +- spec/services/authorize_follow_service_spec.rb | 2 +- .../services/batched_remove_status_service_spec.rb | 2 +- .../block_domain_from_account_service_spec.rb | 2 +- spec/services/block_domain_service_spec.rb | 2 +- spec/services/block_service_spec.rb | 2 +- spec/services/bootstrap_timeline_service_spec.rb | 2 +- spec/services/fan_out_on_write_service_spec.rb | 2 +- spec/services/favourite_service_spec.rb | 2 +- spec/services/fetch_atom_service_spec.rb | 2 +- spec/services/fetch_link_card_service_spec.rb | 2 +- spec/services/fetch_oembed_service_spec.rb | 125 +++++++ spec/services/fetch_remote_account_service_spec.rb | 2 +- spec/services/fetch_remote_status_service_spec.rb | 2 +- spec/services/follow_service_spec.rb | 2 +- spec/services/mute_service_spec.rb | 2 +- spec/services/notify_service_spec.rb | 2 +- spec/services/post_status_service_spec.rb | 2 +- spec/services/precompute_feed_service_spec.rb | 2 +- spec/services/process_feed_service_spec.rb | 2 +- spec/services/process_interaction_service_spec.rb | 2 +- spec/services/process_mentions_service_spec.rb | 2 +- .../pubsubhubbub/subscribe_service_spec.rb | 2 +- .../pubsubhubbub/unsubscribe_service_spec.rb | 2 +- spec/services/reblog_service_spec.rb | 2 +- spec/services/reject_follow_service_spec.rb | 2 +- spec/services/remove_status_service_spec.rb | 2 +- spec/services/report_service_spec.rb | 2 +- spec/services/resolve_account_service_spec.rb | 2 +- spec/services/resolve_url_service_spec.rb | 2 +- spec/services/search_service_spec.rb | 2 +- spec/services/send_interaction_service_spec.rb | 2 +- spec/services/subscribe_service_spec.rb | 2 +- spec/services/suspend_account_service_spec.rb | 2 +- spec/services/unblock_domain_service_spec.rb | 2 +- spec/services/unblock_service_spec.rb | 2 +- spec/services/unfollow_service_spec.rb | 2 +- spec/services/unmute_service_spec.rb | 2 +- spec/services/unsubscribe_service_spec.rb | 2 +- .../services/update_remote_profile_service_spec.rb | 2 +- spec/spec_helper.rb | 12 +- 77 files changed, 881 insertions(+), 568 deletions(-) delete mode 100644 app/lib/provider_discovery.rb create mode 100644 app/services/fetch_oembed_service.rb delete mode 100644 config/initializers/json_ld.rb delete mode 100644 config/initializers/oembed.rb delete mode 100644 lib/json_ld/activitystreams.rb delete mode 100644 lib/json_ld/identity.rb delete mode 100644 lib/json_ld/security.rb create mode 100644 spec/fixtures/requests/json-ld.activitystreams.txt create mode 100644 spec/fixtures/requests/json-ld.identity.txt create mode 100644 spec/fixtures/requests/json-ld.security.txt delete mode 100644 spec/lib/provider_discovery_spec.rb create mode 100644 spec/services/fetch_oembed_service_spec.rb (limited to 'spec/fixtures/requests') diff --git a/Gemfile b/Gemfile index a33748568..f1665ce95 100644 --- a/Gemfile +++ b/Gemfile @@ -54,7 +54,7 @@ gem 'httplog', '~> 1.0' gem 'idn-ruby', require: 'idn' gem 'kaminari', '~> 1.1' gem 'link_header', '~> 0.0' -gem 'mime-types', '~> 3.1' +gem 'mime-types', '~> 3.1', require: 'mime/types/columnar' gem 'nokogiri', '~> 1.8' gem 'nsa', '~> 0.2' gem 'oj', '~> 3.5' @@ -70,7 +70,6 @@ gem 'rails-settings-cached', '~> 0.6' gem 'redis', '~> 4.0', require: ['redis', 'redis/connection/hiredis'] gem 'mario-redis-lock', '~> 1.2', require: 'redis_lock' gem 'rqrcode', '~> 0.10' -gem 'ruby-oembed', '~> 0.12', require: 'oembed' gem 'ruby-progressbar', '~> 1.4' gem 'sanitize', '~> 4.6' gem 'sidekiq', '~> 5.1' @@ -82,14 +81,14 @@ gem 'simple_form', '~> 4.0' gem 'sprockets-rails', '~> 3.2', require: 'sprockets/railtie' gem 'stoplight', '~> 2.1.3' gem 'strong_migrations', '~> 0.2' -gem 'tty-command', '~> 0.8' -gem 'tty-prompt', '~> 0.16' +gem 'tty-command', '~> 0.8', require: false +gem 'tty-prompt', '~> 0.16', require: false gem 'twitter-text', '~> 1.14' gem 'tzinfo-data', '~> 1.2018' gem 'webpacker', '~> 3.4' gem 'webpush' -gem 'json-ld-preloaded', '~> 2.2' +gem 'json-ld', '~> 2.2' gem 'rdf-normalize', '~> 0.3' group :development, :test do @@ -135,6 +134,9 @@ group :development do gem 'capistrano-rails', '~> 1.3' gem 'capistrano-rbenv', '~> 2.1' gem 'capistrano-yarn', '~> 2.0' + + gem 'derailed_benchmarks' + gem 'stackprof' end group :production do diff --git a/Gemfile.lock b/Gemfile.lock index d96165dcf..94ab0b7ca 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -75,6 +75,7 @@ GEM aws-sigv4 (~> 1.0) aws-sigv4 (1.0.2) bcrypt (3.1.11) + benchmark-ips (2.7.2) better_errors (2.4.0) coderay (>= 1.0.0) erubi (>= 1.0.0) @@ -138,6 +139,14 @@ GEM css_parser (1.6.0) addressable debug_inspector (0.0.3) + derailed_benchmarks (1.3.4) + benchmark-ips (~> 2) + get_process_mem (~> 0) + heapy (~> 0) + memory_profiler (~> 0) + rack (>= 1) + rake (> 10, < 13) + thor (~> 0.19) devise (4.4.3) bcrypt (~> 3.0) orm_adapter (~> 0.1) @@ -206,6 +215,7 @@ GEM fuubar (2.3.1) rspec-core (~> 3.0) ruby-progressbar (~> 1.4) + get_process_mem (0.2.1) globalid (0.4.1) activesupport (>= 4.2.0) goldfinger (2.1.0) @@ -226,6 +236,7 @@ GEM concurrent-ruby (~> 1.0) hashdiff (0.3.7) hashie (3.5.7) + heapy (0.1.3) highline (1.7.10) hiredis (0.6.1) hitimes (1.2.6) @@ -264,10 +275,6 @@ GEM json-ld (2.2.1) multi_json (~> 1.12) rdf (>= 2.2.8, < 4.0) - json-ld-preloaded (2.2.3) - json-ld (>= 2.2, < 4.0) - multi_json (~> 1.12) - rdf (>= 2.2, < 4.0) jsonapi-renderer (0.2.0) jwt (2.1.0) kaminari (1.1.1) @@ -502,7 +509,6 @@ GEM rainbow (>= 2.2.2, < 4.0) ruby-progressbar (~> 1.7) unicode-display_width (~> 1.0, >= 1.0.1) - ruby-oembed (0.12.0) ruby-progressbar (1.9.0) ruby-saml (1.7.2) nokogiri (>= 1.5.10) @@ -557,6 +563,7 @@ GEM sshkit (1.16.0) net-scp (>= 1.1.2) net-ssh (>= 2.8.0) + stackprof (0.2.11) statsd-ruby (1.2.1) stoplight (2.1.3) streamio-ffmpeg (3.0.2) @@ -645,6 +652,7 @@ DEPENDENCIES chewy (~> 5.0) cld3 (~> 3.2.0) climate_control (~> 0.2) + derailed_benchmarks devise (~> 4.4) devise-two-factor (~> 3.0) devise_pam_authenticatable2 (~> 9.1) @@ -668,7 +676,7 @@ DEPENDENCIES i18n-tasks (~> 0.9) idn-ruby iso-639 - json-ld-preloaded (~> 2.2) + json-ld (~> 2.2) kaminari (~> 1.1) letter_opener (~> 1.4) letter_opener_web (~> 1.3) @@ -714,7 +722,6 @@ DEPENDENCIES rspec-retry (~> 0.5) rspec-sidekiq (~> 3.0) rubocop (~> 0.55) - ruby-oembed (~> 0.12) ruby-progressbar (~> 1.4) sanitize (~> 4.6) scss_lint (~> 0.57) @@ -726,6 +733,7 @@ DEPENDENCIES simple_form (~> 4.0) simplecov (~> 0.16) sprockets-rails (~> 3.2) + stackprof stoplight (~> 2.1.3) streamio-ffmpeg (~> 3.0) strong_migrations (~> 0.2) diff --git a/app/controllers/api/web/embeds_controller.rb b/app/controllers/api/web/embeds_controller.rb index f2fe74b17..987290a14 100644 --- a/app/controllers/api/web/embeds_controller.rb +++ b/app/controllers/api/web/embeds_controller.rb @@ -9,9 +9,12 @@ class Api::Web::EmbedsController < Api::Web::BaseController status = StatusFinder.new(params[:url]).status render json: status, serializer: OEmbedSerializer, width: 400 rescue ActiveRecord::RecordNotFound - oembed = OEmbed::Providers.get(params[:url]) - render json: Oj.dump(oembed.fields) - rescue OEmbed::NotFound - render json: {}, status: :not_found + oembed = FetchOEmbedService.new.call(params[:url]) + + if oembed + render json: oembed + else + render json: {}, status: :not_found + end end end diff --git a/app/controllers/settings/follower_domains_controller.rb b/app/controllers/settings/follower_domains_controller.rb index 213d9e96d..91b521e7f 100644 --- a/app/controllers/settings/follower_domains_controller.rb +++ b/app/controllers/settings/follower_domains_controller.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require 'sidekiq-bulk' - class Settings::FollowerDomainsController < ApplicationController layout 'admin' diff --git a/app/helpers/jsonld_helper.rb b/app/helpers/jsonld_helper.rb index a3cfdadb8..e9056166c 100644 --- a/app/helpers/jsonld_helper.rb +++ b/app/helpers/jsonld_helper.rb @@ -48,7 +48,7 @@ module JsonLdHelper end def canonicalize(json) - graph = RDF::Graph.new << JSON::LD::API.toRdf(json) + graph = RDF::Graph.new << JSON::LD::API.toRdf(json, documentLoader: method(:load_jsonld_context)) graph.dump(:normalize) end @@ -90,4 +90,19 @@ module JsonLdHelper request.add_headers('Accept' => 'application/activity+json, application/ld+json') request end + + def load_jsonld_context(url, _options = {}, &_block) + json = Rails.cache.fetch("jsonld:context:#{url}", expires_in: 30.days, raw: true) do + request = Request.new(:get, url) + request.add_headers('Accept' => 'application/ld+json') + + request.perform do |res| + raise JSON::LD::JsonLdError::LoadingDocumentFailed unless res.code == 200 && res.mime_type == 'application/ld+json' + res.body_with_limit + end + end + + doc = JSON::LD::API::RemoteDocument.new(url, json) + block_given? ? yield(doc) : doc + end end diff --git a/app/lib/provider_discovery.rb b/app/lib/provider_discovery.rb deleted file mode 100644 index 3bec7211b..000000000 --- a/app/lib/provider_discovery.rb +++ /dev/null @@ -1,47 +0,0 @@ -# frozen_string_literal: true - -class ProviderDiscovery < OEmbed::ProviderDiscovery - class << self - def get(url, **options) - provider = discover_provider(url, options) - - options.delete(:html) - - provider.get(url, options) - end - - def discover_provider(url, **options) - format = options[:format] - - html = if options[:html] - Nokogiri::HTML(options[:html]) - else - Request.new(:get, url).perform do |res| - raise OEmbed::NotFound, url if res.code != 200 || res.mime_type != 'text/html' - Nokogiri::HTML(res.body_with_limit) - end - end - - if format.nil? || format == :json - provider_endpoint ||= html.at_xpath('//link[@type="application/json+oembed"]')&.attribute('href')&.value - format ||= :json if provider_endpoint - end - - if format.nil? || format == :xml - provider_endpoint ||= html.at_xpath('//link[@type="text/xml+oembed"]')&.attribute('href')&.value - format ||= :xml if provider_endpoint - end - - raise OEmbed::NotFound, url if provider_endpoint.nil? - begin - provider_endpoint = Addressable::URI.parse(provider_endpoint) - provider_endpoint.query = nil - provider_endpoint = provider_endpoint.to_s - rescue Addressable::URI::InvalidURIError - raise OEmbed::NotFound, url - end - - OEmbed::Provider.new(provider_endpoint, format) - end - end -end diff --git a/app/services/fan_out_on_write_service.rb b/app/services/fan_out_on_write_service.rb index 0f77556dc..510b80c82 100644 --- a/app/services/fan_out_on_write_service.rb +++ b/app/services/fan_out_on_write_service.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require 'sidekiq-bulk' - class FanOutOnWriteService < BaseService # Push a status into home and mentions feeds # @param [Status] status diff --git a/app/services/fetch_link_card_service.rb b/app/services/fetch_link_card_service.rb index d5920a417..77d4aa538 100644 --- a/app/services/fetch_link_card_service.rb +++ b/app/services/fetch_link_card_service.rb @@ -85,42 +85,40 @@ class FetchLinkCardService < BaseService end def attempt_oembed - embed = OEmbed::Providers.get(@url, html: @html) + embed = FetchOEmbedService.new.call(@url, html: @html) - return false unless embed.respond_to?(:type) + return false if embed.nil? - @card.type = embed.type - @card.title = embed.respond_to?(:title) ? embed.title : '' - @card.author_name = embed.respond_to?(:author_name) ? embed.author_name : '' - @card.author_url = embed.respond_to?(:author_url) ? embed.author_url : '' - @card.provider_name = embed.respond_to?(:provider_name) ? embed.provider_name : '' - @card.provider_url = embed.respond_to?(:provider_url) ? embed.provider_url : '' + @card.type = embed[:type] + @card.title = embed[:title] || '' + @card.author_name = embed[:author_name] || '' + @card.author_url = embed[:author_url] || '' + @card.provider_name = embed[:provider_name] || '' + @card.provider_url = embed[:provider_url] || '' @card.width = 0 @card.height = 0 case @card.type when 'link' - @card.image_remote_url = embed.thumbnail_url if embed.respond_to?(:thumbnail_url) + @card.image_remote_url = embed[:thumbnail_url] if embed[:thumbnail_url].present? when 'photo' - return false unless embed.respond_to?(:url) + return false if embed[:url].blank? - @card.embed_url = embed.url - @card.image_remote_url = embed.url - @card.width = embed.width.presence || 0 - @card.height = embed.height.presence || 0 + @card.embed_url = embed[:url] + @card.image_remote_url = embed[:url] + @card.width = embed[:width].presence || 0 + @card.height = embed[:height].presence || 0 when 'video' - @card.width = embed.width.presence || 0 - @card.height = embed.height.presence || 0 - @card.html = Formatter.instance.sanitize(embed.html, Sanitize::Config::MASTODON_OEMBED) - @card.image_remote_url = embed.thumbnail_url if embed.respond_to?(:thumbnail_url) + @card.width = embed[:width].presence || 0 + @card.height = embed[:height].presence || 0 + @card.html = Formatter.instance.sanitize(embed[:html], Sanitize::Config::MASTODON_OEMBED) + @card.image_remote_url = embed[:thumbnail_url] if embed[:thumbnail_url].present? when 'rich' # Most providers rely on