From 495303d9b86919c72bf1948a714bf8d00b41fa0f Mon Sep 17 00:00:00 2001 From: Emelia Smith Date: Mon, 23 Apr 2018 21:27:18 +0200 Subject: Prevent suspended accounts from appearing in AccountSearchService (#7246) --- spec/services/account_search_service_spec.rb | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) (limited to 'spec/services') diff --git a/spec/services/account_search_service_spec.rb b/spec/services/account_search_service_spec.rb index 9bb27edad..c5ddc5844 100644 --- a/spec/services/account_search_service_spec.rb +++ b/spec/services/account_search_service_spec.rb @@ -137,5 +137,24 @@ describe AccountSearchService do expect(service).not_to have_received(:call) end end + + describe 'should not include suspended accounts' do + it 'returns the fuzzy match first, and does not return suspended exacts' do + partial = Fabricate(:account, username: 'exactness') + exact = Fabricate(:account, username: 'exact', suspended: true) + + results = subject.call('exact', 10) + expect(results.size).to eq 1 + expect(results).to eq [partial] + end + + it "does not return suspended remote accounts" do + remote = Fabricate(:account, username: 'a', domain: 'remote', display_name: 'e', suspended: true) + + results = subject.call('a@example.com', 2) + expect(results.size).to eq 0 + expect(results).to eq [] + end + end end end -- cgit From eb593a5a0c9ee9749115f867a5675ceb1f231379 Mon Sep 17 00:00:00 2001 From: MIYAGI Hikaru Date: Wed, 25 Apr 2018 21:12:28 +0900 Subject: Append '.test' to hostname in stub data (#7260) --- spec/fixtures/requests/oembed_json.html | 2 +- spec/fixtures/requests/oembed_json_xml.html | 4 +-- spec/fixtures/requests/oembed_xml.html | 2 +- spec/helpers/jsonld_helper_spec.rb | 24 +++++++------- spec/lib/formatter_spec.rb | 6 ++-- spec/lib/ostatus/atom_serializer_spec.rb | 26 +++++++-------- spec/lib/provider_discovery_spec.rb | 38 +++++++++++----------- spec/lib/tag_manager_spec.rb | 34 +++++++++---------- spec/models/account_spec.rb | 10 +++--- spec/models/status_pin_spec.rb | 2 +- .../activitypub/process_account_service_spec.rb | 4 +-- 11 files changed, 76 insertions(+), 76 deletions(-) (limited to 'spec/services') diff --git a/spec/fixtures/requests/oembed_json.html b/spec/fixtures/requests/oembed_json.html index 773a4f92a..167085871 100644 --- a/spec/fixtures/requests/oembed_json.html +++ b/spec/fixtures/requests/oembed_json.html @@ -1,7 +1,7 @@ - + diff --git a/spec/fixtures/requests/oembed_json_xml.html b/spec/fixtures/requests/oembed_json_xml.html index 8afd8e997..9f5b9e8be 100644 --- a/spec/fixtures/requests/oembed_json_xml.html +++ b/spec/fixtures/requests/oembed_json_xml.html @@ -7,8 +7,8 @@ > The type attribute must contain either application/json+oembed for JSON > responses, or text/xml+oembed for XML. --> - - + + diff --git a/spec/fixtures/requests/oembed_xml.html b/spec/fixtures/requests/oembed_xml.html index bdfcca170..788dfaabd 100644 --- a/spec/fixtures/requests/oembed_xml.html +++ b/spec/fixtures/requests/oembed_xml.html @@ -7,7 +7,7 @@ > The type attribute must contain either application/json+oembed for JSON > responses, or text/xml+oembed for XML. --> - + diff --git a/spec/helpers/jsonld_helper_spec.rb b/spec/helpers/jsonld_helper_spec.rb index 48bfdc306..a5ab249c2 100644 --- a/spec/helpers/jsonld_helper_spec.rb +++ b/spec/helpers/jsonld_helper_spec.rb @@ -32,37 +32,37 @@ describe JsonLdHelper do describe '#fetch_resource' do context 'when the second argument is false' do it 'returns resource even if the retrieved ID and the given URI does not match' do - stub_request(:get, 'https://bob/').to_return body: '{"id": "https://alice/"}' - stub_request(:get, 'https://alice/').to_return body: '{"id": "https://alice/"}' + stub_request(:get, 'https://bob.test/').to_return body: '{"id": "https://alice.test/"}' + stub_request(:get, 'https://alice.test/').to_return body: '{"id": "https://alice.test/"}' - expect(fetch_resource('https://bob/', false)).to eq({ 'id' => 'https://alice/' }) + expect(fetch_resource('https://bob.test/', false)).to eq({ 'id' => 'https://alice.test/' }) end it 'returns nil if the object identified by the given URI and the object identified by the retrieved ID does not match' do - stub_request(:get, 'https://mallory/').to_return body: '{"id": "https://marvin/"}' - stub_request(:get, 'https://marvin/').to_return body: '{"id": "https://alice/"}' + stub_request(:get, 'https://mallory.test/').to_return body: '{"id": "https://marvin.test/"}' + stub_request(:get, 'https://marvin.test/').to_return body: '{"id": "https://alice.test/"}' - expect(fetch_resource('https://mallory/', false)).to eq nil + expect(fetch_resource('https://mallory.test/', false)).to eq nil end end context 'when the second argument is true' do it 'returns nil if the retrieved ID and the given URI does not match' do - stub_request(:get, 'https://mallory/').to_return body: '{"id": "https://alice/"}' - expect(fetch_resource('https://mallory/', true)).to eq nil + stub_request(:get, 'https://mallory.test/').to_return body: '{"id": "https://alice.test/"}' + expect(fetch_resource('https://mallory.test/', true)).to eq nil end end end describe '#fetch_resource_without_id_validation' do it 'returns nil if the status code is not 200' do - stub_request(:get, 'https://host/').to_return status: 400, body: '{}' - expect(fetch_resource_without_id_validation('https://host/')).to eq nil + stub_request(:get, 'https://host.test/').to_return status: 400, body: '{}' + expect(fetch_resource_without_id_validation('https://host.test/')).to eq nil end it 'returns hash' do - stub_request(:get, 'https://host/').to_return status: 200, body: '{}' - expect(fetch_resource_without_id_validation('https://host/')).to eq({}) + stub_request(:get, 'https://host.test/').to_return status: 200, body: '{}' + expect(fetch_resource_without_id_validation('https://host.test/')).to eq({}) end end end diff --git a/spec/lib/formatter_spec.rb b/spec/lib/formatter_spec.rb index 6e849f379..b8683e720 100644 --- a/spec/lib/formatter_spec.rb +++ b/spec/lib/formatter_spec.rb @@ -2,7 +2,7 @@ require 'rails_helper' RSpec.describe Formatter do let(:local_account) { Fabricate(:account, domain: nil, username: 'alice') } - let(:remote_account) { Fabricate(:account, domain: 'remote', username: 'bob', url: 'https://remote/') } + let(:remote_account) { Fabricate(:account, domain: 'remote.test', username: 'bob', url: 'https://remote.test/') } shared_examples 'encode and link URLs' do context 'matches a stand-alone medium URL' do @@ -377,12 +377,12 @@ RSpec.describe Formatter do end context 'contains linkable mentions for remote accounts' do - let(:text) { '@bob@remote' } + let(:text) { '@bob@remote.test' } before { remote_account } it 'links' do - is_expected.to eq '

' + is_expected.to eq '

' end end diff --git a/spec/lib/ostatus/atom_serializer_spec.rb b/spec/lib/ostatus/atom_serializer_spec.rb index d46791307..0bd22880e 100644 --- a/spec/lib/ostatus/atom_serializer_spec.rb +++ b/spec/lib/ostatus/atom_serializer_spec.rb @@ -30,13 +30,13 @@ RSpec.describe OStatus::AtomSerializer do end it 'appends activity:object with target account' do - target_account = Fabricate(:account, domain: 'domain', uri: 'https://domain/id') + target_account = Fabricate(:account, domain: 'domain.test', uri: 'https://domain.test/id') follow_request = Fabricate(:follow_request, target_account: target_account) follow_request_salmon = serialize(follow_request) object = follow_request_salmon.nodes.find { |node| node.name == 'activity:object' } - expect(object.id.text).to eq 'https://domain/id' + expect(object.id.text).to eq 'https://domain.test/id' end end @@ -413,20 +413,20 @@ RSpec.describe OStatus::AtomSerializer do entry = OStatus::AtomSerializer.new.entry(remote_status.stream_entry, true) entry.nodes.delete_if { |node| node[:type] == 'application/activity+json' } # Remove ActivityPub link to simplify test - xml = OStatus::AtomSerializer.render(entry).gsub('cb6e6126.ngrok.io', 'remote') + xml = OStatus::AtomSerializer.render(entry).gsub('cb6e6126.ngrok.io', 'remote.test') remote_status.destroy! remote_account.destroy! account = Account.create!( - domain: 'remote', + domain: 'remote.test', username: 'username', last_webfingered_at: Time.now.utc ) ProcessFeedService.new.call(xml, account) - expect(Status.find_by(uri: "https://remote/users/#{remote_status.account.to_param}/statuses/#{remote_status.id}")).to be_instance_of Status + expect(Status.find_by(uri: "https://remote.test/users/#{remote_status.account.to_param}/statuses/#{remote_status.id}")).to be_instance_of Status end end @@ -776,13 +776,13 @@ RSpec.describe OStatus::AtomSerializer do end it 'appends activity:object element with target account' do - target_account = Fabricate(:account, domain: 'domain', uri: 'https://domain/id') + target_account = Fabricate(:account, domain: 'domain.test', uri: 'https://domain.test/id') block = Fabricate(:block, target_account: target_account) block_salmon = OStatus::AtomSerializer.new.block_salmon(block) object = block_salmon.nodes.find { |node| node.name == 'activity:object' } - expect(object.id.text).to eq 'https://domain/id' + expect(object.id.text).to eq 'https://domain.test/id' end it 'returns element whose rendered view triggers block when processed' do @@ -863,13 +863,13 @@ RSpec.describe OStatus::AtomSerializer do end it 'appends activity:object element with target account' do - target_account = Fabricate(:account, domain: 'domain', uri: 'https://domain/id') + target_account = Fabricate(:account, domain: 'domain.test', uri: 'https://domain.test/id') block = Fabricate(:block, target_account: target_account) unblock_salmon = OStatus::AtomSerializer.new.unblock_salmon(block) object = unblock_salmon.nodes.find { |node| node.name == 'activity:object' } - expect(object.id.text).to eq 'https://domain/id' + expect(object.id.text).to eq 'https://domain.test/id' end it 'returns element whose rendered view triggers block when processed' do @@ -1124,13 +1124,13 @@ RSpec.describe OStatus::AtomSerializer do end it 'appends activity:object element with target account' do - target_account = Fabricate(:account, domain: 'domain', uri: 'https://domain/id') + target_account = Fabricate(:account, domain: 'domain.test', uri: 'https://domain.test/id') follow = Fabricate(:follow, target_account: target_account) follow_salmon = OStatus::AtomSerializer.new.follow_salmon(follow) object = follow_salmon.nodes.find { |node| node.name == 'activity:object' } - expect(object.id.text).to eq 'https://domain/id' + expect(object.id.text).to eq 'https://domain.test/id' end it 'includes description' do @@ -1242,14 +1242,14 @@ RSpec.describe OStatus::AtomSerializer do end it 'appends activity:object element with target account' do - target_account = Fabricate(:account, domain: 'domain', uri: 'https://domain/id') + target_account = Fabricate(:account, domain: 'domain.test', uri: 'https://domain.test/id') follow = Fabricate(:follow, target_account: target_account) follow.destroy! unfollow_salmon = OStatus::AtomSerializer.new.unfollow_salmon(follow) object = unfollow_salmon.nodes.find { |node| node.name == 'activity:object' } - expect(object.id.text).to eq 'https://domain/id' + expect(object.id.text).to eq 'https://domain.test/id' end it 'returns element whose rendered view triggers unfollow when processed' do diff --git a/spec/lib/provider_discovery_spec.rb b/spec/lib/provider_discovery_spec.rb index 12e2616c9..de2ac16f9 100644 --- a/spec/lib/provider_discovery_spec.rb +++ b/spec/lib/provider_discovery_spec.rb @@ -7,7 +7,7 @@ describe ProviderDiscovery do context 'when status code is 200 and MIME type is text/html' do context 'Both of JSON and XML provider are discoverable' do before do - stub_request(:get, 'https://host/oembed.html').to_return( + stub_request(:get, 'https://host.test/oembed.html').to_return( status: 200, headers: { 'Content-Type': 'text/html' }, body: request_fixture('oembed_json_xml.html') @@ -15,21 +15,21 @@ describe ProviderDiscovery do end it 'returns new OEmbed::Provider for JSON provider if :format option is set to :json' do - provider = ProviderDiscovery.discover_provider('https://host/oembed.html', format: :json) - expect(provider.endpoint).to eq 'https://host/provider.json' + provider = ProviderDiscovery.discover_provider('https://host.test/oembed.html', format: :json) + expect(provider.endpoint).to eq 'https://host.test/provider.json' expect(provider.format).to eq :json end it 'returns new OEmbed::Provider for XML provider if :format option is set to :xml' do - provider = ProviderDiscovery.discover_provider('https://host/oembed.html', format: :xml) - expect(provider.endpoint).to eq 'https://host/provider.xml' + provider = ProviderDiscovery.discover_provider('https://host.test/oembed.html', format: :xml) + expect(provider.endpoint).to eq 'https://host.test/provider.xml' expect(provider.format).to eq :xml end end context 'JSON provider is discoverable while XML provider is not' do before do - stub_request(:get, 'https://host/oembed.html').to_return( + stub_request(:get, 'https://host.test/oembed.html').to_return( status: 200, headers: { 'Content-Type': 'text/html' }, body: request_fixture('oembed_json.html') @@ -37,15 +37,15 @@ describe ProviderDiscovery do end it 'returns new OEmbed::Provider for JSON provider' do - provider = ProviderDiscovery.discover_provider('https://host/oembed.html') - expect(provider.endpoint).to eq 'https://host/provider.json' + provider = ProviderDiscovery.discover_provider('https://host.test/oembed.html') + expect(provider.endpoint).to eq 'https://host.test/provider.json' expect(provider.format).to eq :json end end context 'XML provider is discoverable while JSON provider is not' do before do - stub_request(:get, 'https://host/oembed.html').to_return( + stub_request(:get, 'https://host.test/oembed.html').to_return( status: 200, headers: { 'Content-Type': 'text/html' }, body: request_fixture('oembed_xml.html') @@ -53,15 +53,15 @@ describe ProviderDiscovery do end it 'returns new OEmbed::Provider for XML provider' do - provider = ProviderDiscovery.discover_provider('https://host/oembed.html') - expect(provider.endpoint).to eq 'https://host/provider.xml' + provider = ProviderDiscovery.discover_provider('https://host.test/oembed.html') + expect(provider.endpoint).to eq 'https://host.test/provider.xml' expect(provider.format).to eq :xml end end context 'Invalid XML provider is discoverable while JSON provider is not' do before do - stub_request(:get, 'https://host/oembed.html').to_return( + stub_request(:get, 'https://host.test/oembed.html').to_return( status: 200, headers: { 'Content-Type': 'text/html' }, body: request_fixture('oembed_invalid_xml.html') @@ -69,13 +69,13 @@ describe ProviderDiscovery do end it 'raises OEmbed::NotFound' do - expect { ProviderDiscovery.discover_provider('https://host/oembed.html') }.to raise_error OEmbed::NotFound + expect { ProviderDiscovery.discover_provider('https://host.test/oembed.html') }.to raise_error OEmbed::NotFound end end context 'Neither of JSON and XML provider is discoverable' do before do - stub_request(:get, 'https://host/oembed.html').to_return( + stub_request(:get, 'https://host.test/oembed.html').to_return( status: 200, headers: { 'Content-Type': 'text/html' }, body: request_fixture('oembed_undiscoverable.html') @@ -83,14 +83,14 @@ describe ProviderDiscovery do end it 'raises OEmbed::NotFound' do - expect { ProviderDiscovery.discover_provider('https://host/oembed.html') }.to raise_error OEmbed::NotFound + expect { ProviderDiscovery.discover_provider('https://host.test/oembed.html') }.to raise_error OEmbed::NotFound end end end context 'when status code is not 200' do before do - stub_request(:get, 'https://host/oembed.html').to_return( + stub_request(:get, 'https://host.test/oembed.html').to_return( status: 400, headers: { 'Content-Type': 'text/html' }, body: request_fixture('oembed_xml.html') @@ -98,20 +98,20 @@ describe ProviderDiscovery do end it 'raises OEmbed::NotFound' do - expect { ProviderDiscovery.discover_provider('https://host/oembed.html') }.to raise_error OEmbed::NotFound + expect { ProviderDiscovery.discover_provider('https://host.test/oembed.html') }.to raise_error OEmbed::NotFound end end context 'when MIME type is not text/html' do before do - stub_request(:get, 'https://host/oembed.html').to_return( + stub_request(:get, 'https://host.test/oembed.html').to_return( status: 200, body: request_fixture('oembed_xml.html') ) end it 'raises OEmbed::NotFound' do - expect { ProviderDiscovery.discover_provider('https://host/oembed.html') }.to raise_error OEmbed::NotFound + expect { ProviderDiscovery.discover_provider('https://host.test/oembed.html') }.to raise_error OEmbed::NotFound end end end diff --git a/spec/lib/tag_manager_spec.rb b/spec/lib/tag_manager_spec.rb index 5427a2929..3a804ac0f 100644 --- a/spec/lib/tag_manager_spec.rb +++ b/spec/lib/tag_manager_spec.rb @@ -6,7 +6,7 @@ RSpec.describe TagManager do around do |example| original_local_domain = Rails.configuration.x.local_domain - Rails.configuration.x.local_domain = 'domain' + Rails.configuration.x.local_domain = 'domain.test' example.run @@ -18,11 +18,11 @@ RSpec.describe TagManager do end it 'returns true if the slash-stripped string equals to local domain' do - expect(TagManager.instance.local_domain?('DoMaIn/')).to eq true + expect(TagManager.instance.local_domain?('DoMaIn.Test/')).to eq true end it 'returns false for irrelevant string' do - expect(TagManager.instance.local_domain?('DoMaIn!')).to eq false + expect(TagManager.instance.local_domain?('DoMaIn.Test!')).to eq false end end @@ -31,7 +31,7 @@ RSpec.describe TagManager do around do |example| original_web_domain = Rails.configuration.x.web_domain - Rails.configuration.x.web_domain = 'domain' + Rails.configuration.x.web_domain = 'domain.test' example.run @@ -43,11 +43,11 @@ RSpec.describe TagManager do end it 'returns true if the slash-stripped string equals to web domain' do - expect(TagManager.instance.web_domain?('DoMaIn/')).to eq true + expect(TagManager.instance.web_domain?('DoMaIn.Test/')).to eq true end it 'returns false for string with irrelevant characters' do - expect(TagManager.instance.web_domain?('DoMaIn!')).to eq false + expect(TagManager.instance.web_domain?('DoMaIn.Test!')).to eq false end end @@ -57,7 +57,7 @@ RSpec.describe TagManager do end it 'returns normalized domain' do - expect(TagManager.instance.normalize_domain('DoMaIn/')).to eq 'domain' + expect(TagManager.instance.normalize_domain('DoMaIn.Test/')).to eq 'domain.test' end end @@ -69,18 +69,18 @@ RSpec.describe TagManager do end it 'returns true if the normalized string with port is local URL' do - Rails.configuration.x.web_domain = 'domain:42' - expect(TagManager.instance.local_url?('https://DoMaIn:42/')).to eq true + Rails.configuration.x.web_domain = 'domain.test:42' + expect(TagManager.instance.local_url?('https://DoMaIn.Test:42/')).to eq true end it 'returns true if the normalized string without port is local URL' do - Rails.configuration.x.web_domain = 'domain' - expect(TagManager.instance.local_url?('https://DoMaIn/')).to eq true + Rails.configuration.x.web_domain = 'domain.test' + expect(TagManager.instance.local_url?('https://DoMaIn.Test/')).to eq true end it 'returns false for string with irrelevant characters' do - Rails.configuration.x.web_domain = 'domain' - expect(TagManager.instance.local_url?('https://domainn/')).to eq false + Rails.configuration.x.web_domain = 'domain.test' + expect(TagManager.instance.local_url?('https://domainn.test/')).to eq false end end @@ -88,19 +88,19 @@ RSpec.describe TagManager do # The following comparisons MUST be case-insensitive. it 'returns true if the needle has a correct username and domain for remote user' do - expect(TagManager.instance.same_acct?('username@domain', 'UsErNaMe@DoMaIn')).to eq true + expect(TagManager.instance.same_acct?('username@domain.test', 'UsErNaMe@DoMaIn.Test')).to eq true end it 'returns false if the needle is missing a domain for remote user' do - expect(TagManager.instance.same_acct?('username@domain', 'UsErNaMe')).to eq false + expect(TagManager.instance.same_acct?('username@domain.test', 'UsErNaMe')).to eq false end it 'returns false if the needle has an incorrect domain for remote user' do - expect(TagManager.instance.same_acct?('username@domain', 'UsErNaMe@incorrect')).to eq false + expect(TagManager.instance.same_acct?('username@domain.test', 'UsErNaMe@incorrect.test')).to eq false end it 'returns false if the needle has an incorrect username for remote user' do - expect(TagManager.instance.same_acct?('username@domain', 'incorrect@DoMaIn')).to eq false + expect(TagManager.instance.same_acct?('username@domain.test', 'incorrect@DoMaIn.test')).to eq false end it 'returns true if the needle has a correct username and domain for local user' do diff --git a/spec/models/account_spec.rb b/spec/models/account_spec.rb index a8b24d0e2..fb7ddfa83 100644 --- a/spec/models/account_spec.rb +++ b/spec/models/account_spec.rb @@ -94,14 +94,14 @@ RSpec.describe Account, type: :model do describe '#save_with_optional_media!' do before do - stub_request(:get, 'https://remote/valid_avatar').to_return(request_fixture('avatar.txt')) - stub_request(:get, 'https://remote/invalid_avatar').to_return(request_fixture('feed.txt')) + stub_request(:get, 'https://remote.test/valid_avatar').to_return(request_fixture('avatar.txt')) + stub_request(:get, 'https://remote.test/invalid_avatar').to_return(request_fixture('feed.txt')) end let(:account) do Fabricate(:account, - avatar_remote_url: 'https://remote/valid_avatar', - header_remote_url: 'https://remote/valid_avatar') + avatar_remote_url: 'https://remote.test/valid_avatar', + header_remote_url: 'https://remote.test/valid_avatar') end let!(:expectation) { account.dup } @@ -121,7 +121,7 @@ RSpec.describe Account, type: :model do context 'with invalid properties' do before do - account.avatar_remote_url = 'https://remote/invalid_avatar' + account.avatar_remote_url = 'https://remote.test/invalid_avatar' account.save_with_optional_media! end diff --git a/spec/models/status_pin_spec.rb b/spec/models/status_pin_spec.rb index 944baf639..6f0b2feb8 100644 --- a/spec/models/status_pin_spec.rb +++ b/spec/models/status_pin_spec.rb @@ -55,7 +55,7 @@ RSpec.describe StatusPin, type: :model do end it 'allows pins above the max for remote accounts' do - account = Fabricate(:account, domain: 'remote', username: 'bob', url: 'https://remote/') + account = Fabricate(:account, domain: 'remote.test', username: 'bob', url: 'https://remote.test/') status = [] (max_pins + 1).times do |i| diff --git a/spec/services/activitypub/process_account_service_spec.rb b/spec/services/activitypub/process_account_service_spec.rb index 15e1f4bb2..d67d72acb 100644 --- a/spec/services/activitypub/process_account_service_spec.rb +++ b/spec/services/activitypub/process_account_service_spec.rb @@ -6,9 +6,9 @@ RSpec.describe ActivityPub::ProcessAccountService do context 'property values' do let(:payload) do { - id: 'https://foo', + id: 'https://foo.test', type: 'Actor', - inbox: 'https://foo/inbox', + inbox: 'https://foo.test/inbox', attachment: [ { type: 'PropertyValue', name: 'Pronouns', value: 'They/them' }, { type: 'PropertyValue', name: 'Occupation', value: 'Unit test' }, -- cgit From dc786c0cf4467ade8db7d8b17e09f16923bfc1e8 Mon Sep 17 00:00:00 2001 From: Surinna Curtis Date: Wed, 2 May 2018 05:40:24 -0500 Subject: Support Actors/Statuses with multiple types (#7305) * Add equals_or_includes_any? helper in JsonLdHelper * Support arrays in JSON-LD type fields for actors/tags/objects. * Spec for resolving accounts with extension types * Style tweaks for codeclimate --- app/helpers/jsonld_helper.rb | 4 ++++ app/lib/activitypub/activity/create.rb | 11 +++++------ app/lib/activitypub/activity/update.rb | 5 +---- app/services/activitypub/fetch_remote_account_service.rb | 2 +- app/services/activitypub/fetch_remote_key_service.rb | 4 ++-- app/services/activitypub/fetch_remote_status_service.rb | 2 +- app/services/activitypub/process_account_service.rb | 5 +---- app/services/fetch_atom_service.rb | 4 ++-- app/services/resolve_account_service.rb | 2 +- app/services/resolve_url_service.rb | 5 ++--- spec/fixtures/requests/activitypub-actor-individual.txt | 9 +++++++++ spec/services/resolve_account_service_spec.rb | 14 ++++++++++++++ 12 files changed, 43 insertions(+), 24 deletions(-) create mode 100644 spec/fixtures/requests/activitypub-actor-individual.txt (limited to 'spec/services') diff --git a/app/helpers/jsonld_helper.rb b/app/helpers/jsonld_helper.rb index dfb8fcb8b..a3cfdadb8 100644 --- a/app/helpers/jsonld_helper.rb +++ b/app/helpers/jsonld_helper.rb @@ -5,6 +5,10 @@ module JsonLdHelper haystack.is_a?(Array) ? haystack.include?(needle) : haystack == needle end + def equals_or_includes_any?(haystack, needles) + needles.any? { |needle| equals_or_includes?(haystack, needle) } + end + def first_of_value(value) value.is_a?(Array) ? value.first : value end diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index 45c0e91cb..411286fa5 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -61,12 +61,11 @@ class ActivityPub::Activity::Create < ActivityPub::Activity return if @object['tag'].nil? as_array(@object['tag']).each do |tag| - case tag['type'] - when 'Hashtag' + if equals_or_includes?(tag['type'], 'Hashtag') process_hashtag tag, status - when 'Mention' + elsif equals_or_includes?(tag['type'], 'Mention') process_mention tag, status - when 'Emoji' + elsif equals_or_includes?(tag['type'], 'Emoji') process_emoji tag, status end end @@ -235,11 +234,11 @@ class ActivityPub::Activity::Create < ActivityPub::Activity end def supported_object_type? - SUPPORTED_TYPES.include?(@object['type']) + equals_or_includes_any?(@object['type'], SUPPORTED_TYPES) end def converted_object_type? - CONVERTED_TYPES.include?(@object['type']) + equals_or_includes_any?(@object['type'], CONVERTED_TYPES) end def skip_download? diff --git a/app/lib/activitypub/activity/update.rb b/app/lib/activitypub/activity/update.rb index 0134b4015..47e98e041 100644 --- a/app/lib/activitypub/activity/update.rb +++ b/app/lib/activitypub/activity/update.rb @@ -2,10 +2,7 @@ class ActivityPub::Activity::Update < ActivityPub::Activity def perform - case @object['type'] - when 'Person' - update_account - end + update_account if equals_or_includes?(@object['type'], 'Person') end private diff --git a/app/services/activitypub/fetch_remote_account_service.rb b/app/services/activitypub/fetch_remote_account_service.rb index 5024853ca..867e70876 100644 --- a/app/services/activitypub/fetch_remote_account_service.rb +++ b/app/services/activitypub/fetch_remote_account_service.rb @@ -56,6 +56,6 @@ class ActivityPub::FetchRemoteAccountService < BaseService end def expected_type? - SUPPORTED_TYPES.include?(@json['type']) + equals_or_includes_any?(@json['type'], SUPPORTED_TYPES) end end diff --git a/app/services/activitypub/fetch_remote_key_service.rb b/app/services/activitypub/fetch_remote_key_service.rb index 41837d462..505baccd4 100644 --- a/app/services/activitypub/fetch_remote_key_service.rb +++ b/app/services/activitypub/fetch_remote_key_service.rb @@ -43,7 +43,7 @@ class ActivityPub::FetchRemoteKeyService < BaseService end def person? - ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES.include?(@json['type']) + equals_or_includes_any?(@json['type'], ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES) end def public_key? @@ -55,6 +55,6 @@ class ActivityPub::FetchRemoteKeyService < BaseService end def confirmed_owner? - ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES.include?(@owner['type']) && value_or_id(@owner['publicKey']) == @json['id'] + equals_or_includes_any?(@owner['type'], ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES) && value_or_id(@owner['publicKey']) == @json['id'] end end diff --git a/app/services/activitypub/fetch_remote_status_service.rb b/app/services/activitypub/fetch_remote_status_service.rb index 503c175d8..930fbad1f 100644 --- a/app/services/activitypub/fetch_remote_status_service.rb +++ b/app/services/activitypub/fetch_remote_status_service.rb @@ -42,7 +42,7 @@ class ActivityPub::FetchRemoteStatusService < BaseService end def expected_type? - (ActivityPub::Activity::Create::SUPPORTED_TYPES + ActivityPub::Activity::Create::CONVERTED_TYPES).include? @json['type'] + equals_or_includes_any?(@json['type'], ActivityPub::Activity::Create::SUPPORTED_TYPES + ActivityPub::Activity::Create::CONVERTED_TYPES) end def needs_update(actor) diff --git a/app/services/activitypub/process_account_service.rb b/app/services/activitypub/process_account_service.rb index da32f9615..f67ebb443 100644 --- a/app/services/activitypub/process_account_service.rb +++ b/app/services/activitypub/process_account_service.rb @@ -201,10 +201,7 @@ class ActivityPub::ProcessAccountService < BaseService return if @json['tag'].blank? as_array(@json['tag']).each do |tag| - case tag['type'] - when 'Emoji' - process_emoji tag - end + process_emoji tag if equals_or_includes?(tag['type'], 'Emoji') end end diff --git a/app/services/fetch_atom_service.rb b/app/services/fetch_atom_service.rb index 0444baf74..550e75f33 100644 --- a/app/services/fetch_atom_service.rb +++ b/app/services/fetch_atom_service.rb @@ -42,7 +42,7 @@ class FetchAtomService < BaseService elsif ['application/activity+json', 'application/ld+json; profile="https://www.w3.org/ns/activitystreams"'].include?(response.mime_type) body = response.body_with_limit json = body_to_json(body) - if supported_context?(json) && ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES.include?(json['type']) && json['inbox'].present? + if supported_context?(json) && equals_or_includes_any?(json['type'], ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES) && json['inbox'].present? [json['id'], { prefetched_body: body, id: true }, :activitypub] elsif supported_context?(json) && expected_type?(json) [json['id'], { prefetched_body: body, id: true }, :activitypub] @@ -62,7 +62,7 @@ class FetchAtomService < BaseService end def expected_type?(json) - (ActivityPub::Activity::Create::SUPPORTED_TYPES + ActivityPub::Activity::Create::CONVERTED_TYPES).include? json['type'] + equals_or_includes_any?(json['type'], ActivityPub::Activity::Create::SUPPORTED_TYPES + ActivityPub::Activity::Create::CONVERTED_TYPES) end def process_html(response) diff --git a/app/services/resolve_account_service.rb b/app/services/resolve_account_service.rb index 8cba88f01..de8d1151d 100644 --- a/app/services/resolve_account_service.rb +++ b/app/services/resolve_account_service.rb @@ -189,7 +189,7 @@ class ResolveAccountService < BaseService return @actor_json if defined?(@actor_json) json = fetch_resource(actor_url, false) - @actor_json = supported_context?(json) && ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES.include?(json['type']) ? json : nil + @actor_json = supported_context?(json) && equals_or_includes_any?(json['type'], ActivityPub::FetchRemoteAccountService::SUPPORTED_TYPES) ? json : nil end def atom diff --git a/app/services/resolve_url_service.rb b/app/services/resolve_url_service.rb index c19b568cb..a068c1ed8 100644 --- a/app/services/resolve_url_service.rb +++ b/app/services/resolve_url_service.rb @@ -16,10 +16,9 @@ class ResolveURLService < BaseService private def process_url - case type - when 'Application', 'Group', 'Organization', 'Person', 'Service' + if equals_or_includes_any?(type, %w(Application Group Organization Person Service)) FetchRemoteAccountService.new.call(atom_url, body, protocol) - when 'Note', 'Article', 'Image', 'Video' + elsif equals_or_includes_any?(type, %w(Note Article Image Video)) FetchRemoteStatusService.new.call(atom_url, body, protocol) end end diff --git a/spec/fixtures/requests/activitypub-actor-individual.txt b/spec/fixtures/requests/activitypub-actor-individual.txt new file mode 100644 index 000000000..74411e544 --- /dev/null +++ b/spec/fixtures/requests/activitypub-actor-individual.txt @@ -0,0 +1,9 @@ +HTTP/1.1 200 OK +Cache-Control: max-age=0, private, must-revalidate +Content-Type: application/activity+json; charset=utf-8 +Link: ; rel="lrdd"; type="application/xrd+xml", ; rel="alternate"; type="application/atom+xml", ; rel="alternate"; type="application/activity+json" +Vary: Accept-Encoding +X-Content-Type-Options: nosniff +X-Xss-Protection: 1; mode=block + +{"@context":["https://www.w3.org/ns/activitystreams","https://w3id.org/security/v1",{"vcard": "http://www.w3.org/2006/vcard/ns#"},{"manuallyApprovesFollowers":"as:manuallyApprovesFollowers","sensitive":"as:sensitive","Hashtag":"as:Hashtag","ostatus":"http://ostatus.org#","atomUri":"ostatus:atomUri","inReplyToAtomUri":"ostatus:inReplyToAtomUri","conversation":"ostatus:conversation"}],"id":"https://ap.example.com/users/foo","type":["Person","vcard:individual"],"following":"https://ap.example.com/users/foo/following","followers":"https://ap.example.com/users/foo/followers","inbox":"https://ap.example.com/users/foo/inbox","outbox":"https://ap.example.com/users/foo/outbox","preferredUsername":"foo","vcard:fn":"foo","name":"","summary":"\u003cp\u003etest\u003c/p\u003e","url":"https://ap.example.com/@foo","manuallyApprovesFollowers":false,"publicKey":{"id":"https://ap.example.com/users/foo#main-key","owner":"https://ap.example.com/users/foo","publicKeyPem":"-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAu3L4vnpNLzVH31MeWI39\n4F0wKeJFsLDAsNXGeOu0QF2x+h1zLWZw/agqD2R3JPU9/kaDJGPIV2Sn5zLyUA9S\n6swCCMOtn7BBR9g9sucgXJmUFB0tACH2QSgHywMAybGfmSb3LsEMNKsGJ9VsvYoh\n8lDET6X4Pyw+ZJU0/OLo/41q9w+OrGtlsTm/PuPIeXnxa6BLqnDaxC+4IcjG/FiP\nahNCTINl/1F/TgSSDZ4Taf4U9XFEIFw8wmgploELozzIzKq+t8nhQYkgAkt64euW\npva3qL5KD1mTIZQEP+LZvh3s2WHrLi3fhbdRuwQ2c0KkJA2oSTFPDpqqbPGZ3Qvu\nHQIDAQAB\n-----END PUBLIC KEY-----\n"},"endpoints":{"sharedInbox":"https://ap.example.com/inbox"},"icon":{"type":"Image","url":"https://quitter.no/avatar/7477-300-20160211190340.png"}} \ No newline at end of file diff --git a/spec/services/resolve_account_service_spec.rb b/spec/services/resolve_account_service_spec.rb index 5f1b4467b..84dfe578a 100644 --- a/spec/services/resolve_account_service_spec.rb +++ b/spec/services/resolve_account_service_spec.rb @@ -105,6 +105,20 @@ RSpec.describe ResolveAccountService do expect(account.inbox_url).to eq 'https://ap.example.com/users/foo/inbox' end + context 'with multiple types' do + before do + stub_request(:get, "https://ap.example.com/users/foo").to_return(request_fixture('activitypub-actor-individual.txt')) + end + + it 'returns new remote account' do + account = subject.call('foo@ap.example.com') + + expect(account.activitypub?).to eq true + expect(account.domain).to eq 'ap.example.com' + expect(account.inbox_url).to eq 'https://ap.example.com/users/foo/inbox' + end + end + pending end -- cgit From cb5b5cb5f79bb2187d8124df91af4c8e1bfd7256 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Wed, 2 May 2018 18:58:48 +0200 Subject: Slightly reduce RAM usage (#7301) * No need to re-require sidekiq plugins, they are required via Gemfile * Add derailed_benchmarks tool, no need to require TTY gems in Gemfile * Replace ruby-oembed with FetchOEmbedService Reduce startup by 45382 allocated objects * Remove preloaded JSON-LD in favour of caching HTTP responses Reduce boot RAM by about 6 MiB * Fix tests * Fix test suite by stubbing out JSON-LD contexts --- Gemfile | 12 +- Gemfile.lock | 22 +- app/controllers/api/web/embeds_controller.rb | 11 +- .../settings/follower_domains_controller.rb | 2 - app/helpers/jsonld_helper.rb | 17 +- app/lib/provider_discovery.rb | 47 --- app/services/fan_out_on_write_service.rb | 2 - app/services/fetch_link_card_service.rb | 38 +- app/services/fetch_oembed_service.rb | 71 ++++ app/workers/scheduler/backup_cleanup_scheduler.rb | 1 - .../scheduler/doorkeeper_cleanup_scheduler.rb | 1 - app/workers/scheduler/email_scheduler.rb | 1 - app/workers/scheduler/feed_cleanup_scheduler.rb | 1 - app/workers/scheduler/ip_cleanup_scheduler.rb | 1 - app/workers/scheduler/media_cleanup_scheduler.rb | 1 - .../scheduler/subscriptions_cleanup_scheduler.rb | 2 - app/workers/scheduler/subscriptions_scheduler.rb | 3 - app/workers/scheduler/user_cleanup_scheduler.rb | 1 - app/workers/soft_block_domain_followers_worker.rb | 2 - config/initializers/json_ld.rb | 5 - config/initializers/oembed.rb | 4 - lib/json_ld/activitystreams.rb | 153 -------- lib/json_ld/identity.rb | 86 ----- lib/json_ld/security.rb | 50 --- lib/tasks/mastodon.rake | 2 + spec/fixtures/requests/json-ld.activitystreams.txt | 391 +++++++++++++++++++++ spec/fixtures/requests/json-ld.identity.txt | 100 ++++++ spec/fixtures/requests/json-ld.security.txt | 61 ++++ spec/lib/activitypub/linked_data_signature_spec.rb | 4 + spec/lib/provider_discovery_spec.rb | 118 ------- spec/rails_helper.rb | 14 + spec/services/account_search_service_spec.rb | 2 +- .../fetch_remote_account_service_spec.rb | 2 +- .../fetch_remote_status_service_spec.rb | 2 +- .../activitypub/process_account_service_spec.rb | 2 +- .../activitypub/process_collection_service_spec.rb | 2 +- spec/services/after_block_service_spec.rb | 2 +- spec/services/authorize_follow_service_spec.rb | 2 +- .../services/batched_remove_status_service_spec.rb | 2 +- .../block_domain_from_account_service_spec.rb | 2 +- spec/services/block_domain_service_spec.rb | 2 +- spec/services/block_service_spec.rb | 2 +- spec/services/bootstrap_timeline_service_spec.rb | 2 +- spec/services/fan_out_on_write_service_spec.rb | 2 +- spec/services/favourite_service_spec.rb | 2 +- spec/services/fetch_atom_service_spec.rb | 2 +- spec/services/fetch_link_card_service_spec.rb | 2 +- spec/services/fetch_oembed_service_spec.rb | 125 +++++++ spec/services/fetch_remote_account_service_spec.rb | 2 +- spec/services/fetch_remote_status_service_spec.rb | 2 +- spec/services/follow_service_spec.rb | 2 +- spec/services/mute_service_spec.rb | 2 +- spec/services/notify_service_spec.rb | 2 +- spec/services/post_status_service_spec.rb | 2 +- spec/services/precompute_feed_service_spec.rb | 2 +- spec/services/process_feed_service_spec.rb | 2 +- spec/services/process_interaction_service_spec.rb | 2 +- spec/services/process_mentions_service_spec.rb | 2 +- .../pubsubhubbub/subscribe_service_spec.rb | 2 +- .../pubsubhubbub/unsubscribe_service_spec.rb | 2 +- spec/services/reblog_service_spec.rb | 2 +- spec/services/reject_follow_service_spec.rb | 2 +- spec/services/remove_status_service_spec.rb | 2 +- spec/services/report_service_spec.rb | 2 +- spec/services/resolve_account_service_spec.rb | 2 +- spec/services/resolve_url_service_spec.rb | 2 +- spec/services/search_service_spec.rb | 2 +- spec/services/send_interaction_service_spec.rb | 2 +- spec/services/subscribe_service_spec.rb | 2 +- spec/services/suspend_account_service_spec.rb | 2 +- spec/services/unblock_domain_service_spec.rb | 2 +- spec/services/unblock_service_spec.rb | 2 +- spec/services/unfollow_service_spec.rb | 2 +- spec/services/unmute_service_spec.rb | 2 +- spec/services/unsubscribe_service_spec.rb | 2 +- .../services/update_remote_profile_service_spec.rb | 2 +- spec/spec_helper.rb | 12 +- 77 files changed, 881 insertions(+), 568 deletions(-) delete mode 100644 app/lib/provider_discovery.rb create mode 100644 app/services/fetch_oembed_service.rb delete mode 100644 config/initializers/json_ld.rb delete mode 100644 config/initializers/oembed.rb delete mode 100644 lib/json_ld/activitystreams.rb delete mode 100644 lib/json_ld/identity.rb delete mode 100644 lib/json_ld/security.rb create mode 100644 spec/fixtures/requests/json-ld.activitystreams.txt create mode 100644 spec/fixtures/requests/json-ld.identity.txt create mode 100644 spec/fixtures/requests/json-ld.security.txt delete mode 100644 spec/lib/provider_discovery_spec.rb create mode 100644 spec/services/fetch_oembed_service_spec.rb (limited to 'spec/services') diff --git a/Gemfile b/Gemfile index a33748568..f1665ce95 100644 --- a/Gemfile +++ b/Gemfile @@ -54,7 +54,7 @@ gem 'httplog', '~> 1.0' gem 'idn-ruby', require: 'idn' gem 'kaminari', '~> 1.1' gem 'link_header', '~> 0.0' -gem 'mime-types', '~> 3.1' +gem 'mime-types', '~> 3.1', require: 'mime/types/columnar' gem 'nokogiri', '~> 1.8' gem 'nsa', '~> 0.2' gem 'oj', '~> 3.5' @@ -70,7 +70,6 @@ gem 'rails-settings-cached', '~> 0.6' gem 'redis', '~> 4.0', require: ['redis', 'redis/connection/hiredis'] gem 'mario-redis-lock', '~> 1.2', require: 'redis_lock' gem 'rqrcode', '~> 0.10' -gem 'ruby-oembed', '~> 0.12', require: 'oembed' gem 'ruby-progressbar', '~> 1.4' gem 'sanitize', '~> 4.6' gem 'sidekiq', '~> 5.1' @@ -82,14 +81,14 @@ gem 'simple_form', '~> 4.0' gem 'sprockets-rails', '~> 3.2', require: 'sprockets/railtie' gem 'stoplight', '~> 2.1.3' gem 'strong_migrations', '~> 0.2' -gem 'tty-command', '~> 0.8' -gem 'tty-prompt', '~> 0.16' +gem 'tty-command', '~> 0.8', require: false +gem 'tty-prompt', '~> 0.16', require: false gem 'twitter-text', '~> 1.14' gem 'tzinfo-data', '~> 1.2018' gem 'webpacker', '~> 3.4' gem 'webpush' -gem 'json-ld-preloaded', '~> 2.2' +gem 'json-ld', '~> 2.2' gem 'rdf-normalize', '~> 0.3' group :development, :test do @@ -135,6 +134,9 @@ group :development do gem 'capistrano-rails', '~> 1.3' gem 'capistrano-rbenv', '~> 2.1' gem 'capistrano-yarn', '~> 2.0' + + gem 'derailed_benchmarks' + gem 'stackprof' end group :production do diff --git a/Gemfile.lock b/Gemfile.lock index d96165dcf..94ab0b7ca 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -75,6 +75,7 @@ GEM aws-sigv4 (~> 1.0) aws-sigv4 (1.0.2) bcrypt (3.1.11) + benchmark-ips (2.7.2) better_errors (2.4.0) coderay (>= 1.0.0) erubi (>= 1.0.0) @@ -138,6 +139,14 @@ GEM css_parser (1.6.0) addressable debug_inspector (0.0.3) + derailed_benchmarks (1.3.4) + benchmark-ips (~> 2) + get_process_mem (~> 0) + heapy (~> 0) + memory_profiler (~> 0) + rack (>= 1) + rake (> 10, < 13) + thor (~> 0.19) devise (4.4.3) bcrypt (~> 3.0) orm_adapter (~> 0.1) @@ -206,6 +215,7 @@ GEM fuubar (2.3.1) rspec-core (~> 3.0) ruby-progressbar (~> 1.4) + get_process_mem (0.2.1) globalid (0.4.1) activesupport (>= 4.2.0) goldfinger (2.1.0) @@ -226,6 +236,7 @@ GEM concurrent-ruby (~> 1.0) hashdiff (0.3.7) hashie (3.5.7) + heapy (0.1.3) highline (1.7.10) hiredis (0.6.1) hitimes (1.2.6) @@ -264,10 +275,6 @@ GEM json-ld (2.2.1) multi_json (~> 1.12) rdf (>= 2.2.8, < 4.0) - json-ld-preloaded (2.2.3) - json-ld (>= 2.2, < 4.0) - multi_json (~> 1.12) - rdf (>= 2.2, < 4.0) jsonapi-renderer (0.2.0) jwt (2.1.0) kaminari (1.1.1) @@ -502,7 +509,6 @@ GEM rainbow (>= 2.2.2, < 4.0) ruby-progressbar (~> 1.7) unicode-display_width (~> 1.0, >= 1.0.1) - ruby-oembed (0.12.0) ruby-progressbar (1.9.0) ruby-saml (1.7.2) nokogiri (>= 1.5.10) @@ -557,6 +563,7 @@ GEM sshkit (1.16.0) net-scp (>= 1.1.2) net-ssh (>= 2.8.0) + stackprof (0.2.11) statsd-ruby (1.2.1) stoplight (2.1.3) streamio-ffmpeg (3.0.2) @@ -645,6 +652,7 @@ DEPENDENCIES chewy (~> 5.0) cld3 (~> 3.2.0) climate_control (~> 0.2) + derailed_benchmarks devise (~> 4.4) devise-two-factor (~> 3.0) devise_pam_authenticatable2 (~> 9.1) @@ -668,7 +676,7 @@ DEPENDENCIES i18n-tasks (~> 0.9) idn-ruby iso-639 - json-ld-preloaded (~> 2.2) + json-ld (~> 2.2) kaminari (~> 1.1) letter_opener (~> 1.4) letter_opener_web (~> 1.3) @@ -714,7 +722,6 @@ DEPENDENCIES rspec-retry (~> 0.5) rspec-sidekiq (~> 3.0) rubocop (~> 0.55) - ruby-oembed (~> 0.12) ruby-progressbar (~> 1.4) sanitize (~> 4.6) scss_lint (~> 0.57) @@ -726,6 +733,7 @@ DEPENDENCIES simple_form (~> 4.0) simplecov (~> 0.16) sprockets-rails (~> 3.2) + stackprof stoplight (~> 2.1.3) streamio-ffmpeg (~> 3.0) strong_migrations (~> 0.2) diff --git a/app/controllers/api/web/embeds_controller.rb b/app/controllers/api/web/embeds_controller.rb index f2fe74b17..987290a14 100644 --- a/app/controllers/api/web/embeds_controller.rb +++ b/app/controllers/api/web/embeds_controller.rb @@ -9,9 +9,12 @@ class Api::Web::EmbedsController < Api::Web::BaseController status = StatusFinder.new(params[:url]).status render json: status, serializer: OEmbedSerializer, width: 400 rescue ActiveRecord::RecordNotFound - oembed = OEmbed::Providers.get(params[:url]) - render json: Oj.dump(oembed.fields) - rescue OEmbed::NotFound - render json: {}, status: :not_found + oembed = FetchOEmbedService.new.call(params[:url]) + + if oembed + render json: oembed + else + render json: {}, status: :not_found + end end end diff --git a/app/controllers/settings/follower_domains_controller.rb b/app/controllers/settings/follower_domains_controller.rb index 213d9e96d..91b521e7f 100644 --- a/app/controllers/settings/follower_domains_controller.rb +++ b/app/controllers/settings/follower_domains_controller.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require 'sidekiq-bulk' - class Settings::FollowerDomainsController < ApplicationController layout 'admin' diff --git a/app/helpers/jsonld_helper.rb b/app/helpers/jsonld_helper.rb index a3cfdadb8..e9056166c 100644 --- a/app/helpers/jsonld_helper.rb +++ b/app/helpers/jsonld_helper.rb @@ -48,7 +48,7 @@ module JsonLdHelper end def canonicalize(json) - graph = RDF::Graph.new << JSON::LD::API.toRdf(json) + graph = RDF::Graph.new << JSON::LD::API.toRdf(json, documentLoader: method(:load_jsonld_context)) graph.dump(:normalize) end @@ -90,4 +90,19 @@ module JsonLdHelper request.add_headers('Accept' => 'application/activity+json, application/ld+json') request end + + def load_jsonld_context(url, _options = {}, &_block) + json = Rails.cache.fetch("jsonld:context:#{url}", expires_in: 30.days, raw: true) do + request = Request.new(:get, url) + request.add_headers('Accept' => 'application/ld+json') + + request.perform do |res| + raise JSON::LD::JsonLdError::LoadingDocumentFailed unless res.code == 200 && res.mime_type == 'application/ld+json' + res.body_with_limit + end + end + + doc = JSON::LD::API::RemoteDocument.new(url, json) + block_given? ? yield(doc) : doc + end end diff --git a/app/lib/provider_discovery.rb b/app/lib/provider_discovery.rb deleted file mode 100644 index 3bec7211b..000000000 --- a/app/lib/provider_discovery.rb +++ /dev/null @@ -1,47 +0,0 @@ -# frozen_string_literal: true - -class ProviderDiscovery < OEmbed::ProviderDiscovery - class << self - def get(url, **options) - provider = discover_provider(url, options) - - options.delete(:html) - - provider.get(url, options) - end - - def discover_provider(url, **options) - format = options[:format] - - html = if options[:html] - Nokogiri::HTML(options[:html]) - else - Request.new(:get, url).perform do |res| - raise OEmbed::NotFound, url if res.code != 200 || res.mime_type != 'text/html' - Nokogiri::HTML(res.body_with_limit) - end - end - - if format.nil? || format == :json - provider_endpoint ||= html.at_xpath('//link[@type="application/json+oembed"]')&.attribute('href')&.value - format ||= :json if provider_endpoint - end - - if format.nil? || format == :xml - provider_endpoint ||= html.at_xpath('//link[@type="text/xml+oembed"]')&.attribute('href')&.value - format ||= :xml if provider_endpoint - end - - raise OEmbed::NotFound, url if provider_endpoint.nil? - begin - provider_endpoint = Addressable::URI.parse(provider_endpoint) - provider_endpoint.query = nil - provider_endpoint = provider_endpoint.to_s - rescue Addressable::URI::InvalidURIError - raise OEmbed::NotFound, url - end - - OEmbed::Provider.new(provider_endpoint, format) - end - end -end diff --git a/app/services/fan_out_on_write_service.rb b/app/services/fan_out_on_write_service.rb index 0f77556dc..510b80c82 100644 --- a/app/services/fan_out_on_write_service.rb +++ b/app/services/fan_out_on_write_service.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require 'sidekiq-bulk' - class FanOutOnWriteService < BaseService # Push a status into home and mentions feeds # @param [Status] status diff --git a/app/services/fetch_link_card_service.rb b/app/services/fetch_link_card_service.rb index d5920a417..77d4aa538 100644 --- a/app/services/fetch_link_card_service.rb +++ b/app/services/fetch_link_card_service.rb @@ -85,42 +85,40 @@ class FetchLinkCardService < BaseService end def attempt_oembed - embed = OEmbed::Providers.get(@url, html: @html) + embed = FetchOEmbedService.new.call(@url, html: @html) - return false unless embed.respond_to?(:type) + return false if embed.nil? - @card.type = embed.type - @card.title = embed.respond_to?(:title) ? embed.title : '' - @card.author_name = embed.respond_to?(:author_name) ? embed.author_name : '' - @card.author_url = embed.respond_to?(:author_url) ? embed.author_url : '' - @card.provider_name = embed.respond_to?(:provider_name) ? embed.provider_name : '' - @card.provider_url = embed.respond_to?(:provider_url) ? embed.provider_url : '' + @card.type = embed[:type] + @card.title = embed[:title] || '' + @card.author_name = embed[:author_name] || '' + @card.author_url = embed[:author_url] || '' + @card.provider_name = embed[:provider_name] || '' + @card.provider_url = embed[:provider_url] || '' @card.width = 0 @card.height = 0 case @card.type when 'link' - @card.image_remote_url = embed.thumbnail_url if embed.respond_to?(:thumbnail_url) + @card.image_remote_url = embed[:thumbnail_url] if embed[:thumbnail_url].present? when 'photo' - return false unless embed.respond_to?(:url) + return false if embed[:url].blank? - @card.embed_url = embed.url - @card.image_remote_url = embed.url - @card.width = embed.width.presence || 0 - @card.height = embed.height.presence || 0 + @card.embed_url = embed[:url] + @card.image_remote_url = embed[:url] + @card.width = embed[:width].presence || 0 + @card.height = embed[:height].presence || 0 when 'video' - @card.width = embed.width.presence || 0 - @card.height = embed.height.presence || 0 - @card.html = Formatter.instance.sanitize(embed.html, Sanitize::Config::MASTODON_OEMBED) - @card.image_remote_url = embed.thumbnail_url if embed.respond_to?(:thumbnail_url) + @card.width = embed[:width].presence || 0 + @card.height = embed[:height].presence || 0 + @card.html = Formatter.instance.sanitize(embed[:html], Sanitize::Config::MASTODON_OEMBED) + @card.image_remote_url = embed[:thumbnail_url] if embed[:thumbnail_url].present? when 'rich' # Most providers rely on