diff options
| author | ThibG <thib@sitedethib.com> | 2018-12-17 19:19:45 +0100 |
|---|---|---|
| committer | Eugen Rochko <eugen@zeonfederated.com> | 2018-12-17 19:19:45 +0100 |
| commit | e709b8da0d685d3cc48d430a9761896094f67d72 (patch) | |
| tree | 8d60d548237c7478bb974e2813288fdda11a8862 | |
| parent | 4ede51743e5b9121a49e9131f91cf012fab410f8 (diff) | |
Ignore low-confidence CharlockHolmes guesses when parsing link cards (#9510)
* Add failing test for windows-1251 link cards * Ignore low-confidence CharlockHolmes guesses Fixes #9466 * Fix no method error when charlock holmes cannot detect charset
| -rw-r--r-- | app/services/fetch_link_card_service.rb | 3 | ||||
| -rw-r--r-- | spec/fixtures/requests/windows-1251.txt | 17 | ||||
| -rw-r--r-- | spec/services/fetch_link_card_service_spec.rb | 11 |
3 files changed, 30 insertions, 1 deletions
diff --git a/app/services/fetch_link_card_service.rb b/app/services/fetch_link_card_service.rb index 38c578de2..7979c312e 100644 --- a/app/services/fetch_link_card_service.rb +++ b/app/services/fetch_link_card_service.rb @@ -137,7 +137,8 @@ class FetchLinkCardService < BaseService detector.strip_tags = true guess = detector.detect(@html, @html_charset) - page = Nokogiri::HTML(@html, nil, guess&.fetch(:encoding, nil)) + encoding = guess&.fetch(:confidence, 0).to_i > 60 ? guess&.fetch(:encoding, nil) : nil + page = Nokogiri::HTML(@html, nil, encoding) player_url = meta_property(page, 'twitter:player') if player_url && !bad_url?(Addressable::URI.parse(player_url)) diff --git a/spec/fixtures/requests/windows-1251.txt b/spec/fixtures/requests/windows-1251.txt new file mode 100644 index 000000000..f573e28b2 --- /dev/null +++ b/spec/fixtures/requests/windows-1251.txt @@ -0,0 +1,17 @@ +HTTP/1.1 200 OK +server: nginx +date: Wed, 12 Dec 2018 13:14:03 GMT +content-type: text/html +content-length: 190 +accept-ranges: bytes + +<!DOCTYPE html> +<html> +<head> + <meta http-equiv="Content-Type" content="text/html; charset=windows-1251" /> + <title> </title> +</head> +<body> + <p> </p> +</body> +</html> diff --git a/spec/services/fetch_link_card_service_spec.rb b/spec/services/fetch_link_card_service_spec.rb index 88c5339db..50c60aafd 100644 --- a/spec/services/fetch_link_card_service_spec.rb +++ b/spec/services/fetch_link_card_service_spec.rb @@ -17,6 +17,8 @@ RSpec.describe FetchLinkCardService, type: :service do stub_request(:head, 'https://github.com/qbi/WannaCry').to_return(status: 404) stub_request(:head, 'http://example.com/test-').to_return(status: 200, headers: { 'Content-Type' => 'text/html' }) stub_request(:get, 'http://example.com/test-').to_return(request_fixture('idn.txt')) + stub_request(:head, 'http://example.com/windows-1251').to_return(status: 200, headers: { 'Content-Type' => 'text/html' }) + stub_request(:get, 'http://example.com/windows-1251').to_return(request_fixture('windows-1251.txt')) subject.call(status) end @@ -58,6 +60,15 @@ RSpec.describe FetchLinkCardService, type: :service do end context do + let(:status) { Fabricate(:status, text: 'Check out http://example.com/windows-1251') } + + it 'works with windows-1251' do + expect(a_request(:get, 'http://example.com/windows-1251')).to have_been_made.at_least_once + expect(status.preview_cards.first.title).to eq('сэмпл текст') + end + end + + context do let(:status) { Fabricate(:status, text: 'テストhttp://example.com/日本語') } it 'works with Japanese path string' do |