Московя начинаетъ только въ XVI ст. привлекать внимане иностранцевъ.

From 007ab330e6ffb1e07995d4e306473d457043e2eb Mon Sep 17 00:00:00 2001 From: nullkal Date: Sun, 9 Jul 2017 05:44:31 +0900 Subject: Use charlock_holmes instead of nkf at FetchLinkCardService (#4080) * Specs for language detection * Use CharlockHolmes instead of NKF * Correct mistakes * Correct style * Set hint_enc instead of falling back and strip_tags * Improve specs * Add dependencies --- spec/fixtures/requests/koi8-r.txt | 20 +++++++++++++++++++ spec/fixtures/requests/sjis.txt | 4 ++-- spec/fixtures/requests/sjis_with_wrong_charset.txt | 20 +++++++++++++++++++ spec/services/fetch_link_card_service_spec.rb | 23 ++++++++++++++++++++++ 4 files changed, 65 insertions(+), 2 deletions(-) create mode 100644 spec/fixtures/requests/koi8-r.txt create mode 100644 spec/fixtures/requests/sjis_with_wrong_charset.txt (limited to 'spec') diff --git a/spec/fixtures/requests/koi8-r.txt b/spec/fixtures/requests/koi8-r.txt new file mode 100644 index 000000000..d4242af01 --- /dev/null +++ b/spec/fixtures/requests/koi8-r.txt @@ -0,0 +1,20 @@ +HTTP/1.1 200 OK +Server: nginx/1.11.10 +Date: Tue, 04 Jul 2017 16:43:39 GMT +Content-Type: text/html +Content-Length: 273 +Connection: keep-alive +Last-Modified: Tue, 04 Jul 2017 16:41:34 GMT +Accept-Ranges: bytes + + + + + + Московя начинаетъ только въ XVI ст. привлекать внимане иностранцевъ. + + +

Московя начинаетъ только въ XVI ст. привлекать внимане иностранцевъ.
+

+ + diff --git a/spec/fixtures/requests/sjis.txt b/spec/fixtures/requests/sjis.txt index 9041aa25d..faf18d35c 100644 --- a/spec/fixtures/requests/sjis.txt +++ b/spec/fixtures/requests/sjis.txt @@ -11,10 +11,10 @@ Accept-Ranges: bytes - JSIS┌л┐y│[┐W + SJIS┌л┐y│[┐W -

SJIS┌л┐y│[┐W
+

▌└┌Ю⌠╞■N┌э┌╣┌д┌╒┌М┌Д┌И▀L■O░l┌а┌д┌Ю┌л┌л▌·┌е┌╣┌═┌Х┌е┌╥│B┌Ю┌╣▌·┼т┌и┬с√║▌р┌м░Ё┌╣┌╜┌г┌Я┌х■╜┴О┌э┌╧┌╬┌э┌е┌╙░\┌╣▐Ц┌╟┌╙┌╒┌Г┌а┌╣┌А┌И┌╫┌и┌м▌Q█l▀A┌И┌╫┌╒┌╬┌╘┌Г│A▐╜┌╣┌и┌Ю┌Б┌а┌═┌а┌э┌╣┌х┌╫│B▀Ю┌╘┌Г┌╒┌╓┌х┌╒┌л┌м┌г┌╓┌Ю▀Ц▄▌┌П┌е┌╚┌И┌╬┌╞┌╫┌╫┌╜┌╫│B┌╞┌а┌╣┌д┴╙⌠c┌Ё┌Я┌и■╫█R█K▐╜┌╣▓╔┌и┴]┌╗┌е┌╣┌Е▀Ю≈м┌╠┌╓┌╣┌╫▄═≈м┌═┌х┌╫┌╘▌w░}┌╙┌ф┌╒┌╓┌╗▐o⌠Э┌Х┌х┌╜┌╬┌К┌х┌═┌Х┌д│A┌╩┌л░л┌м▌└┌╘▀Ю≈м┴A┌П⌠{┌Г┌╘┌Г│A▀v▄╢┌Ё┌Я┌л┌Ю┌л┌П┌╙┌╫┌л┌╒┌б┌╙┌╣┌╘┌И┌и┌╡┼С√]┌ф▄Э┌╒┌н┌╩┌Йman┌и┌╡√╣▐┌┌ж▌Q┌Х┌Ф┌╓┌и⌠╞▌·┌и┌╡┴┴░Ю┌╙┌╣┌е┌х┌Г┌л┌е│A▒╫∙╙┌Ю┌╣∙\≈═┌и∙о┌а┌╫┌д┌╜┌Й┌е┌╥▌√┌е█l┌╕┌╫┌╫│B┌╣┌╘┌Ю≈А┌╕┌н┌╡┌╙┌╫┌╙┌ф┌г┌э┌Г┌Ю┌л┌Ю▌ю█ш┌ч┌Б┌щ┌ф┌═┌Х┌е┌╥┌д│A┌╠┌л▌╘∙╙┌е┌м░\┌╣┌Я┌д┌ф┌╣┌д░╒┼т┌и∙ю┌в┌л┌и█s┌╘┌х┌╘┌а┌х│B

diff --git a/spec/fixtures/requests/sjis_with_wrong_charset.txt b/spec/fixtures/requests/sjis_with_wrong_charset.txt new file mode 100644 index 000000000..456750c6b --- /dev/null +++ b/spec/fixtures/requests/sjis_with_wrong_charset.txt @@ -0,0 +1,20 @@ +HTTP/1.1 200 OK +Server: nginx/1.11.10 +Date: Tue, 04 Jul 2017 16:43:39 GMT +Content-Type: text/html; charset=utf-8 +Content-Length: 273 +Connection: keep-alive +Last-Modified: Tue, 04 Jul 2017 16:41:34 GMT +Accept-Ranges: bytes + + + + + + SJIS┌л┐y│[┐W + + +

+ + diff --git a/spec/services/fetch_link_card_service_spec.rb b/spec/services/fetch_link_card_service_spec.rb index 7d7f8e748..698eb0324 100644 --- a/spec/services/fetch_link_card_service_spec.rb +++ b/spec/services/fetch_link_card_service_spec.rb @@ -8,6 +8,10 @@ RSpec.describe FetchLinkCardService do stub_request(:get, 'http://example.xn--fiqs8s/').to_return(request_fixture('idn.txt')) stub_request(:head, 'http://example.com/sjis').to_return(status: 200, headers: { 'Content-Type' => 'text/html' }) stub_request(:get, 'http://example.com/sjis').to_return(request_fixture('sjis.txt')) + stub_request(:head, 'http://example.com/sjis_with_wrong_charset').to_return(status: 200, headers: { 'Content-Type' => 'text/html' }) + stub_request(:get, 'http://example.com/sjis_with_wrong_charset').to_return(request_fixture('sjis_with_wrong_charset.txt')) + stub_request(:head, 'http://example.com/koi8-r').to_return(status: 200, headers: { 'Content-Type' => 'text/html' }) + stub_request(:get, 'http://example.com/koi8-r').to_return(request_fixture('koi8-r.txt')) stub_request(:head, 'https://github.com/qbi/WannaCry').to_return(status: 404) subject.call(status) @@ -27,6 +31,25 @@ RSpec.describe FetchLinkCardService do it 'works with SJIS' do expect(a_request(:get, 'http://example.com/sjis')).to have_been_made.at_least_once + expect(status.preview_card.title).to eq("SJISЦ│╝Ц┐ Ц┐╪Ц┌╦") + end + end + + context do + let(:status) { Fabricate(:status, text: 'Check out http://example.com/sjis_with_wrong_charset') } + + it 'works with SJIS even with wrong charset header' do + expect(a_request(:get, 'http://example.com/sjis_with_wrong_charset')).to have_been_made.at_least_once + expect(status.preview_card.title).to eq("SJISЦ│╝Ц┐ Ц┐╪Ц┌╦") + end + end + + context do + let(:status) { Fabricate(:status, text: 'Check out http://example.com/koi8-r') } + + it 'works with koi8-r' do + expect(a_request(:get, 'http://example.com/koi8-r')).to have_been_made.at_least_once + expect(status.preview_card.title).to eq("п°п╬я│п╨п╬п╡я▐ п╫п╟я┤п╦п╫п╟п╣я┌я┼ я┌п╬п╩я▄п╨п╬ п╡я┼ XVI я│я┌. п©я─п╦п╡п╩п╣п╨п╟я┌я▄ п╡п╫п╦п╪п╟п╫п╣ п╦п╫п╬я│я┌я─п╟п╫я├п╣п╡я┼.") end end end -- cgit