11 files changed, 78 insertions, 4 deletions
diff --git a/.travis.yml b/.travis.yml
index 4bb332666..4d4dc0893 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -32,6 +32,7 @@ addons:
     - g++-6
     - libprotobuf-dev
     - protobuf-compiler
+    - libicu-dev
 
 rvm:
   - 2.3.4
diff --git a/Aptfile b/Aptfile
index 0456343ef..3af0956e3 100644
--- a/Aptfile
+++ b/Aptfile
@@ -3,3 +3,4 @@ libprotobuf-dev
 ffmpeg
 libxdamage1
 libxfixes3
+libicu-dev
diff --git a/Dockerfile b/Dockerfile
index 7033cddd4..97a691393 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -25,6 +25,7 @@ RUN echo "@edge https://nl.alpinelinux.org/alpine/edge/main" >> /etc/apk/reposit
     ffmpeg \
     file \
     git \
+    icu-dev \
     imagemagick@edge \
     libpq \
     libxml2 \
diff --git a/Gemfile b/Gemfile
index 95c74eef9..b52685cba 100644
--- a/Gemfile
+++ b/Gemfile
@@ -22,6 +22,7 @@ gem 'active_model_serializers', '~> 0.10'
 gem 'addressable', '~> 2.5'
 gem 'bootsnap'
 gem 'browser'
+gem 'charlock_holmes', '~> 0.7.3'
 gem 'cld3', '~> 3.1'
 gem 'devise', '~> 4.2'
 gem 'devise-two-factor', '~> 3.0'
diff --git a/Gemfile.lock b/Gemfile.lock
index 71f83f736..de0d6a107 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -106,6 +106,7 @@ GEM
       rack (>= 1.0.0)
       rack-test (>= 0.5.4)
       xpath (~> 2.0)
+    charlock_holmes (0.7.3)
     case_transform (0.2)
       activesupport
     chunky_png (1.3.8)
@@ -501,6 +502,7 @@ DEPENDENCIES
   capistrano-rbenv (~> 2.1)
   capistrano-yarn (~> 2.0)
   capybara (~> 2.14)
+  charlock_holmes (~> 0.7.3)
   cld3 (~> 3.1)
   climate_control (~> 0.2)
   devise (~> 4.2)
diff --git a/Vagrantfile b/Vagrantfile
index 1f56fcfb3..cbe6623b3 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -37,6 +37,7 @@ sudo apt-get install \
   yarn \
   libprotobuf-dev \
   libreadline-dev \
+  libicu-dev \
   -y
 
 # Install rvm
diff --git a/app/services/fetch_link_card_service.rb b/app/services/fetch_link_card_service.rb
index 8ddaa2bf4..6ef3abb66 100644
--- a/app/services/fetch_link_card_service.rb
+++ b/app/services/fetch_link_card_service.rb
@@ -1,5 +1,4 @@
 # frozen_string_literal: true
-require 'nkf'
 
 class FetchLinkCardService < BaseService
   include HttpHelper
@@ -86,7 +85,12 @@ class FetchLinkCardService < BaseService
     return if response.code != 200 || response.mime_type != 'text/html'
 
     html = response.to_s
-    page = Nokogiri::HTML(html, nil, NKF.guess(html).to_s)
+
+    detector = CharlockHolmes::EncodingDetector.new
+    detector.strip_tags = true
+
+    guess = detector.detect(html, response.charset)
+    page = Nokogiri::HTML(html, nil, guess&.fetch(:encoding))
 
     card.type             = :link
     card.title            = meta_property(page, 'og:title') || page.at_xpath('//title')&.content
diff --git a/spec/fixtures/requests/koi8-r.txt b/spec/fixtures/requests/koi8-r.txt
new file mode 100644
index 000000000..d4242af01
--- /dev/null
+++ b/spec/fixtures/requests/koi8-r.txt
@@ -0,0 +1,20 @@
+HTTP/1.1 200 OK
+Server: nginx/1.11.10
+Date: Tue, 04 Jul 2017 16:43:39 GMT
+Content-Type: text/html
+Content-Length: 273
+Connection: keep-alive
+Last-Modified: Tue, 04 Jul 2017 16:41:34 GMT
+Accept-Ranges: bytes
+
+<HTML>
+<HEAD>
+  <META NAME="GENERATOR" CONTENT="Adobe PageMill 3.0J Mac">
+  <META HTTP-EQUIV="Content-Type" CONTENT="text/html;CHARSET=koi8-r">
+  <TITLE>Московя начинаетъ только въ XVI ст. привлекать внимане иностранцевъ.</TITLE>
+</HEAD>
+<BODY>
+<P><CENTER><B><FONT SIZE="+2">Московя начинаетъ только въ XVI ст. привлекать внимане иностранцевъ.</FONT></B><BR>
+<HR><BR>
+</BODY>
+</HTML>
diff --git a/spec/fixtures/requests/sjis.txt b/spec/fixtures/requests/sjis.txt
index 9041aa25d..faf18d35c 100644
--- a/spec/fixtures/requests/sjis.txt
+++ b/spec/fixtures/requests/sjis.txt
@@ -11,10 +11,10 @@ Accept-Ranges: bytes
 <HEAD>
   <META NAME="GENERATOR" CONTENT="Adobe PageMill 3.0J Mac">
   <META HTTP-EQUIV="Content-Type" CONTENT="text/html;CHARSET=x-sjis">
-  <TITLE>JSIS┌л┐y│[┐W</TITLE>
+  <TITLE>SJIS┌л┐y│[┐W</TITLE>
 </HEAD>
 <BODY>
-<P><CENTER><B><FONT SIZE="+2">SJIS┌л┐y│[┐W</FONT></B><BR>
+<P><CENTER><B><FONT SIZE="+2">▌└┌Ю⌠╞■N┌э┌╣┌д┌╒┌М┌Д┌И▀L■O░l┌а┌д┌Ю┌л┌л▌·┌е┌╣┌═┌Х┌е┌Ї│B┌Ю┌╣▌·┼т┌и┬с√║▌р┌м░Ё┌╣┌ґ┌г┌Я┌х■ґ┴О┌э┌╧┌Ў┌э┌е┌╙░\┌╣▐Ц┌╟┌╙┌╒┌Г┌а┌╣┌А┌И┌Ґ┌и┌м▌Q█l▀A┌И┌Ґ┌╒┌Ў┌╘┌Г│A▐ґ┌╣┌и┌Ю┌Б┌а┌═┌а┌э┌╣┌х┌Ґ│B▀Ю┌╘┌Г┌╒┌є┌х┌╒┌л┌м┌г┌є┌Ю▀Ц▄▌┌П┌е┌╚┌И┌Ў┌╞┌Ґ┌Ґ┌ґ┌Ґ│B┌╞┌а┌╣┌д┴╙⌠c┌Ё┌Я┌и■Ґ█R█K▐ґ┌╣▓╔┌и┴]┌╗┌е┌╣┌Е▀Ю≈м┌╠┌є┌╣┌Ґ▄═≈м┌═┌х┌Ґ┌╘▌w░}┌╙┌ф┌╒┌є┌╗▐o⌠Э┌Х┌х┌ґ┌Ў┌К┌х┌═┌Х┌д│A┌╩┌л░л┌м▌└┌╘▀Ю≈м┴A┌П⌠{┌Г┌╘┌Г│A▀v▄Є┌Ё┌Я┌л┌Ю┌л┌П┌╙┌Ґ┌л┌╒┌б┌╙┌╣┌╘┌И┌и┌╡┼С√]┌ф▄Э┌╒┌н┌╩┌Йman┌и┌╡√╣▐┌┌ж▌Q┌Х┌Ф┌є┌и⌠╞▌·┌и┌╡┴┴░Ю┌╙┌╣┌е┌х┌Г┌л┌е│A▒Ґ∙╙┌Ю┌╣∙\≈═┌и∙о┌а┌Ґ┌д┌ґ┌Й┌е┌Ї▌√┌е█l┌і┌Ґ┌Ґ│B┌╣┌╘┌Ю≈А┌і┌н┌╡┌╙┌Ґ┌╙┌ф┌г┌э┌Г┌Ю┌л┌Ю▌ю█ш┌ч┌Б┌щ┌ф┌═┌Х┌е┌Ї┌д│A┌╠┌л▌╘∙╙┌е┌м░\┌╣┌Я┌д┌ф┌╣┌д░╒┼т┌и∙ю┌в┌л┌и█s┌╘┌х┌╘┌а┌х│B</FONT></B><BR>
 <HR><BR>
 </BODY>
 </HTML>
diff --git a/spec/fixtures/requests/sjis_with_wrong_charset.txt b/spec/fixtures/requests/sjis_with_wrong_charset.txt
new file mode 100644
index 000000000..456750c6b
--- /dev/null
+++ b/spec/fixtures/requests/sjis_with_wrong_charset.txt
@@ -0,0 +1,20 @@
+HTTP/1.1 200 OK
+Server: nginx/1.11.10
+Date: Tue, 04 Jul 2017 16:43:39 GMT
+Content-Type: text/html; charset=utf-8
+Content-Length: 273
+Connection: keep-alive
+Last-Modified: Tue, 04 Jul 2017 16:41:34 GMT
+Accept-Ranges: bytes
+
+<HTML>
+<HEAD>
+  <META NAME="GENERATOR" CONTENT="Adobe PageMill 3.0J Mac">
+  <META HTTP-EQUIV="Content-Type" CONTENT="text/html;CHARSET=x-sjis">
+  <TITLE>SJIS┌л┐y│[┐W</TITLE>
+</HEAD>
+<BODY>
+<P><CENTER><B><FONT SIZE="+2">▌└┌Ю⌠╞■N┌э┌╣┌д┌╒┌М┌Д┌И▀L■O░l┌а┌д┌Ю┌л┌л▌·┌е┌╣┌═┌Х┌е┌Ї│B┌Ю┌╣▌·┼т┌и┬с√║▌р┌м░Ё┌╣┌ґ┌г┌Я┌х■ґ┴О┌э┌╧┌Ў┌э┌е┌╙░\┌╣▐Ц┌╟┌╙┌╒┌Г┌а┌╣┌А┌И┌Ґ┌и┌м▌Q█l▀A┌И┌Ґ┌╒┌Ў┌╘┌Г│A▐ґ┌╣┌и┌Ю┌Б┌а┌═┌а┌э┌╣┌х┌Ґ│B▀Ю┌╘┌Г┌╒┌є┌х┌╒┌л┌м┌г┌є┌Ю▀Ц▄▌┌П┌е┌╚┌И┌Ў┌╞┌Ґ┌Ґ┌ґ┌Ґ│B┌╞┌а┌╣┌д┴╙⌠c┌Ё┌Я┌и■Ґ█R█K▐ґ┌╣▓╔┌и┴]┌╗┌е┌╣┌Е▀Ю≈м┌╠┌є┌╣┌Ґ▄═≈м┌═┌х┌Ґ┌╘▌w░}┌╙┌ф┌╒┌є┌╗▐o⌠Э┌Х┌х┌ґ┌Ў┌К┌х┌═┌Х┌д│A┌╩┌л░л┌м▌└┌╘▀Ю≈м┴A┌П⌠{┌Г┌╘┌Г│A▀v▄Є┌Ё┌Я┌л┌Ю┌л┌П┌╙┌Ґ┌л┌╒┌б┌╙┌╣┌╘┌И┌и┌╡┼С√]┌ф▄Э┌╒┌н┌╩┌Йman┌и┌╡√╣▐┌┌ж▌Q┌Х┌Ф┌є┌и⌠╞▌·┌и┌╡┴┴░Ю┌╙┌╣┌е┌х┌Г┌л┌е│A▒Ґ∙╙┌Ю┌╣∙\≈═┌и∙о┌а┌Ґ┌д┌ґ┌Й┌е┌Ї▌√┌е█l┌і┌Ґ┌Ґ│B┌╣┌╘┌Ю≈А┌і┌н┌╡┌╙┌Ґ┌╙┌ф┌г┌э┌Г┌Ю┌л┌Ю▌ю█ш┌ч┌Б┌щ┌ф┌═┌Х┌е┌Ї┌д│A┌╠┌л▌╘∙╙┌е┌м░\┌╣┌Я┌д┌ф┌╣┌д░╒┼т┌и∙ю┌в┌л┌и█s┌╘┌х┌╘┌а┌х│B</FONT></B><BR>
+<HR><BR>
+</BODY>
+</HTML>
diff --git a/spec/services/fetch_link_card_service_spec.rb b/spec/services/fetch_link_card_service_spec.rb
index 7d7f8e748..698eb0324 100644
--- a/spec/services/fetch_link_card_service_spec.rb
+++ b/spec/services/fetch_link_card_service_spec.rb
@@ -8,6 +8,10 @@ RSpec.describe FetchLinkCardService do
     stub_request(:get, 'http://example.xn--fiqs8s/').to_return(request_fixture('idn.txt'))
     stub_request(:head, 'http://example.com/sjis').to_return(status: 200, headers: { 'Content-Type' => 'text/html' })
     stub_request(:get, 'http://example.com/sjis').to_return(request_fixture('sjis.txt'))
+    stub_request(:head, 'http://example.com/sjis_with_wrong_charset').to_return(status: 200, headers: { 'Content-Type' => 'text/html' })
+    stub_request(:get, 'http://example.com/sjis_with_wrong_charset').to_return(request_fixture('sjis_with_wrong_charset.txt'))
+    stub_request(:head, 'http://example.com/koi8-r').to_return(status: 200, headers: { 'Content-Type' => 'text/html' })
+    stub_request(:get, 'http://example.com/koi8-r').to_return(request_fixture('koi8-r.txt'))
     stub_request(:head, 'https://github.com/qbi/WannaCry').to_return(status: 404)
 
     subject.call(status)
@@ -27,6 +31,25 @@ RSpec.describe FetchLinkCardService do
 
       it 'works with SJIS' do
         expect(a_request(:get, 'http://example.com/sjis')).to have_been_made.at_least_once
+        expect(status.preview_card.title).to eq("SJISЦ│ўЦ┐ Ц┐╪Ц┌╦")
+      end
+    end
+
+    context do
+      let(:status) { Fabricate(:status, text: 'Check out http://example.com/sjis_with_wrong_charset') }
+
+      it 'works with SJIS even with wrong charset header' do
+        expect(a_request(:get, 'http://example.com/sjis_with_wrong_charset')).to have_been_made.at_least_once
+        expect(status.preview_card.title).to eq("SJISЦ│ўЦ┐ Ц┐╪Ц┌╦")
+      end
+    end
+
+    context do
+      let(:status) { Fabricate(:status, text: 'Check out http://example.com/koi8-r') }
+
+      it 'works with koi8-r' do
+        expect(a_request(:get, 'http://example.com/koi8-r')).to have_been_made.at_least_once
+        expect(status.preview_card.title).to eq("п°пЎя│п╨пЎп╡я▐ пҐп╟я┤п╦пҐп╟п╣я┌я┼ я┌пЎп╩я▄п╨пЎ п╡я┼ XVI я│я┌. п©я─п╦п╡п╩п╣п╨п╟я┌я▄ п╡пҐп╦п╪п╟пҐп╣ п╦пҐпЎя│я┌я─п╟пҐя├п╣п╡я┼.")
       end
     end
   end