From 7d8e3721aea71315b0ef8e66cdc2ede0fe6ffc2a Mon Sep 17 00:00:00 2001 From: "Akihiko Odaki (@fn_aki@pawoo.net)" Date: Wed, 28 Jun 2017 21:50:23 +0900 Subject: Overwrite old statuses with reblogs in PrecomputeFeedService (#3984) --- spec/services/precompute_feed_service_spec.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'spec/services') diff --git a/spec/services/precompute_feed_service_spec.rb b/spec/services/precompute_feed_service_spec.rb index 9f56b0256..72235a966 100644 --- a/spec/services/precompute_feed_service_spec.rb +++ b/spec/services/precompute_feed_service_spec.rb @@ -11,12 +11,12 @@ RSpec.describe PrecomputeFeedService do account = Fabricate(:account) followed_account = Fabricate(:account) Fabricate(:follow, account: account, target_account: followed_account) - status = Fabricate(:status, account: followed_account) - - expected_redis_args = FeedManager.instance.key(:home, account.id), status.id, status.id - expect_any_instance_of(Redis).to receive(:zadd).with(*expected_redis_args) + reblog = Fabricate(:status, account: followed_account) + status = Fabricate(:status, account: account, reblog: reblog) subject.call(account) + + expect(Redis.current.zscore(FeedManager.instance.key(:home, account.id), reblog.id)).to eq status.id end end end -- cgit From 7362469d8956d5f972283aadd4157631aa66b085 Mon Sep 17 00:00:00 2001 From: "Akihiko Odaki (@fn_aki@pawoo.net)" Date: Fri, 30 Jun 2017 20:39:42 +0900 Subject: Do not raise an error if PrecomputeFeed could not find any status (#4015) --- app/services/precompute_feed_service.rb | 2 +- spec/services/precompute_feed_service_spec.rb | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) (limited to 'spec/services') diff --git a/app/services/precompute_feed_service.rb b/app/services/precompute_feed_service.rb index e2f6ff0cb..a32ba1dae 100644 --- a/app/services/precompute_feed_service.rb +++ b/app/services/precompute_feed_service.rb @@ -16,7 +16,7 @@ class PrecomputeFeedService < BaseService pairs = statuses.reverse_each.map(&method(:process_status)) redis.pipelined do - redis.zadd(account_home_key, pairs) + redis.zadd(account_home_key, pairs) if pairs.any? redis.del("account:#{@account.id}:regeneration") end end diff --git a/spec/services/precompute_feed_service_spec.rb b/spec/services/precompute_feed_service_spec.rb index 72235a966..e2294469c 100644 --- a/spec/services/precompute_feed_service_spec.rb +++ b/spec/services/precompute_feed_service_spec.rb @@ -18,5 +18,10 @@ RSpec.describe PrecomputeFeedService do expect(Redis.current.zscore(FeedManager.instance.key(:home, account.id), reblog.id)).to eq status.id end + + it 'does not raise an error even if it could not find any status' do + account = Fabricate(:account) + subject.call(account) + end end end -- cgit From 5e6acf960183aea9440ce0d9e28c86f043e88c54 Mon Sep 17 00:00:00 2001 From: abcang Date: Wed, 5 Jul 2017 21:54:21 +0900 Subject: Fix Nokogiri::HTML at FetchLinkCardService (#4072) --- app/services/fetch_link_card_service.rb | 4 +++- spec/fixtures/requests/sjis.txt | 20 ++++++++++++++++++++ spec/services/fetch_link_card_service_spec.rb | 10 ++++++++++ 3 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 spec/fixtures/requests/sjis.txt (limited to 'spec/services') diff --git a/app/services/fetch_link_card_service.rb b/app/services/fetch_link_card_service.rb index 4ce221267..8ddaa2bf4 100644 --- a/app/services/fetch_link_card_service.rb +++ b/app/services/fetch_link_card_service.rb @@ -1,4 +1,5 @@ # frozen_string_literal: true +require 'nkf' class FetchLinkCardService < BaseService include HttpHelper @@ -84,7 +85,8 @@ class FetchLinkCardService < BaseService return if response.code != 200 || response.mime_type != 'text/html' - page = Nokogiri::HTML(response.to_s) + html = response.to_s + page = Nokogiri::HTML(html, nil, NKF.guess(html).to_s) card.type = :link card.title = meta_property(page, 'og:title') || page.at_xpath('//title')&.content diff --git a/spec/fixtures/requests/sjis.txt b/spec/fixtures/requests/sjis.txt new file mode 100644 index 000000000..9041aa25d --- /dev/null +++ b/spec/fixtures/requests/sjis.txt @@ -0,0 +1,20 @@ +HTTP/1.1 200 OK +Server: nginx/1.11.10 +Date: Tue, 04 Jul 2017 16:43:39 GMT +Content-Type: text/html +Content-Length: 273 +Connection: keep-alive +Last-Modified: Tue, 04 Jul 2017 16:41:34 GMT +Accept-Ranges: bytes + + + + + + JSISのページ + + +

SJISのページ
+

+ + diff --git a/spec/services/fetch_link_card_service_spec.rb b/spec/services/fetch_link_card_service_spec.rb index 9df41cf55..7d7f8e748 100644 --- a/spec/services/fetch_link_card_service_spec.rb +++ b/spec/services/fetch_link_card_service_spec.rb @@ -6,6 +6,8 @@ RSpec.describe FetchLinkCardService do before do stub_request(:head, 'http://example.xn--fiqs8s/').to_return(status: 200, headers: { 'Content-Type' => 'text/html' }) stub_request(:get, 'http://example.xn--fiqs8s/').to_return(request_fixture('idn.txt')) + stub_request(:head, 'http://example.com/sjis').to_return(status: 200, headers: { 'Content-Type' => 'text/html' }) + stub_request(:get, 'http://example.com/sjis').to_return(request_fixture('sjis.txt')) stub_request(:head, 'https://github.com/qbi/WannaCry').to_return(status: 404) subject.call(status) @@ -19,6 +21,14 @@ RSpec.describe FetchLinkCardService do expect(a_request(:get, 'http://example.xn--fiqs8s/')).to have_been_made.at_least_once end end + + context do + let(:status) { Fabricate(:status, text: 'Check out http://example.com/sjis') } + + it 'works with SJIS' do + expect(a_request(:get, 'http://example.com/sjis')).to have_been_made.at_least_once + end + end end context 'in a remote status' do -- cgit From 007ab330e6ffb1e07995d4e306473d457043e2eb Mon Sep 17 00:00:00 2001 From: nullkal Date: Sun, 9 Jul 2017 05:44:31 +0900 Subject: Use charlock_holmes instead of nkf at FetchLinkCardService (#4080) * Specs for language detection * Use CharlockHolmes instead of NKF * Correct mistakes * Correct style * Set hint_enc instead of falling back and strip_tags * Improve specs * Add dependencies --- .travis.yml | 1 + Aptfile | 1 + Dockerfile | 1 + Gemfile | 1 + Gemfile.lock | 2 ++ Vagrantfile | 1 + app/services/fetch_link_card_service.rb | 8 ++++++-- spec/fixtures/requests/koi8-r.txt | 20 +++++++++++++++++++ spec/fixtures/requests/sjis.txt | 4 ++-- spec/fixtures/requests/sjis_with_wrong_charset.txt | 20 +++++++++++++++++++ spec/services/fetch_link_card_service_spec.rb | 23 ++++++++++++++++++++++ 11 files changed, 78 insertions(+), 4 deletions(-) create mode 100644 spec/fixtures/requests/koi8-r.txt create mode 100644 spec/fixtures/requests/sjis_with_wrong_charset.txt (limited to 'spec/services') diff --git a/.travis.yml b/.travis.yml index 4bb332666..4d4dc0893 100644 --- a/.travis.yml +++ b/.travis.yml @@ -32,6 +32,7 @@ addons: - g++-6 - libprotobuf-dev - protobuf-compiler + - libicu-dev rvm: - 2.3.4 diff --git a/Aptfile b/Aptfile index 0456343ef..3af0956e3 100644 --- a/Aptfile +++ b/Aptfile @@ -3,3 +3,4 @@ libprotobuf-dev ffmpeg libxdamage1 libxfixes3 +libicu-dev diff --git a/Dockerfile b/Dockerfile index 7033cddd4..97a691393 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,6 +25,7 @@ RUN echo "@edge https://nl.alpinelinux.org/alpine/edge/main" >> /etc/apk/reposit ffmpeg \ file \ git \ + icu-dev \ imagemagick@edge \ libpq \ libxml2 \ diff --git a/Gemfile b/Gemfile index 95c74eef9..b52685cba 100644 --- a/Gemfile +++ b/Gemfile @@ -22,6 +22,7 @@ gem 'active_model_serializers', '~> 0.10' gem 'addressable', '~> 2.5' gem 'bootsnap' gem 'browser' +gem 'charlock_holmes', '~> 0.7.3' gem 'cld3', '~> 3.1' gem 'devise', '~> 4.2' gem 'devise-two-factor', '~> 3.0' diff --git a/Gemfile.lock b/Gemfile.lock index 71f83f736..de0d6a107 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -106,6 +106,7 @@ GEM rack (>= 1.0.0) rack-test (>= 0.5.4) xpath (~> 2.0) + charlock_holmes (0.7.3) case_transform (0.2) activesupport chunky_png (1.3.8) @@ -501,6 +502,7 @@ DEPENDENCIES capistrano-rbenv (~> 2.1) capistrano-yarn (~> 2.0) capybara (~> 2.14) + charlock_holmes (~> 0.7.3) cld3 (~> 3.1) climate_control (~> 0.2) devise (~> 4.2) diff --git a/Vagrantfile b/Vagrantfile index 1f56fcfb3..cbe6623b3 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -37,6 +37,7 @@ sudo apt-get install \ yarn \ libprotobuf-dev \ libreadline-dev \ + libicu-dev \ -y # Install rvm diff --git a/app/services/fetch_link_card_service.rb b/app/services/fetch_link_card_service.rb index 8ddaa2bf4..6ef3abb66 100644 --- a/app/services/fetch_link_card_service.rb +++ b/app/services/fetch_link_card_service.rb @@ -1,5 +1,4 @@ # frozen_string_literal: true -require 'nkf' class FetchLinkCardService < BaseService include HttpHelper @@ -86,7 +85,12 @@ class FetchLinkCardService < BaseService return if response.code != 200 || response.mime_type != 'text/html' html = response.to_s - page = Nokogiri::HTML(html, nil, NKF.guess(html).to_s) + + detector = CharlockHolmes::EncodingDetector.new + detector.strip_tags = true + + guess = detector.detect(html, response.charset) + page = Nokogiri::HTML(html, nil, guess&.fetch(:encoding)) card.type = :link card.title = meta_property(page, 'og:title') || page.at_xpath('//title')&.content diff --git a/spec/fixtures/requests/koi8-r.txt b/spec/fixtures/requests/koi8-r.txt new file mode 100644 index 000000000..d4242af01 --- /dev/null +++ b/spec/fixtures/requests/koi8-r.txt @@ -0,0 +1,20 @@ +HTTP/1.1 200 OK +Server: nginx/1.11.10 +Date: Tue, 04 Jul 2017 16:43:39 GMT +Content-Type: text/html +Content-Length: 273 +Connection: keep-alive +Last-Modified: Tue, 04 Jul 2017 16:41:34 GMT +Accept-Ranges: bytes + + + + + + ﾏﾓﾋﾏﾗﾑ ﾎﾁﾞﾉﾎﾁﾅﾔﾟ ﾔﾏﾌﾘﾋﾏ ﾗﾟ XVI ﾓﾔ. ﾐﾒﾉﾗﾌﾅﾋﾁﾔﾘ ﾗﾎﾉﾍﾁﾎﾅ ﾉﾎﾏﾓﾔﾒﾁﾎﾃﾅﾗﾟ. + + +

ﾏﾓﾋﾏﾗﾑ ﾎﾁﾞﾉﾎﾁﾅﾔﾟ ﾔﾏﾌﾘﾋﾏ ﾗﾟ XVI ﾓﾔ. ﾐﾒﾉﾗﾌﾅﾋﾁﾔﾘ ﾗﾎﾉﾍﾁﾎﾅ ﾉﾎﾏﾓﾔﾒﾁﾎﾃﾅﾗﾟ.
+

+ + diff --git a/spec/fixtures/requests/sjis.txt b/spec/fixtures/requests/sjis.txt index 9041aa25d..faf18d35c 100644 --- a/spec/fixtures/requests/sjis.txt +++ b/spec/fixtures/requests/sjis.txt @@ -11,10 +11,10 @@ Accept-Ranges: bytes - JSISのページ + SJISのページ -

SJISのページ
+

私も同年ましていわゆる記念人ってものの時でしありです。もし時間に意味者は正しくどんな発会ませだまでが申し上げがいらっしゃるたには参考帰るたいだから、少しにもやっあっましなた。金からいうないのはどうも九月をできるだけたたくた。けっして岡田さんに反抗幸少し徴に云おでしょ金力こうした権力あなたか指図がというお出入りなくだろなありて、その昔は私か金力陰を怒らから、久原さんのものをがたのいつがしかるにご希望と向いばそれmanにご矛盾へ参りように同時にご演説がしでならので、多分もし表裏に変ったてくれです事で考えたた。しかも例えばごがたがとどまらものも実際むやみとありですて、この自分では申しんてとして世間に並べのに行かなかっな。

diff --git a/spec/fixtures/requests/sjis_with_wrong_charset.txt b/spec/fixtures/requests/sjis_with_wrong_charset.txt new file mode 100644 index 000000000..456750c6b --- /dev/null +++ b/spec/fixtures/requests/sjis_with_wrong_charset.txt @@ -0,0 +1,20 @@ +HTTP/1.1 200 OK +Server: nginx/1.11.10 +Date: Tue, 04 Jul 2017 16:43:39 GMT +Content-Type: text/html; charset=utf-8 +Content-Length: 273 +Connection: keep-alive +Last-Modified: Tue, 04 Jul 2017 16:41:34 GMT +Accept-Ranges: bytes + + + + + + SJISのページ + + +

私も同年ましていわゆる記念人ってものの時でしありです。もし時間に意味者は正しくどんな発会ませだまでが申し上げがいらっしゃるたには参考帰るたいだから、少しにもやっあっましなた。金からいうないのはどうも九月をできるだけたたくた。けっして岡田さんに反抗幸少し徴に云おでしょ金力こうした権力あなたか指図がというお出入りなくだろなありて、その昔は私か金力陰を怒らから、久原さんのものをがたのいつがしかるにご希望と向いばそれmanにご矛盾へ参りように同時にご演説がしでならので、多分もし表裏に変ったてくれです事で考えたた。しかも例えばごがたがとどまらものも実際むやみとありですて、この自分では申しんてとして世間に並べのに行かなかっな。
+

+ + diff --git a/spec/services/fetch_link_card_service_spec.rb b/spec/services/fetch_link_card_service_spec.rb index 7d7f8e748..698eb0324 100644 --- a/spec/services/fetch_link_card_service_spec.rb +++ b/spec/services/fetch_link_card_service_spec.rb @@ -8,6 +8,10 @@ RSpec.describe FetchLinkCardService do stub_request(:get, 'http://example.xn--fiqs8s/').to_return(request_fixture('idn.txt')) stub_request(:head, 'http://example.com/sjis').to_return(status: 200, headers: { 'Content-Type' => 'text/html' }) stub_request(:get, 'http://example.com/sjis').to_return(request_fixture('sjis.txt')) + stub_request(:head, 'http://example.com/sjis_with_wrong_charset').to_return(status: 200, headers: { 'Content-Type' => 'text/html' }) + stub_request(:get, 'http://example.com/sjis_with_wrong_charset').to_return(request_fixture('sjis_with_wrong_charset.txt')) + stub_request(:head, 'http://example.com/koi8-r').to_return(status: 200, headers: { 'Content-Type' => 'text/html' }) + stub_request(:get, 'http://example.com/koi8-r').to_return(request_fixture('koi8-r.txt')) stub_request(:head, 'https://github.com/qbi/WannaCry').to_return(status: 404) subject.call(status) @@ -27,6 +31,25 @@ RSpec.describe FetchLinkCardService do it 'works with SJIS' do expect(a_request(:get, 'http://example.com/sjis')).to have_been_made.at_least_once + expect(status.preview_card.title).to eq("SJIS縺ｮ繝壹ｼ繧ｸ") + end + end + + context do + let(:status) { Fabricate(:status, text: 'Check out http://example.com/sjis_with_wrong_charset') } + + it 'works with SJIS even with wrong charset header' do + expect(a_request(:get, 'http://example.com/sjis_with_wrong_charset')).to have_been_made.at_least_once + expect(status.preview_card.title).to eq("SJIS縺ｮ繝壹ｼ繧ｸ") + end + end + + context do + let(:status) { Fabricate(:status, text: 'Check out http://example.com/koi8-r') } + + it 'works with koi8-r' do + expect(a_request(:get, 'http://example.com/koi8-r')).to have_been_made.at_least_once + expect(status.preview_card.title).to eq("ﾐ慴ｾﾑﾐｺﾐｾﾐｲﾑ ﾐｽﾐｰﾑﾐｸﾐｽﾐｰﾐｵﾑび ﾑひｾﾐｻﾑ糊ｺﾐｾ ﾐｲﾑ XVI ﾑﾑ. ﾐｿﾑﾐｸﾐｲﾐｻﾐｵﾐｺﾐｰﾑび ﾐｲﾐｽﾐｸﾐｼﾐｰﾐｽﾐｵ ﾐｸﾐｽﾐｾﾑﾑびﾐｰﾐｽﾑﾐｵﾐｲﾑ.") end end end -- cgit From 7bacdd718a143f54f47ddc3afa39504636be65c0 Mon Sep 17 00:00:00 2001 From: "Akihiko Odaki (@fn_aki@pawoo.net)" Date: Tue, 11 Jul 2017 08:00:01 +0900 Subject: Fix PrecomputeFeedService for filtered statuses (#4148) --- app/services/precompute_feed_service.rb | 4 ++-- spec/services/precompute_feed_service_spec.rb | 12 ++++++++++++ 2 files changed, 14 insertions(+), 2 deletions(-) (limited to 'spec/services') diff --git a/app/services/precompute_feed_service.rb b/app/services/precompute_feed_service.rb index a32ba1dae..85635a008 100644 --- a/app/services/precompute_feed_service.rb +++ b/app/services/precompute_feed_service.rb @@ -13,7 +13,7 @@ class PrecomputeFeedService < BaseService attr_reader :account def populate_feed - pairs = statuses.reverse_each.map(&method(:process_status)) + pairs = statuses.reverse_each.lazy.reject(&method(:status_filtered?)).map(&method(:process_status)).to_a redis.pipelined do redis.zadd(account_home_key, pairs) if pairs.any? @@ -22,7 +22,7 @@ class PrecomputeFeedService < BaseService end def process_status(status) - [status.id, status.reblog? ? status.reblog_of_id : status.id] unless status_filtered?(status) + [status.id, status.reblog? ? status.reblog_of_id : status.id] end def status_filtered?(status) diff --git a/spec/services/precompute_feed_service_spec.rb b/spec/services/precompute_feed_service_spec.rb index e2294469c..dbd08ac1b 100644 --- a/spec/services/precompute_feed_service_spec.rb +++ b/spec/services/precompute_feed_service_spec.rb @@ -23,5 +23,17 @@ RSpec.describe PrecomputeFeedService do account = Fabricate(:account) subject.call(account) end + + it 'filters statuses' do + account = Fabricate(:account) + muted_account = Fabricate(:account) + Fabricate(:mute, account: account, target_account: muted_account) + reblog = Fabricate(:status, account: muted_account) + status = Fabricate(:status, account: account, reblog: reblog) + + subject.call(account) + + expect(Redis.current.zscore(FeedManager.instance.key(:home, account.id), reblog.id)).to eq nil + end end end -- cgit