From 3018043fc2b405e42070b49e965bc7a0a38ecf2a Mon Sep 17 00:00:00 2001 From: Patrick Figel Date: Mon, 11 Sep 2017 15:11:13 +0200 Subject: Add OpenStack Keystone V3 support (#4889) Keystone V2 is deprecated in favour of V3. This adds the necessary connection parameters for establishing a V3 connection. Connections to V2 endpoints are still possible and the configuration should remain compatible. This also introduces a SWIFT_REGION variable for multi-region OpenStack environments and a SWIFT_CACHE_TTL that controls how long tokens and other meta-data is cached for. Caching tokens avoids rate-limiting errors that would result in media uploads becoming unavailable during high load or when using tasks like media:remove_remote. fog-openstack only supports token caching for V3 endpoints, so a recommendation for using V3 was added. --- config/initializers/paperclip.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'config/initializers') diff --git a/config/initializers/paperclip.rb b/config/initializers/paperclip.rb index e9f455251..f27aae7ec 100644 --- a/config/initializers/paperclip.rb +++ b/config/initializers/paperclip.rb @@ -47,9 +47,12 @@ elsif ENV['SWIFT_ENABLED'] == 'true' fog_credentials: { provider: 'OpenStack', openstack_username: ENV.fetch('SWIFT_USERNAME'), - openstack_tenant: ENV.fetch('SWIFT_TENANT'), + openstack_project_name: ENV.fetch('SWIFT_TENANT'), openstack_api_key: ENV.fetch('SWIFT_PASSWORD'), openstack_auth_url: ENV.fetch('SWIFT_AUTH_URL'), + openstack_domain_name: ENV['SWIFT_DOMAIN_NAME'] || 'default', + openstack_region: ENV['SWIFT_REGION'], + openstack_cache_ttl: ENV['SWIFT_CACHE_TTL'] || 60, }, fog_directory: ENV.fetch('SWIFT_CONTAINER'), fog_host: ENV.fetch('SWIFT_OBJECT_URL'), -- cgit From 1aad015bbbe7957827c2b921a21c53ce11c6ac36 Mon Sep 17 00:00:00 2001 From: abcang Date: Thu, 14 Sep 2017 22:12:43 +0900 Subject: Revert unique retry job (#4937) * Revert "Enable UniqueRetryJobMiddleware even when called from sidekiq worker (#4836)" This reverts commit 6859d4c0289e767955aac3f345074220fe200604. * Revert "Do not execute the job with the same arguments as the retry job (#4814)" This reverts commit be7ffa2d7539d5a1946a3933cb9d242b9fac0ddc. --- app/workers/pubsubhubbub/subscribe_worker.rb | 2 +- config/application.rb | 1 - config/initializers/sidekiq.rb | 6 ------ lib/mastodon/unique_retry_job_middleware.rb | 20 -------------------- 4 files changed, 1 insertion(+), 28 deletions(-) delete mode 100644 lib/mastodon/unique_retry_job_middleware.rb (limited to 'config/initializers') diff --git a/app/workers/pubsubhubbub/subscribe_worker.rb b/app/workers/pubsubhubbub/subscribe_worker.rb index 130c967e0..7560c2671 100644 --- a/app/workers/pubsubhubbub/subscribe_worker.rb +++ b/app/workers/pubsubhubbub/subscribe_worker.rb @@ -3,7 +3,7 @@ class Pubsubhubbub::SubscribeWorker include Sidekiq::Worker - sidekiq_options queue: 'push', retry: 10, unique: :until_executed, dead: false, unique_retry: true + sidekiq_options queue: 'push', retry: 10, unique: :until_executed, dead: false sidekiq_retry_in do |count| case count diff --git a/config/application.rb b/config/application.rb index f98f7af16..b6ce74147 100644 --- a/config/application.rb +++ b/config/application.rb @@ -10,7 +10,6 @@ require_relative '../app/lib/exceptions' require_relative '../lib/paperclip/gif_transcoder' require_relative '../lib/paperclip/video_transcoder' require_relative '../lib/mastodon/version' -require_relative '../lib/mastodon/unique_retry_job_middleware' Dotenv::Railtie.load diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb index 0ee77730e..b70784d79 100644 --- a/config/initializers/sidekiq.rb +++ b/config/initializers/sidekiq.rb @@ -9,14 +9,8 @@ end Sidekiq.configure_server do |config| config.redis = redis_params - config.client_middleware do |chain| - chain.add Mastodon::UniqueRetryJobMiddleware - end end Sidekiq.configure_client do |config| config.redis = redis_params - config.client_middleware do |chain| - chain.add Mastodon::UniqueRetryJobMiddleware - end end diff --git a/lib/mastodon/unique_retry_job_middleware.rb b/lib/mastodon/unique_retry_job_middleware.rb deleted file mode 100644 index 75da8a0c9..000000000 --- a/lib/mastodon/unique_retry_job_middleware.rb +++ /dev/null @@ -1,20 +0,0 @@ -# frozen_string_literal: true - -class Mastodon::UniqueRetryJobMiddleware - def call(_worker_class, item, _queue, _redis_pool) - return if item['unique_retry'] && retried?(item) - yield - end - - private - - def retried?(item) - # Use unique digest key of SidekiqUniqueJobs - unique_key = SidekiqUniqueJobs::UNIQUE_DIGEST_KEY - unique_digest = item[unique_key] - class_name = item['class'] - retries = Sidekiq::RetrySet.new - - retries.any? { |job| job.item['class'] == class_name && job.item[unique_key] == unique_digest } - end -end -- cgit From 3816943e6b5e86b22c35f3c068521f7a9007deec Mon Sep 17 00:00:00 2001 From: ふぁぼ原 Date: Fri, 15 Sep 2017 01:03:20 +0900 Subject: Enable to recognize most kinds of characters as URL paths (#4941) --- app/lib/formatter.rb | 2 +- app/services/fetch_link_card_service.rb | 14 ++++++--- config/initializers/twitter_regex.rb | 42 +++++++++++++++++++++++++++ spec/lib/formatter_spec.rb | 32 ++++++++++++++++++++ spec/services/fetch_link_card_service_spec.rb | 11 +++++++ 5 files changed, 96 insertions(+), 5 deletions(-) create mode 100644 config/initializers/twitter_regex.rb (limited to 'config/initializers') diff --git a/app/lib/formatter.rb b/app/lib/formatter.rb index cacc0364f..d9f843f44 100644 --- a/app/lib/formatter.rb +++ b/app/lib/formatter.rb @@ -131,7 +131,7 @@ class Formatter end def link_html(url) - url = Addressable::URI.parse(url).display_uri.to_s + url = Addressable::URI.parse(url).to_s prefix = url.match(/\Ahttps?:\/\/(www\.)?/).to_s text = url[prefix.length, 30] suffix = url[prefix.length + 30..-1] diff --git a/app/services/fetch_link_card_service.rb b/app/services/fetch_link_card_service.rb index 215c69fe4..4acbfae7a 100644 --- a/app/services/fetch_link_card_service.rb +++ b/app/services/fetch_link_card_service.rb @@ -1,9 +1,15 @@ # frozen_string_literal: true class FetchLinkCardService < BaseService - include ActionView::Helpers::TagHelper - - URL_PATTERN = %r{https?://\S+} + URL_PATTERN = %r{ + ( # $1 URL + (https?:\/\/)? # $2 Protocol (optional) + (#{Twitter::Regex[:valid_domain]}) # $3 Domain(s) + (?::(#{Twitter::Regex[:valid_port_number]}))? # $4 Port number (optional) + (/#{Twitter::Regex[:valid_url_path]}*)? # $5 URL Path and anchor + (\?#{Twitter::Regex[:valid_url_query_chars]}*#{Twitter::Regex[:valid_url_query_ending_chars]})? # $6 Query String + ) + }iox def call(status) @status = status @@ -42,7 +48,7 @@ class FetchLinkCardService < BaseService def parse_urls if @status.local? - urls = @status.text.match(URL_PATTERN).to_a.map { |uri| Addressable::URI.parse(uri).normalize } + urls = @status.text.scan(URL_PATTERN).map { |array| Addressable::URI.parse(array[0]).normalize } else html = Nokogiri::HTML(@status.text) links = html.css('a') diff --git a/config/initializers/twitter_regex.rb b/config/initializers/twitter_regex.rb new file mode 100644 index 000000000..5a0723d24 --- /dev/null +++ b/config/initializers/twitter_regex.rb @@ -0,0 +1,42 @@ +module Twitter + class Regex + + REGEXEN[:valid_general_url_path_chars] = /[^\p{White_Space}\(\)\?]/iou + REGEXEN[:valid_url_path_ending_chars] = /[^\p{White_Space}\(\)\?!\*';:=\,\.\$%\[\]\p{Pd}_~&\|@]|(?:#{REGEXEN[:valid_url_balanced_parens]})/iou + REGEXEN[:valid_url_balanced_parens] = / + \( + (?: + #{REGEXEN[:valid_general_url_path_chars]}+ + | + # allow one nested level of balanced parentheses + (?: + #{REGEXEN[:valid_general_url_path_chars]}* + \( + #{REGEXEN[:valid_general_url_path_chars]}+ + \) + #{REGEXEN[:valid_general_url_path_chars]}* + ) + ) + \) + /iox + REGEXEN[:valid_url_path] = /(?: + (?: + #{REGEXEN[:valid_general_url_path_chars]}* + (?:#{REGEXEN[:valid_url_balanced_parens]} #{REGEXEN[:valid_general_url_path_chars]}*)* + #{REGEXEN[:valid_url_path_ending_chars]} + )|(?:#{REGEXEN[:valid_general_url_path_chars]}+\/) + )/iox + REGEXEN[:valid_url] = %r{ + ( # $1 total match + (#{REGEXEN[:valid_url_preceding_chars]}) # $2 Preceeding chracter + ( # $3 URL + (https?:\/\/)? # $4 Protocol (optional) + (#{REGEXEN[:valid_domain]}) # $5 Domain(s) + (?::(#{REGEXEN[:valid_port_number]}))? # $6 Port number (optional) + (/#{REGEXEN[:valid_url_path]}*)? # $7 URL Path and anchor + (\?#{REGEXEN[:valid_url_query_chars]}*#{REGEXEN[:valid_url_query_ending_chars]})? # $8 Query String + ) + ) + }iox + end +end diff --git a/spec/lib/formatter_spec.rb b/spec/lib/formatter_spec.rb index ab04ccbab..f9b7efac5 100644 --- a/spec/lib/formatter_spec.rb +++ b/spec/lib/formatter_spec.rb @@ -89,6 +89,38 @@ RSpec.describe Formatter do end end + context 'matches a URL with Japanese path string' do + let(:text) { 'https://ja.wikipedia.org/wiki/日本' } + + it 'has valid URL' do + is_expected.to include 'href="https://ja.wikipedia.org/wiki/%E6%97%A5%E6%9C%AC"' + end + end + + context 'matches a URL with Korean path string' do + let(:text) { 'https://ko.wikipedia.org/wiki/대한민국' } + + it 'has valid URL' do + is_expected.to include 'href="https://ko.wikipedia.org/wiki/%EB%8C%80%ED%95%9C%EB%AF%BC%EA%B5%AD"' + end + end + + context 'matches a URL with Simplified Chinese path string' do + let(:text) { 'https://baike.baidu.com/item/中华人民共和国' } + + it 'has valid URL' do + is_expected.to include 'href="https://baike.baidu.com/item/%E4%B8%AD%E5%8D%8E%E4%BA%BA%E6%B0%91%E5%85%B1%E5%92%8C%E5%9B%BD"' + end + end + + context 'matches a URL with Traditional Chinese path string' do + let(:text) { 'https://zh.wikipedia.org/wiki/臺灣' } + + it 'has valid URL' do + is_expected.to include 'href="https://zh.wikipedia.org/wiki/%E8%87%BA%E7%81%A3"' + end + end + context 'contains HTML (script tag)' do let(:text) { '' } diff --git a/spec/services/fetch_link_card_service_spec.rb b/spec/services/fetch_link_card_service_spec.rb index b0aa740ac..ba61d22c3 100644 --- a/spec/services/fetch_link_card_service_spec.rb +++ b/spec/services/fetch_link_card_service_spec.rb @@ -12,6 +12,8 @@ RSpec.describe FetchLinkCardService do stub_request(:get, 'http://example.com/sjis_with_wrong_charset').to_return(request_fixture('sjis_with_wrong_charset.txt')) stub_request(:head, 'http://example.com/koi8-r').to_return(status: 200, headers: { 'Content-Type' => 'text/html' }) stub_request(:get, 'http://example.com/koi8-r').to_return(request_fixture('koi8-r.txt')) + stub_request(:head, 'http://example.com/日本語').to_return(status: 200, headers: { 'Content-Type' => 'text/html' }) + stub_request(:get, 'http://example.com/日本語').to_return(request_fixture('sjis.txt')) stub_request(:head, 'https://github.com/qbi/WannaCry').to_return(status: 404) subject.call(status) @@ -52,6 +54,15 @@ RSpec.describe FetchLinkCardService do expect(status.preview_cards.first.title).to eq("Московя начинаетъ только въ XVI ст. привлекать внимане иностранцевъ.") end end + + context do + let(:status) { Fabricate(:status, text: 'テストhttp://example.com/日本語') } + + it 'works with Japanese path string' do + expect(a_request(:get, 'http://example.com/日本語')).to have_been_made.at_least_once + expect(status.preview_cards.first.title).to eq("SJISのページ") + end + end end context 'in a remote status' do -- cgit From 6db034a866b78e4e98b122461ca84763f0104381 Mon Sep 17 00:00:00 2001 From: unarist Date: Tue, 19 Sep 2017 04:25:40 +0900 Subject: Re-allow underscore on valid_url_path_ending_chars (#4999) Limiting allowed characters in the last character of the URL is came from twitter-text, but underscore is allowed on there, and Mastodon before #4941. --- app/javascript/mastodon/features/compose/util/url_regex.js | 2 +- config/initializers/twitter_regex.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'config/initializers') diff --git a/app/javascript/mastodon/features/compose/util/url_regex.js b/app/javascript/mastodon/features/compose/util/url_regex.js index c15b9696a..e676d1879 100644 --- a/app/javascript/mastodon/features/compose/util/url_regex.js +++ b/app/javascript/mastodon/features/compose/util/url_regex.js @@ -172,7 +172,7 @@ export const urlRegex = (function() { , 'i'); // Valid end-of-path chracters (so /foo. does not gobble the period). // 1. Allow =&# for empty URL parameters and other URL-join artifacts - regexen.validUrlPathEndingChars = regexSupplant(/[^#{spaces_group}\(\)\?!\*';:=\,\.\$%\[\]#{pd}_~&\|@]|(?:#{validUrlBalancedParens})/i); + regexen.validUrlPathEndingChars = regexSupplant(/[^#{spaces_group}\(\)\?!\*';:=\,\.\$%\[\]#{pd}~&\|@]|(?:#{validUrlBalancedParens})/i); // Allow @ in a url, but only in the middle. Catch things like http://example.com/@user/ regexen.validUrlPath = regexSupplant('(?:' + '(?:' + diff --git a/config/initializers/twitter_regex.rb b/config/initializers/twitter_regex.rb index 5a0723d24..e924fac22 100644 --- a/config/initializers/twitter_regex.rb +++ b/config/initializers/twitter_regex.rb @@ -2,7 +2,7 @@ module Twitter class Regex REGEXEN[:valid_general_url_path_chars] = /[^\p{White_Space}\(\)\?]/iou - REGEXEN[:valid_url_path_ending_chars] = /[^\p{White_Space}\(\)\?!\*';:=\,\.\$%\[\]\p{Pd}_~&\|@]|(?:#{REGEXEN[:valid_url_balanced_parens]})/iou + REGEXEN[:valid_url_path_ending_chars] = /[^\p{White_Space}\(\)\?!\*';:=\,\.\$%\[\]\p{Pd}~&\|@]|(?:#{REGEXEN[:valid_url_balanced_parens]})/iou REGEXEN[:valid_url_balanced_parens] = / \( (?: -- cgit