From 43eff898a0b0f31aaf042d9d387aaece2627a01d Mon Sep 17 00:00:00 2001 From: Claire Date: Wed, 17 Mar 2021 10:09:55 +0100 Subject: Prepare Mastodon for Rails 6 (#15911) * Fix misuse of foreign_type * Fix use of removed "add_template_helper" * Use response.media_type instead of response.content_type in tests * Fix CSV export controller test on Rails 6 Rails 6 sets a "filename*" field in the Content-Disposition header to explicitly encode the filename as UTF-8. This changes checks the first part of the Content-Disposition header so it matches in both Rails 5 and Rails 6. * Fix emoji formatting with Rails 6 * Make emoji output more idiomatic and robust * Switch from redis-rails gem to built-in Rails redis cache storage --- app/lib/formatter.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'app/lib/formatter.rb') diff --git a/app/lib/formatter.rb b/app/lib/formatter.rb index 7252234d6..6fb5d5419 100644 --- a/app/lib/formatter.rb +++ b/app/lib/formatter.rb @@ -158,9 +158,9 @@ class Formatter original_url, static_url = emoji replacement = begin if animate - "\":#{encode(shortcode)}:\"" + image_tag(original_url, draggable: false, class: 'emojione', alt: ":#{shortcode}:", title: ":#{shortcode}:") else - "\":#{encode(shortcode)}:\"" + image_tag(original_url, draggable: false, class: 'emojione custom-emoji', alt: ":#{shortcode}:", title: ":#{shortcode}:", data: { original: original_url, static: static_url }) end end before_html = shortname_start_index.positive? ? html[0..shortname_start_index - 1] : '' -- cgit From a4dcaef53b97c58fd153de6f151b6fada40f3442 Mon Sep 17 00:00:00 2001 From: Claire Date: Fri, 19 Mar 2021 02:42:43 +0100 Subject: Prepare Mastodon for zeitwerk autoloader (#15917) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Prepare Mastodon for zeitwerk autoloader (Rails 6) Add inflections and rename/move a few classes. In particular, app/lib/exceptions.rb and app/lib/sanitize_config.rb were manually loaded while still in autoload paths. * Add inflection for Url → URL --- app/lib/exceptions.rb | 23 ---- app/lib/formatter.rb | 1 - app/lib/sanitize_config.rb | 119 --------------------- app/validators/url_validator.rb | 2 +- config/application.rb | 3 +- config/initializers/inflections.rb | 4 + db/migrate/20160223165723_add_url_to_statuses.rb | 2 +- db/migrate/20160223165855_add_url_to_accounts.rb | 2 +- ...0322193748_add_avatar_remote_url_to_accounts.rb | 2 +- ...0318214217_add_header_remote_url_to_accounts.rb | 2 +- ...0171130000000_add_embed_url_to_preview_cards.rb | 2 +- ...3859_add_featured_collection_url_to_accounts.rb | 2 +- .../20200529214050_add_devices_url_to_accounts.rb | 2 +- lib/exceptions.rb | 23 ++++ lib/sanitize_ext/sanitize_config.rb | 119 +++++++++++++++++++++ spec/lib/sanitize_config_spec.rb | 1 - spec/validators/url_validator_spec.rb | 2 +- 17 files changed, 157 insertions(+), 154 deletions(-) delete mode 100644 app/lib/exceptions.rb delete mode 100644 app/lib/sanitize_config.rb create mode 100644 lib/exceptions.rb create mode 100644 lib/sanitize_ext/sanitize_config.rb (limited to 'app/lib/formatter.rb') diff --git a/app/lib/exceptions.rb b/app/lib/exceptions.rb deleted file mode 100644 index 7c8e77871..000000000 --- a/app/lib/exceptions.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -module Mastodon - class Error < StandardError; end - class NotPermittedError < Error; end - class ValidationError < Error; end - class HostValidationError < ValidationError; end - class LengthValidationError < ValidationError; end - class DimensionsValidationError < ValidationError; end - class StreamValidationError < ValidationError; end - class RaceConditionError < Error; end - class RateLimitExceededError < Error; end - - class UnexpectedResponseError < Error - def initialize(response = nil) - if response.respond_to? :uri - super("#{response.uri} returned code #{response.code}") - else - super - end - end - end -end diff --git a/app/lib/formatter.rb b/app/lib/formatter.rb index 6fb5d5419..2611bcbae 100644 --- a/app/lib/formatter.rb +++ b/app/lib/formatter.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true require 'singleton' -require_relative './sanitize_config' class Formatter include Singleton diff --git a/app/lib/sanitize_config.rb b/app/lib/sanitize_config.rb deleted file mode 100644 index a2e1d9d01..000000000 --- a/app/lib/sanitize_config.rb +++ /dev/null @@ -1,119 +0,0 @@ -# frozen_string_literal: true - -class Sanitize - module Config - HTTP_PROTOCOLS = %w( - http - https - ).freeze - - LINK_PROTOCOLS = %w( - http - https - dat - dweb - ipfs - ipns - ssb - gopher - xmpp - magnet - gemini - ).freeze - - CLASS_WHITELIST_TRANSFORMER = lambda do |env| - node = env[:node] - class_list = node['class']&.split(/[\t\n\f\r ]/) - - return unless class_list - - class_list.keep_if do |e| - next true if /^(h|p|u|dt|e)-/.match?(e) # microformats classes - next true if /^(mention|hashtag)$/.match?(e) # semantic classes - next true if /^(ellipsis|invisible)$/.match?(e) # link formatting classes - end - - node['class'] = class_list.join(' ') - end - - UNSUPPORTED_HREF_TRANSFORMER = lambda do |env| - return unless env[:node_name] == 'a' - - current_node = env[:node] - - scheme = begin - if current_node['href'] =~ Sanitize::REGEX_PROTOCOL - Regexp.last_match(1).downcase - else - :relative - end - end - - current_node.replace(current_node.text) unless LINK_PROTOCOLS.include?(scheme) - end - - UNSUPPORTED_ELEMENTS_TRANSFORMER = lambda do |env| - return unless %w(h1 h2 h3 h4 h5 h6 blockquote pre ul ol li).include?(env[:node_name]) - - current_node = env[:node] - - case env[:node_name] - when 'li' - current_node.traverse do |node| - next unless %w(p ul ol li).include?(node.name) - - node.add_next_sibling('
') if node.next_sibling - node.replace(node.children) unless node.text? - end - else - current_node.name = 'p' - end - end - - MASTODON_STRICT ||= freeze_config( - elements: %w(p br span a), - - attributes: { - 'a' => %w(href rel class), - 'span' => %w(class), - }, - - add_attributes: { - 'a' => { - 'rel' => 'nofollow noopener noreferrer', - 'target' => '_blank', - }, - }, - - protocols: {}, - - transformers: [ - CLASS_WHITELIST_TRANSFORMER, - UNSUPPORTED_ELEMENTS_TRANSFORMER, - UNSUPPORTED_HREF_TRANSFORMER, - ] - ) - - MASTODON_OEMBED ||= freeze_config merge( - RELAXED, - elements: RELAXED[:elements] + %w(audio embed iframe source video), - - attributes: merge( - RELAXED[:attributes], - 'audio' => %w(controls), - 'embed' => %w(height src type width), - 'iframe' => %w(allowfullscreen frameborder height scrolling src width), - 'source' => %w(src type), - 'video' => %w(controls height loop width), - 'div' => [:data] - ), - - protocols: merge( - RELAXED[:protocols], - 'embed' => { 'src' => HTTP_PROTOCOLS }, - 'iframe' => { 'src' => HTTP_PROTOCOLS }, - 'source' => { 'src' => HTTP_PROTOCOLS } - ) - ) - end -end diff --git a/app/validators/url_validator.rb b/app/validators/url_validator.rb index d95a03fbf..f50abbe24 100644 --- a/app/validators/url_validator.rb +++ b/app/validators/url_validator.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class UrlValidator < ActiveModel::EachValidator +class URLValidator < ActiveModel::EachValidator def validate_each(record, attribute, value) record.errors.add(attribute, I18n.t('applications.invalid_url')) unless compliant?(value) end diff --git a/config/application.rb b/config/application.rb index 116eaf29d..0960247b3 100644 --- a/config/application.rb +++ b/config/application.rb @@ -6,8 +6,9 @@ require 'rails/all' # you've limited to :test, :development, or :production. Bundler.require(*Rails.groups) -require_relative '../app/lib/exceptions' +require_relative '../lib/exceptions' require_relative '../lib/enumerable' +require_relative '../lib/sanitize_ext/sanitize_config' require_relative '../lib/redis/namespace_extensions' require_relative '../lib/paperclip/url_generator_extensions' require_relative '../lib/paperclip/attachment_extensions' diff --git a/config/initializers/inflections.rb b/config/initializers/inflections.rb index ebb7541eb..9bc9a54b2 100644 --- a/config/initializers/inflections.rb +++ b/config/initializers/inflections.rb @@ -20,6 +20,10 @@ ActiveSupport::Inflector.inflections(:en) do |inflect| inflect.acronym 'JsonLd' inflect.acronym 'NodeInfo' inflect.acronym 'Ed25519' + inflect.acronym 'TOC' + inflect.acronym 'RSS' + inflect.acronym 'REST' + inflect.acronym 'URL' inflect.singular 'data', 'data' end diff --git a/db/migrate/20160223165723_add_url_to_statuses.rb b/db/migrate/20160223165723_add_url_to_statuses.rb index 80f4b3289..fee7f9c59 100644 --- a/db/migrate/20160223165723_add_url_to_statuses.rb +++ b/db/migrate/20160223165723_add_url_to_statuses.rb @@ -1,4 +1,4 @@ -class AddUrlToStatuses < ActiveRecord::Migration[4.2] +class AddURLToStatuses < ActiveRecord::Migration[4.2] def change add_column :statuses, :url, :string, null: true, default: nil end diff --git a/db/migrate/20160223165855_add_url_to_accounts.rb b/db/migrate/20160223165855_add_url_to_accounts.rb index c81b1c64f..a4db8814a 100644 --- a/db/migrate/20160223165855_add_url_to_accounts.rb +++ b/db/migrate/20160223165855_add_url_to_accounts.rb @@ -1,4 +1,4 @@ -class AddUrlToAccounts < ActiveRecord::Migration[4.2] +class AddURLToAccounts < ActiveRecord::Migration[4.2] def change add_column :accounts, :url, :string, null: true, default: nil end diff --git a/db/migrate/20160322193748_add_avatar_remote_url_to_accounts.rb b/db/migrate/20160322193748_add_avatar_remote_url_to_accounts.rb index f9c213d9b..0792863a3 100644 --- a/db/migrate/20160322193748_add_avatar_remote_url_to_accounts.rb +++ b/db/migrate/20160322193748_add_avatar_remote_url_to_accounts.rb @@ -1,4 +1,4 @@ -class AddAvatarRemoteUrlToAccounts < ActiveRecord::Migration[4.2] +class AddAvatarRemoteURLToAccounts < ActiveRecord::Migration[4.2] def change add_column :accounts, :avatar_remote_url, :string, null: true, default: nil end diff --git a/db/migrate/20170318214217_add_header_remote_url_to_accounts.rb b/db/migrate/20170318214217_add_header_remote_url_to_accounts.rb index 0ba38d3e0..20c965988 100644 --- a/db/migrate/20170318214217_add_header_remote_url_to_accounts.rb +++ b/db/migrate/20170318214217_add_header_remote_url_to_accounts.rb @@ -1,4 +1,4 @@ -class AddHeaderRemoteUrlToAccounts < ActiveRecord::Migration[5.0] +class AddHeaderRemoteURLToAccounts < ActiveRecord::Migration[5.0] def change add_column :accounts, :header_remote_url, :string, null: false, default: '' end diff --git a/db/migrate/20171130000000_add_embed_url_to_preview_cards.rb b/db/migrate/20171130000000_add_embed_url_to_preview_cards.rb index d19c0091b..8fcabef9f 100644 --- a/db/migrate/20171130000000_add_embed_url_to_preview_cards.rb +++ b/db/migrate/20171130000000_add_embed_url_to_preview_cards.rb @@ -1,6 +1,6 @@ require Rails.root.join('lib', 'mastodon', 'migration_helpers') -class AddEmbedUrlToPreviewCards < ActiveRecord::Migration[5.1] +class AddEmbedURLToPreviewCards < ActiveRecord::Migration[5.1] include Mastodon::MigrationHelpers disable_ddl_transaction! diff --git a/db/migrate/20180304013859_add_featured_collection_url_to_accounts.rb b/db/migrate/20180304013859_add_featured_collection_url_to_accounts.rb index e0b8ed5cc..1964b5121 100644 --- a/db/migrate/20180304013859_add_featured_collection_url_to_accounts.rb +++ b/db/migrate/20180304013859_add_featured_collection_url_to_accounts.rb @@ -1,4 +1,4 @@ -class AddFeaturedCollectionUrlToAccounts < ActiveRecord::Migration[5.1] +class AddFeaturedCollectionURLToAccounts < ActiveRecord::Migration[5.1] def change add_column :accounts, :featured_collection_url, :string end diff --git a/db/migrate/20200529214050_add_devices_url_to_accounts.rb b/db/migrate/20200529214050_add_devices_url_to_accounts.rb index 564877e5d..1323f8df7 100644 --- a/db/migrate/20200529214050_add_devices_url_to_accounts.rb +++ b/db/migrate/20200529214050_add_devices_url_to_accounts.rb @@ -1,4 +1,4 @@ -class AddDevicesUrlToAccounts < ActiveRecord::Migration[5.2] +class AddDevicesURLToAccounts < ActiveRecord::Migration[5.2] def change add_column :accounts, :devices_url, :string end diff --git a/lib/exceptions.rb b/lib/exceptions.rb new file mode 100644 index 000000000..7c8e77871 --- /dev/null +++ b/lib/exceptions.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Mastodon + class Error < StandardError; end + class NotPermittedError < Error; end + class ValidationError < Error; end + class HostValidationError < ValidationError; end + class LengthValidationError < ValidationError; end + class DimensionsValidationError < ValidationError; end + class StreamValidationError < ValidationError; end + class RaceConditionError < Error; end + class RateLimitExceededError < Error; end + + class UnexpectedResponseError < Error + def initialize(response = nil) + if response.respond_to? :uri + super("#{response.uri} returned code #{response.code}") + else + super + end + end + end +end diff --git a/lib/sanitize_ext/sanitize_config.rb b/lib/sanitize_ext/sanitize_config.rb new file mode 100644 index 000000000..a2e1d9d01 --- /dev/null +++ b/lib/sanitize_ext/sanitize_config.rb @@ -0,0 +1,119 @@ +# frozen_string_literal: true + +class Sanitize + module Config + HTTP_PROTOCOLS = %w( + http + https + ).freeze + + LINK_PROTOCOLS = %w( + http + https + dat + dweb + ipfs + ipns + ssb + gopher + xmpp + magnet + gemini + ).freeze + + CLASS_WHITELIST_TRANSFORMER = lambda do |env| + node = env[:node] + class_list = node['class']&.split(/[\t\n\f\r ]/) + + return unless class_list + + class_list.keep_if do |e| + next true if /^(h|p|u|dt|e)-/.match?(e) # microformats classes + next true if /^(mention|hashtag)$/.match?(e) # semantic classes + next true if /^(ellipsis|invisible)$/.match?(e) # link formatting classes + end + + node['class'] = class_list.join(' ') + end + + UNSUPPORTED_HREF_TRANSFORMER = lambda do |env| + return unless env[:node_name] == 'a' + + current_node = env[:node] + + scheme = begin + if current_node['href'] =~ Sanitize::REGEX_PROTOCOL + Regexp.last_match(1).downcase + else + :relative + end + end + + current_node.replace(current_node.text) unless LINK_PROTOCOLS.include?(scheme) + end + + UNSUPPORTED_ELEMENTS_TRANSFORMER = lambda do |env| + return unless %w(h1 h2 h3 h4 h5 h6 blockquote pre ul ol li).include?(env[:node_name]) + + current_node = env[:node] + + case env[:node_name] + when 'li' + current_node.traverse do |node| + next unless %w(p ul ol li).include?(node.name) + + node.add_next_sibling('
') if node.next_sibling + node.replace(node.children) unless node.text? + end + else + current_node.name = 'p' + end + end + + MASTODON_STRICT ||= freeze_config( + elements: %w(p br span a), + + attributes: { + 'a' => %w(href rel class), + 'span' => %w(class), + }, + + add_attributes: { + 'a' => { + 'rel' => 'nofollow noopener noreferrer', + 'target' => '_blank', + }, + }, + + protocols: {}, + + transformers: [ + CLASS_WHITELIST_TRANSFORMER, + UNSUPPORTED_ELEMENTS_TRANSFORMER, + UNSUPPORTED_HREF_TRANSFORMER, + ] + ) + + MASTODON_OEMBED ||= freeze_config merge( + RELAXED, + elements: RELAXED[:elements] + %w(audio embed iframe source video), + + attributes: merge( + RELAXED[:attributes], + 'audio' => %w(controls), + 'embed' => %w(height src type width), + 'iframe' => %w(allowfullscreen frameborder height scrolling src width), + 'source' => %w(src type), + 'video' => %w(controls height loop width), + 'div' => [:data] + ), + + protocols: merge( + RELAXED[:protocols], + 'embed' => { 'src' => HTTP_PROTOCOLS }, + 'iframe' => { 'src' => HTTP_PROTOCOLS }, + 'source' => { 'src' => HTTP_PROTOCOLS } + ) + ) + end +end diff --git a/spec/lib/sanitize_config_spec.rb b/spec/lib/sanitize_config_spec.rb index d66302e64..747d81158 100644 --- a/spec/lib/sanitize_config_spec.rb +++ b/spec/lib/sanitize_config_spec.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true require 'rails_helper' -require Rails.root.join('app', 'lib', 'sanitize_config.rb') describe Sanitize::Config do describe '::MASTODON_STRICT' do diff --git a/spec/validators/url_validator_spec.rb b/spec/validators/url_validator_spec.rb index e8d0e6494..a44878a44 100644 --- a/spec/validators/url_validator_spec.rb +++ b/spec/validators/url_validator_spec.rb @@ -2,7 +2,7 @@ require 'rails_helper' -RSpec.describe UrlValidator, type: :validator do +RSpec.describe URLValidator, type: :validator do describe '#validate_each' do before do allow(validator).to receive(:compliant?).with(value) { compliant } -- cgit