From cefa526c6d3a45df2d0fcb7643ced828e2e87dea Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sat, 26 Mar 2022 02:53:34 +0100 Subject: Refactor formatter (#17828) * Refactor formatter * Move custom emoji pre-rendering logic to view helpers * Move more methods out of Formatter * Fix code style issues * Remove Formatter * Add inline poll options to RSS feeds * Remove unused helper method * Fix code style issues * Various fixes and improvements * Fix test --- app/chewy/statuses_index.rb | 2 +- app/controllers/api/web/embeds_controller.rb | 2 +- app/helpers/accounts_helper.rb | 6 +- app/helpers/admin/trends/statuses_helper.rb | 5 +- app/helpers/application_helper.rb | 4 + app/helpers/formatting_helper.rb | 19 ++ app/helpers/routing_helper.rb | 3 +- app/helpers/statuses_helper.rb | 14 -- app/lib/activitypub/activity/create.rb | 4 +- app/lib/emoji_formatter.rb | 98 ++++++++ app/lib/extractor.rb | 82 +++++-- app/lib/feed_manager.rb | 3 +- app/lib/formatter.rb | 294 ------------------------ app/lib/html_aware_formatter.rb | 38 +++ app/lib/plain_text_formatter.rb | 30 +++ app/lib/rss/serializer.rb | 23 +- app/lib/text_formatter.rb | 158 +++++++++++++ app/mailers/application_mailer.rb | 1 + app/serializers/activitypub/actor_serializer.rb | 7 +- app/serializers/activitypub/note_serializer.rb | 6 +- app/serializers/rest/account_serializer.rb | 7 +- app/serializers/rest/announcement_serializer.rb | 4 +- app/serializers/rest/status_edit_serializer.rb | 4 +- app/serializers/rest/status_serializer.rb | 4 +- app/services/fetch_link_card_service.rb | 2 +- app/views/accounts/_bio.html.haml | 6 +- app/views/admin/accounts/show.html.haml | 6 +- app/views/admin/reports/_status.html.haml | 6 +- app/views/admin/reports/show.html.haml | 2 +- app/views/directories/index.html.haml | 2 +- app/views/disputes/strikes/show.html.haml | 2 +- app/views/notification_mailer/_status.html.haml | 4 +- app/views/notification_mailer/_status.text.erb | 2 +- app/views/notification_mailer/digest.text.erb | 2 +- app/views/statuses/_detailed_status.html.haml | 5 +- app/views/statuses/_poll.html.haml | 4 +- app/views/statuses/_simple_status.html.haml | 5 +- app/views/user_mailer/warning.html.haml | 2 +- 38 files changed, 496 insertions(+), 372 deletions(-) create mode 100644 app/helpers/formatting_helper.rb create mode 100644 app/lib/emoji_formatter.rb delete mode 100644 app/lib/formatter.rb create mode 100644 app/lib/html_aware_formatter.rb create mode 100644 app/lib/plain_text_formatter.rb create mode 100644 app/lib/text_formatter.rb (limited to 'app') diff --git a/app/chewy/statuses_index.rb b/app/chewy/statuses_index.rb index 65cbb6fcd..d119f7cac 100644 --- a/app/chewy/statuses_index.rb +++ b/app/chewy/statuses_index.rb @@ -57,7 +57,7 @@ class StatusesIndex < Chewy::Index field :id, type: 'long' field :account_id, type: 'long' - field :text, type: 'text', value: ->(status) { [status.spoiler_text, Formatter.instance.plaintext(status)].concat(status.ordered_media_attachments.map(&:description)).concat(status.preloadable_poll ? status.preloadable_poll.options : []).join("\n\n") } do + field :text, type: 'text', value: ->(status) { [status.spoiler_text, PlainTextFormatter.new(status.text, status.local?).to_s].concat(status.ordered_media_attachments.map(&:description)).concat(status.preloadable_poll ? status.preloadable_poll.options : []).join("\n\n") } do field :stemmed, type: 'text', analyzer: 'content' end diff --git a/app/controllers/api/web/embeds_controller.rb b/app/controllers/api/web/embeds_controller.rb index 741ba910f..58f6345e6 100644 --- a/app/controllers/api/web/embeds_controller.rb +++ b/app/controllers/api/web/embeds_controller.rb @@ -15,7 +15,7 @@ class Api::Web::EmbedsController < Api::Web::BaseController return not_found if oembed.nil? begin - oembed[:html] = Formatter.instance.sanitize(oembed[:html], Sanitize::Config::MASTODON_OEMBED) + oembed[:html] = Sanitize.fragment(oembed[:html], Sanitize::Config::MASTODON_OEMBED) rescue ArgumentError return not_found end diff --git a/app/helpers/accounts_helper.rb b/app/helpers/accounts_helper.rb index a33961724..557f60f26 100644 --- a/app/helpers/accounts_helper.rb +++ b/app/helpers/accounts_helper.rb @@ -2,10 +2,12 @@ module AccountsHelper def display_name(account, **options) + str = account.display_name.presence || account.username + if options[:custom_emojify] - Formatter.instance.format_display_name(account, **options) + prerender_custom_emojis(h(str), account.emojis) else - account.display_name.presence || account.username + str end end diff --git a/app/helpers/admin/trends/statuses_helper.rb b/app/helpers/admin/trends/statuses_helper.rb index d16e3dd12..214c1e2a6 100644 --- a/app/helpers/admin/trends/statuses_helper.rb +++ b/app/helpers/admin/trends/statuses_helper.rb @@ -12,9 +12,6 @@ module Admin::Trends::StatusesHelper return '' if text.blank? - html = Formatter.instance.send(:encode, text) - html = Formatter.instance.send(:encode_custom_emojis, html, status.emojis, prefers_autoplay?) - - html.html_safe # rubocop:disable Rails/OutputSafety + prerender_custom_emojis(h(text), status.emojis) end end diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb index e997570b5..651a98a85 100644 --- a/app/helpers/application_helper.rb +++ b/app/helpers/application_helper.rb @@ -239,4 +239,8 @@ module ApplicationHelper end end.values end + + def prerender_custom_emojis(html, custom_emojis) + EmojiFormatter.new(html, custom_emojis, animate: prefers_autoplay?).to_s + end end diff --git a/app/helpers/formatting_helper.rb b/app/helpers/formatting_helper.rb new file mode 100644 index 000000000..66e9e1e91 --- /dev/null +++ b/app/helpers/formatting_helper.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module FormattingHelper + def html_aware_format(text, local, options = {}) + HtmlAwareFormatter.new(text, local, options).to_s + end + + def linkify(text, options = {}) + TextFormatter.new(text, options).to_s + end + + def extract_plain_text(text, local) + PlainTextFormatter.new(text, local).to_s + end + + def status_content_format(status) + html_aware_format(status.text, status.local?, preloaded_accounts: [status.account] + (status.respond_to?(:active_mentions) ? status.active_mentions.map(&:account) : [])) + end +end diff --git a/app/helpers/routing_helper.rb b/app/helpers/routing_helper.rb index fb24a1b28..f95f46a56 100644 --- a/app/helpers/routing_helper.rb +++ b/app/helpers/routing_helper.rb @@ -2,6 +2,7 @@ module RoutingHelper extend ActiveSupport::Concern + include Rails.application.routes.url_helpers include ActionView::Helpers::AssetTagHelper include Webpacker::Helper @@ -22,8 +23,6 @@ module RoutingHelper full_asset_url(asset_pack_path(source, **options)) end - private - def use_storage? Rails.configuration.x.use_s3 || Rails.configuration.x.use_swift end diff --git a/app/helpers/statuses_helper.rb b/app/helpers/statuses_helper.rb index d328f89b7..e92b4c839 100644 --- a/app/helpers/statuses_helper.rb +++ b/app/helpers/statuses_helper.rb @@ -113,20 +113,6 @@ module StatusesHelper end end - private - - def simplified_text(text) - text.dup.tap do |new_text| - URI.extract(new_text).each do |url| - new_text.gsub!(url, '') - end - - new_text.gsub!(Account::MENTION_RE, '') - new_text.gsub!(Tag::HASHTAG_RE, '') - new_text.gsub!(/\s+/, '') - end - end - def embedded_view? params[:controller] == EMBEDDED_CONTROLLER && params[:action] == EMBEDDED_ACTION end diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index ea8d146d4..f4f98e29c 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class ActivityPub::Activity::Create < ActivityPub::Activity + include FormattingHelper + def perform dereference_object! @@ -367,7 +369,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity end def converted_text - Formatter.instance.linkify([@status_parser.title.presence, @status_parser.spoiler_text.presence, @status_parser.url || @status_parser.uri].compact.join("\n\n")) + linkify([@status_parser.title.presence, @status_parser.spoiler_text.presence, @status_parser.url || @status_parser.uri].compact.join("\n\n")) end def unsupported_media_type?(mime_type) diff --git a/app/lib/emoji_formatter.rb b/app/lib/emoji_formatter.rb new file mode 100644 index 000000000..f808f3a22 --- /dev/null +++ b/app/lib/emoji_formatter.rb @@ -0,0 +1,98 @@ +# frozen_string_literal: true + +class EmojiFormatter + include RoutingHelper + + DISALLOWED_BOUNDING_REGEX = /[[:alnum:]:]/.freeze + + attr_reader :html, :custom_emojis, :options + + # @param [ActiveSupport::SafeBuffer] html + # @param [Array] custom_emojis + # @param [Hash] options + # @option options [Boolean] :animate + def initialize(html, custom_emojis, options = {}) + raise ArgumentError unless html.html_safe? + + @html = html + @custom_emojis = custom_emojis + @options = options + end + + def to_s + return html if custom_emojis.empty? || html.blank? + + i = -1 + tag_open_index = nil + inside_shortname = false + shortname_start_index = -1 + invisible_depth = 0 + last_index = 0 + result = ''.dup + + while i + 1 < html.size + i += 1 + + if invisible_depth.zero? && inside_shortname && html[i] == ':' + inside_shortname = false + shortcode = html[shortname_start_index + 1..i - 1] + char_after = html[i + 1] + + next unless (char_after.nil? || !DISALLOWED_BOUNDING_REGEX.match?(char_after)) && (emoji = emoji_map[shortcode]) + + result << html[last_index..shortname_start_index - 1] if shortname_start_index.positive? + result << image_for_emoji(shortcode, emoji) + last_index = i + 1 + elsif tag_open_index && html[i] == '>' + tag = html[tag_open_index..i] + tag_open_index = nil + + if invisible_depth.positive? + invisible_depth += count_tag_nesting(tag) + elsif tag == '