diff options
Diffstat (limited to 'app/lib')
| -rw-r--r-- | app/lib/activitypub/activity/create.rb | 4 | ||||
| -rw-r--r-- | app/lib/activitypub/parser/media_attachment_parser.rb | 4 | ||||
| -rw-r--r-- | app/lib/admin/system_check.rb | 1 | ||||
| -rw-r--r-- | app/lib/admin/system_check/elasticsearch_check.rb | 39 | ||||
| -rw-r--r-- | app/lib/advanced_text_formatter.rb | 131 | ||||
| -rw-r--r-- | app/lib/emoji_formatter.rb | 98 | ||||
| -rw-r--r-- | app/lib/extractor.rb | 88 | ||||
| -rw-r--r-- | app/lib/feed_manager.rb | 3 | ||||
| -rw-r--r-- | app/lib/formatter.rb | 382 | ||||
| -rw-r--r-- | app/lib/html_aware_formatter.rb | 42 | ||||
| -rw-r--r-- | app/lib/link_details_extractor.rb | 2 | ||||
| -rw-r--r-- | app/lib/plain_text_formatter.rb | 30 | ||||
| -rw-r--r-- | app/lib/rss/serializer.rb | 23 | ||||
| -rw-r--r-- | app/lib/text_formatter.rb | 158 |
14 files changed, 600 insertions, 405 deletions
diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index cf31b6ff6..1ac509f18 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class ActivityPub::Activity::Create < ActivityPub::Activity + include FormattingHelper + def perform dereference_object! @@ -367,7 +369,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity end def converted_text - Formatter.instance.linkify([@status_parser.title.presence, @status_parser.spoiler_text.presence, @status_parser.url || @status_parser.uri].compact.join("\n\n")) + linkify([@status_parser.title.presence, @status_parser.spoiler_text.presence, @status_parser.url || @status_parser.uri].compact.join("\n\n")) end def unsupported_media_type?(mime_type) diff --git a/app/lib/activitypub/parser/media_attachment_parser.rb b/app/lib/activitypub/parser/media_attachment_parser.rb index 1798e58a4..30bea1f0e 100644 --- a/app/lib/activitypub/parser/media_attachment_parser.rb +++ b/app/lib/activitypub/parser/media_attachment_parser.rb @@ -27,7 +27,9 @@ class ActivityPub::Parser::MediaAttachmentParser end def description - @json['summary'].presence || @json['name'].presence + str = @json['summary'].presence || @json['name'].presence + str = str.strip[0...MediaAttachment::MAX_DESCRIPTION_LENGTH] if str.present? + str end def focus diff --git a/app/lib/admin/system_check.rb b/app/lib/admin/system_check.rb index afb20cb47..877a42ef6 100644 --- a/app/lib/admin/system_check.rb +++ b/app/lib/admin/system_check.rb @@ -5,6 +5,7 @@ class Admin::SystemCheck Admin::SystemCheck::DatabaseSchemaCheck, Admin::SystemCheck::SidekiqProcessCheck, Admin::SystemCheck::RulesCheck, + Admin::SystemCheck::ElasticsearchCheck, ].freeze def self.perform diff --git a/app/lib/admin/system_check/elasticsearch_check.rb b/app/lib/admin/system_check/elasticsearch_check.rb new file mode 100644 index 000000000..1b48a5415 --- /dev/null +++ b/app/lib/admin/system_check/elasticsearch_check.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +class Admin::SystemCheck::ElasticsearchCheck < Admin::SystemCheck::BaseCheck + def pass? + return true unless Chewy.enabled? + + running_version.present? && compatible_version? + end + + def message + if running_version.present? + Admin::SystemCheck::Message.new(:elasticsearch_version_check, I18n.t('admin.system_checks.elasticsearch_version_check.version_comparison', running_version: running_version, required_version: required_version)) + else + Admin::SystemCheck::Message.new(:elasticsearch_running_check) + end + end + + private + + def running_version + @running_version ||= begin + Chewy.client.info['version']['number'] + rescue Faraday::ConnectionFailed + nil + end + end + + def required_version + '7.x' + end + + def compatible_version? + Gem::Version.new(running_version) >= Gem::Version.new(required_version) + end + + def missing_queues + @missing_queues ||= Sidekiq::ProcessSet.new.reduce(SIDEKIQ_QUEUES) { |queues, process| queues - process['queues'] } + end +end diff --git a/app/lib/advanced_text_formatter.rb b/app/lib/advanced_text_formatter.rb new file mode 100644 index 000000000..728400819 --- /dev/null +++ b/app/lib/advanced_text_formatter.rb @@ -0,0 +1,131 @@ +# frozen_string_literal: true + +class AdvancedTextFormatter < TextFormatter + class HTMLRenderer < Redcarpet::Render::HTML + def initialize(options, &block) + super(options) + @format_link = block + end + + def block_code(code, _language) + <<~HTML.squish + <pre><code>#{ERB::Util.h(code).gsub("\n", '<br/>')}</code></pre> + HTML + end + + def autolink(link, link_type) + return link if link_type == :email + @format_link.call(link) + end + end + + # @param [String] text + # @param [Hash] options + # @option options [Boolean] :multiline + # @option options [Boolean] :with_domains + # @option options [Boolean] :with_rel_me + # @option options [Array<Account>] :preloaded_accounts + # @option options [String] :content_type + def initialize(text, options = {}) + content_type = options.delete(:content_type) + super(text, options) + + @text = format_markdown(text) if content_type == 'text/markdown' + end + + # Differs from TextFormatter by not messing with newline after parsing + def to_s + return ''.html_safe if text.blank? + + html = rewrite do |entity| + if entity[:url] + link_to_url(entity) + elsif entity[:hashtag] + link_to_hashtag(entity) + elsif entity[:screen_name] + link_to_mention(entity) + end + end + + html.html_safe # rubocop:disable Rails/OutputSafety + end + + # Differs from `TextFormatter` by skipping HTML tags and entities + def entities + @entities ||= begin + gaps = [] + total_offset = 0 + + escaped = text.gsub(/<[^>]*>|&#[0-9]+;/) do |match| + total_offset += match.length - 1 + end_offset = Regexp.last_match.end(0) + gaps << [end_offset - total_offset, total_offset] + ' ' + end + + Extractor.extract_entities_with_indices(escaped, extract_url_without_protocol: false).map do |entity| + start_pos, end_pos = entity[:indices] + offset_idx = gaps.rindex { |gap| gap.first <= start_pos } + offset = offset_idx.nil? ? 0 : gaps[offset_idx].last + entity.merge(indices: [start_pos + offset, end_pos + offset]) + end + end + end + + private + + # Differs from `TextFormatter` in that it keeps HTML; but it sanitizes at the end to remain safe + def rewrite + entities.sort_by! do |entity| + entity[:indices].first + end + + result = ''.dup + + last_index = entities.reduce(0) do |index, entity| + indices = entity[:indices] + result << text[index...indices.first] + result << yield(entity) + indices.last + end + + result << text[last_index..-1] + + Sanitize.fragment(result, Sanitize::Config::MASTODON_OUTGOING) + end + + def format_markdown(html) + html = markdown_formatter.render(html) + html.delete("\r").delete("\n") + end + + def markdown_formatter + extensions = { + autolink: true, + no_intra_emphasis: true, + fenced_code_blocks: true, + disable_indented_code_blocks: true, + strikethrough: true, + lax_spacing: true, + space_after_headers: true, + superscript: true, + underline: true, + highlight: true, + footnotes: false, + } + + renderer = HTMLRenderer.new({ + filter_html: false, + escape_html: false, + no_images: true, + no_styles: true, + safe_links_only: true, + hard_wrap: true, + link_attributes: { target: '_blank', rel: 'nofollow noopener' }, + }) do |url| + link_to_url({ url: url }) + end + + Redcarpet::Markdown.new(renderer, extensions) + end +end diff --git a/app/lib/emoji_formatter.rb b/app/lib/emoji_formatter.rb new file mode 100644 index 000000000..f808f3a22 --- /dev/null +++ b/app/lib/emoji_formatter.rb @@ -0,0 +1,98 @@ +# frozen_string_literal: true + +class EmojiFormatter + include RoutingHelper + + DISALLOWED_BOUNDING_REGEX = /[[:alnum:]:]/.freeze + + attr_reader :html, :custom_emojis, :options + + # @param [ActiveSupport::SafeBuffer] html + # @param [Array<CustomEmoji>] custom_emojis + # @param [Hash] options + # @option options [Boolean] :animate + def initialize(html, custom_emojis, options = {}) + raise ArgumentError unless html.html_safe? + + @html = html + @custom_emojis = custom_emojis + @options = options + end + + def to_s + return html if custom_emojis.empty? || html.blank? + + i = -1 + tag_open_index = nil + inside_shortname = false + shortname_start_index = -1 + invisible_depth = 0 + last_index = 0 + result = ''.dup + + while i + 1 < html.size + i += 1 + + if invisible_depth.zero? && inside_shortname && html[i] == ':' + inside_shortname = false + shortcode = html[shortname_start_index + 1..i - 1] + char_after = html[i + 1] + + next unless (char_after.nil? || !DISALLOWED_BOUNDING_REGEX.match?(char_after)) && (emoji = emoji_map[shortcode]) + + result << html[last_index..shortname_start_index - 1] if shortname_start_index.positive? + result << image_for_emoji(shortcode, emoji) + last_index = i + 1 + elsif tag_open_index && html[i] == '>' + tag = html[tag_open_index..i] + tag_open_index = nil + + if invisible_depth.positive? + invisible_depth += count_tag_nesting(tag) + elsif tag == '<span class="invisible">' + invisible_depth = 1 + end + elsif html[i] == '<' + tag_open_index = i + inside_shortname = false + elsif !tag_open_index && html[i] == ':' && (i.zero? || !DISALLOWED_BOUNDING_REGEX.match?(html[i - 1])) + inside_shortname = true + shortname_start_index = i + end + end + + result << html[last_index..-1] + + result.html_safe # rubocop:disable Rails/OutputSafety + end + + private + + def emoji_map + @emoji_map ||= custom_emojis.each_with_object({}) { |e, h| h[e.shortcode] = [full_asset_url(e.image.url), full_asset_url(e.image.url(:static))] } + end + + def count_tag_nesting(tag) + if tag[1] == '/' + -1 + elsif tag[-2] == '/' + 0 + else + 1 + end + end + + def image_for_emoji(shortcode, emoji) + original_url, static_url = emoji + + if animate? + image_tag(original_url, draggable: false, class: 'emojione', alt: ":#{shortcode}:", title: ":#{shortcode}:") + else + image_tag(original_url, draggable: false, class: 'emojione custom-emoji', alt: ":#{shortcode}:", title: ":#{shortcode}:", data: { original: original_url, static: static_url }) + end + end + + def animate? + @options[:animate] + end +end diff --git a/app/lib/extractor.rb b/app/lib/extractor.rb index 8020aa916..aea60dae5 100644 --- a/app/lib/extractor.rb +++ b/app/lib/extractor.rb @@ -1,22 +1,44 @@ # frozen_string_literal: true module Extractor + MAX_DOMAIN_LENGTH = 253 + extend Twitter::TwitterText::Extractor module_function - # :yields: username, list_slug, start, end + def extract_entities_with_indices(text, options = {}, &block) + entities = begin + extract_urls_with_indices(text, options) + + extract_hashtags_with_indices(text, check_url_overlap: false) + + extract_mentions_or_lists_with_indices(text) + + extract_extra_uris_with_indices(text) + end + + return [] if entities.empty? + + entities = remove_overlapping_entities(entities) + entities.each(&block) if block_given? + entities + end + def extract_mentions_or_lists_with_indices(text) - return [] unless Twitter::TwitterText::Regex[:at_signs].match?(text) + return [] unless text && Twitter::TwitterText::Regex[:at_signs].match?(text) possible_entries = [] - text.to_s.scan(Account::MENTION_RE) do |screen_name, _| + text.scan(Account::MENTION_RE) do |screen_name, _| match_data = $LAST_MATCH_INFO - after = $' + after = $' + unless Twitter::TwitterText::Regex[:end_mention_match].match?(after) + _, domain = screen_name.split('@') + + next if domain.present? && domain.length > MAX_DOMAIN_LENGTH + start_position = match_data.char_begin(1) - 1 - end_position = match_data.char_end(1) + end_position = match_data.char_end(1) + possible_entries << { screen_name: screen_name, indices: [start_position, end_position], @@ -29,36 +51,70 @@ module Extractor yield mention[:screen_name], mention[:indices].first, mention[:indices].last end end + possible_entries end - def extract_hashtags_with_indices(text, **) - return [] unless /#/.match?(text) + def extract_hashtags_with_indices(text, _options = {}) + return [] unless text&.index('#') + + possible_entries = [] - tags = [] text.scan(Tag::HASHTAG_RE) do |hash_text, _| - match_data = $LAST_MATCH_INFO + match_data = $LAST_MATCH_INFO start_position = match_data.char_begin(1) - 1 - end_position = match_data.char_end(1) - after = $' + end_position = match_data.char_end(1) + after = $' + if %r{\A://}.match?(after) hash_text.match(/(.+)(https?\Z)/) do |matched| - hash_text = matched[1] + hash_text = matched[1] end_position -= matched[2].codepoint_length end end - tags << { + possible_entries << { hashtag: hash_text, indices: [start_position, end_position], } end - tags.each { |tag| yield tag[:hashtag], tag[:indices].first, tag[:indices].last } if block_given? - tags + if block_given? + possible_entries.each do |tag| + yield tag[:hashtag], tag[:indices].first, tag[:indices].last + end + end + + possible_entries end def extract_cashtags_with_indices(_text) - [] # always returns empty array + [] + end + + def extract_extra_uris_with_indices(text) + return [] unless text&.index(':') + + possible_entries = [] + + text.scan(Twitter::TwitterText::Regex[:valid_extended_uri]) do + valid_uri_match_data = $LAST_MATCH_INFO + + start_position = valid_uri_match_data.char_begin(3) + end_position = valid_uri_match_data.char_end(3) + + possible_entries << { + url: valid_uri_match_data[3], + indices: [start_position, end_position], + } + end + + if block_given? + possible_entries.each do |url| + yield url[:url], url[:indices].first, url[:indices].last + end + end + + possible_entries end end diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb index efc9da34b..6994f00ae 100644 --- a/app/lib/feed_manager.rb +++ b/app/lib/feed_manager.rb @@ -5,6 +5,7 @@ require 'singleton' class FeedManager include Singleton include Redisable + include FormattingHelper # Maximum number of items stored in a single feed MAX_ITEMS = 400 @@ -503,7 +504,7 @@ class FeedManager status = status.reblog if status.reblog? combined_text = [ - Formatter.instance.plaintext(status), + extract_status_plain_text(status), status.spoiler_text, status.preloadable_poll ? status.preloadable_poll.options.join("\n\n") : nil, status.ordered_media_attachments.map(&:description).join("\n\n"), diff --git a/app/lib/formatter.rb b/app/lib/formatter.rb deleted file mode 100644 index dfa493ed5..000000000 --- a/app/lib/formatter.rb +++ /dev/null @@ -1,382 +0,0 @@ -# frozen_string_literal: true - -require 'singleton' - -class HTMLRenderer < Redcarpet::Render::HTML - def block_code(code, language) - "<pre><code>#{encode(code).gsub("\n", "<br/>")}</code></pre>" - end - - def autolink(link, link_type) - return link if link_type == :email - Formatter.instance.link_url(link) - rescue Addressable::URI::InvalidURIError, IDN::Idna::IdnaError - encode(link) - end - - private - - def html_entities - @html_entities ||= HTMLEntities.new - end - - def encode(html) - html_entities.encode(html) - end -end - -class Formatter - include Singleton - include RoutingHelper - - include ActionView::Helpers::TextHelper - - def format(status, **options) - if status.respond_to?(:reblog?) && status.reblog? - prepend_reblog = status.reblog.account.acct - status = status.proper - else - prepend_reblog = false - end - - raw_content = status.text - - if options[:inline_poll_options] && status.preloadable_poll - raw_content = raw_content + "\n\n" + status.preloadable_poll.options.map { |title| "[ ] #{title}" }.join("\n") - end - - return '' if raw_content.blank? - - unless status.local? - html = reformat(raw_content) - html = encode_custom_emojis(html, status.emojis, options[:autoplay]) if options[:custom_emojify] - return html.html_safe # rubocop:disable Rails/OutputSafety - end - - linkable_accounts = status.respond_to?(:active_mentions) ? status.active_mentions.map(&:account) : [] - linkable_accounts << status.account - - html = raw_content - html = "RT @#{prepend_reblog} #{html}" if prepend_reblog - html = format_markdown(html) if status.content_type == 'text/markdown' - html = encode_and_link_urls(html, linkable_accounts, keep_html: %w(text/markdown text/html).include?(status.content_type)) - html = reformat(html, true) if %w(text/markdown text/html).include?(status.content_type) - html = encode_custom_emojis(html, status.emojis, options[:autoplay]) if options[:custom_emojify] - - unless %w(text/markdown text/html).include?(status.content_type) - html = simple_format(html, {}, sanitize: false) - html = html.delete("\n") - end - - html.html_safe # rubocop:disable Rails/OutputSafety - end - - def format_markdown(html) - html = markdown_formatter.render(html) - html.delete("\r").delete("\n") - end - - def reformat(html, outgoing = false) - sanitize(html, Sanitize::Config::MASTODON_STRICT.merge(outgoing: outgoing)) - rescue ArgumentError - '' - end - - def plaintext(status) - return status.text if status.local? - - text = status.text.gsub(/(<br \/>|<br>|<\/p>)+/) { |match| "#{match}\n" } - strip_tags(text) - end - - def simplified_format(account, **options) - return '' if account.note.blank? - - html = account.local? ? linkify(account.note) : reformat(account.note) - html = encode_custom_emojis(html, account.emojis, options[:autoplay]) if options[:custom_emojify] - html.html_safe # rubocop:disable Rails/OutputSafety - end - - def sanitize(html, config) - Sanitize.fragment(html, config) - end - - def format_spoiler(status, **options) - html = encode(status.spoiler_text) - html = encode_custom_emojis(html, status.emojis, options[:autoplay]) - html.html_safe # rubocop:disable Rails/OutputSafety - end - - def format_poll_option(status, option, **options) - html = encode(option.title) - html = encode_custom_emojis(html, status.emojis, options[:autoplay]) - html.html_safe # rubocop:disable Rails/OutputSafety - end - - def format_display_name(account, **options) - html = encode(account.display_name.presence || account.username) - html = encode_custom_emojis(html, account.emojis, options[:autoplay]) if options[:custom_emojify] - html.html_safe # rubocop:disable Rails/OutputSafety - end - - def format_field(account, str, **options) - html = account.local? ? encode_and_link_urls(str, me: true, with_domain: true) : reformat(str) - html = encode_custom_emojis(html, account.emojis, options[:autoplay]) if options[:custom_emojify] - html.html_safe # rubocop:disable Rails/OutputSafety - end - - def linkify(text) - html = encode_and_link_urls(text) - html = simple_format(html, {}, sanitize: false) - html = html.delete("\n") - - html.html_safe # rubocop:disable Rails/OutputSafety - end - - def link_url(url) - "<a href=\"#{encode(url)}\" target=\"blank\" rel=\"nofollow noopener noreferrer\">#{link_html(url)}</a>" - end - - private - - def markdown_formatter - extensions = { - autolink: true, - no_intra_emphasis: true, - fenced_code_blocks: true, - disable_indented_code_blocks: true, - strikethrough: true, - lax_spacing: true, - space_after_headers: true, - superscript: true, - underline: true, - highlight: true, - footnotes: false, - } - - renderer = HTMLRenderer.new({ - filter_html: false, - escape_html: false, - no_images: true, - no_styles: true, - safe_links_only: true, - hard_wrap: true, - link_attributes: { target: '_blank', rel: 'nofollow noopener' }, - }) - - Redcarpet::Markdown.new(renderer, extensions) - end - - def html_entities - @html_entities ||= HTMLEntities.new - end - - def encode(html) - html_entities.encode(html) - end - - def encode_and_link_urls(html, accounts = nil, options = {}) - if accounts.is_a?(Hash) - options = accounts - accounts = nil - end - - entities = options[:keep_html] ? html_friendly_extractor(html) : utf8_friendly_extractor(html, extract_url_without_protocol: false) - - rewrite(html.dup, entities, options[:keep_html]) do |entity| - if entity[:url] - link_to_url(entity, options) - elsif entity[:hashtag] - link_to_hashtag(entity) - elsif entity[:screen_name] - link_to_mention(entity, accounts, options) - end - end - end - - def count_tag_nesting(tag) - if tag[1] == '/' then -1 - elsif tag[-2] == '/' then 0 - else 1 - end - end - - # rubocop:disable Metrics/BlockNesting - def encode_custom_emojis(html, emojis, animate = false) - return html if emojis.empty? - - emoji_map = emojis.each_with_object({}) { |e, h| h[e.shortcode] = [full_asset_url(e.image.url), full_asset_url(e.image.url(:static))] } - - i = -1 - tag_open_index = nil - inside_shortname = false - shortname_start_index = -1 - invisible_depth = 0 - - while i + 1 < html.size - i += 1 - - if invisible_depth.zero? && inside_shortname && html[i] == ':' - shortcode = html[shortname_start_index + 1..i - 1] - emoji = emoji_map[shortcode] - - if emoji - original_url, static_url = emoji - replacement = begin - if animate - image_tag(original_url, draggable: false, class: 'emojione', alt: ":#{shortcode}:", title: ":#{shortcode}:") - else - image_tag(original_url, draggable: false, class: 'emojione custom-emoji', alt: ":#{shortcode}:", title: ":#{shortcode}:", data: { original: original_url, static: static_url }) - end - end - before_html = shortname_start_index.positive? ? html[0..shortname_start_index - 1] : '' - html = before_html + replacement + html[i + 1..-1] - i += replacement.size - (shortcode.size + 2) - 1 - else - i -= 1 - end - - inside_shortname = false - elsif tag_open_index && html[i] == '>' - tag = html[tag_open_index..i] - tag_open_index = nil - if invisible_depth.positive? - invisible_depth += count_tag_nesting(tag) - elsif tag == '<span class="invisible">' - invisible_depth = 1 - end - elsif html[i] == '<' - tag_open_index = i - inside_shortname = false - elsif !tag_open_index && html[i] == ':' - inside_shortname = true - shortname_start_index = i - end - end - - html - end - # rubocop:enable Metrics/BlockNesting - - def rewrite(text, entities, keep_html = false) - text = text.to_s - - # Sort by start index - entities = entities.sort_by do |entity| - indices = entity.respond_to?(:indices) ? entity.indices : entity[:indices] - indices.first - end - - result = [] - - last_index = entities.reduce(0) do |index, entity| - indices = entity.respond_to?(:indices) ? entity.indices : entity[:indices] - result << (keep_html ? text[index...indices.first] : encode(text[index...indices.first])) - result << yield(entity) - indices.last - end - - result << (keep_html ? text[last_index..-1] : encode(text[last_index..-1])) - - result.flatten.join - end - - def utf8_friendly_extractor(text, options = {}) - # Note: I couldn't obtain list_slug with @user/list-name format - # for mention so this requires additional check - special = Extractor.extract_urls_with_indices(text, options) - standard = Extractor.extract_entities_with_indices(text, options) - extra = Extractor.extract_extra_uris_with_indices(text, options) - - Extractor.remove_overlapping_entities(special + standard + extra) - end - - def html_friendly_extractor(html, options = {}) - gaps = [] - total_offset = 0 - - escaped = html.gsub(/<[^>]*>|&#[0-9]+;/) do |match| - total_offset += match.length - 1 - end_offset = Regexp.last_match.end(0) - gaps << [end_offset - total_offset, total_offset] - "\u200b" - end - - entities = Extractor.extract_hashtags_with_indices(escaped, :check_url_overlap => false) + - Extractor.extract_mentions_or_lists_with_indices(escaped) - Extractor.remove_overlapping_entities(entities).map do |extract| - pos = extract[:indices].first - offset_idx = gaps.rindex { |gap| gap.first <= pos } - offset = offset_idx.nil? ? 0 : gaps[offset_idx].last - next extract.merge( - :indices => [extract[:indices].first + offset, extract[:indices].last + offset] - ) - end - end - - def link_to_url(entity, options = {}) - url = Addressable::URI.parse(entity[:url]) - html_attrs = { target: '_blank', rel: 'nofollow noopener noreferrer' } - - html_attrs[:rel] = "me #{html_attrs[:rel]}" if options[:me] - - Twitter::TwitterText::Autolink.send(:link_to_text, entity, link_html(entity[:url]), url, html_attrs) - rescue Addressable::URI::InvalidURIError, IDN::Idna::IdnaError - encode(entity[:url]) - end - - def link_to_mention(entity, linkable_accounts, options = {}) - acct = entity[:screen_name] - - return link_to_account(acct, options) unless linkable_accounts - - same_username_hits = 0 - account = nil - username, domain = acct.split('@') - domain = nil if TagManager.instance.local_domain?(domain) - - linkable_accounts.each do |item| - same_username = item.username.casecmp(username).zero? - same_domain = item.domain.nil? ? domain.nil? : item.domain.casecmp(domain)&.zero? - - if same_username && !same_domain - same_username_hits += 1 - elsif same_username && same_domain - account = item - end - end - - account ? mention_html(account, with_domain: same_username_hits.positive? || options[:with_domain]) : "@#{encode(acct)}" - end - - def link_to_account(acct, options = {}) - username, domain = acct.split('@') - - domain = nil if TagManager.instance.local_domain?(domain) - account = EntityCache.instance.mention(username, domain) - - account ? mention_html(account, with_domain: options[:with_domain]) : "@#{encode(acct)}" - end - - def link_to_hashtag(entity) - hashtag_html(entity[:hashtag]) - end - - def link_html(url) - url = Addressable::URI.parse(url).to_s - prefix = url.match(/\A(https?:\/\/(www\.)?|xmpp:)/).to_s - text = url[prefix.length, 30] - suffix = url[prefix.length + 30..-1] - cutoff = url[prefix.length..-1].length > 30 - - "<span class=\"invisible\">#{encode(prefix)}</span><span class=\"#{cutoff ? 'ellipsis' : ''}\">#{encode(text)}</span><span class=\"invisible\">#{encode(suffix)}</span>" - end - - def hashtag_html(tag) - "<a href=\"#{encode(tag_url(tag))}\" class=\"mention hashtag\" rel=\"tag\">#<span>#{encode(tag)}</span></a>" - end - - def mention_html(account, with_domain: false) - "<span class=\"h-card\"><a href=\"#{encode(ActivityPub::TagManager.instance.url_for(account))}\" class=\"u-url mention\">@<span>#{encode(with_domain ? account.pretty_acct : account.username)}</span></a></span>" - end -end diff --git a/app/lib/html_aware_formatter.rb b/app/lib/html_aware_formatter.rb new file mode 100644 index 000000000..7a1cd0340 --- /dev/null +++ b/app/lib/html_aware_formatter.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +class HtmlAwareFormatter + attr_reader :text, :local, :options + + alias local? local + + # @param [String] text + # @param [Boolean] local + # @param [Hash] options + def initialize(text, local, options = {}) + @text = text + @local = local + @options = options + end + + def to_s + return ''.html_safe if text.blank? + + if local? + linkify + else + reformat.html_safe # rubocop:disable Rails/OutputSafety + end + rescue ArgumentError + ''.html_safe + end + + private + + def reformat + Sanitize.fragment(text, Sanitize::Config::MASTODON_STRICT) + end + + def linkify + if %w(text/markdown text/html).include?(@options[:content_type]) + AdvancedTextFormatter.new(text, options).to_s + else + TextFormatter.new(text, options).to_s + end + end +end diff --git a/app/lib/link_details_extractor.rb b/app/lib/link_details_extractor.rb index fabbd244d..b0c4e4f42 100644 --- a/app/lib/link_details_extractor.rb +++ b/app/lib/link_details_extractor.rb @@ -208,7 +208,7 @@ class LinkDetailsExtractor end def valid_url_or_nil(str, same_origin_only: false) - return if str.blank? + return if str.blank? || str == 'null' url = @original_url + Addressable::URI.parse(str) diff --git a/app/lib/plain_text_formatter.rb b/app/lib/plain_text_formatter.rb new file mode 100644 index 000000000..08aa29696 --- /dev/null +++ b/app/lib/plain_text_formatter.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +class PlainTextFormatter + include ActionView::Helpers::TextHelper + + NEWLINE_TAGS_RE = /(<br \/>|<br>|<\/p>)+/.freeze + + attr_reader :text, :local + + alias local? local + + def initialize(text, local) + @text = text + @local = local + end + + def to_s + if local? + text + else + strip_tags(insert_newlines).chomp + end + end + + private + + def insert_newlines + text.gsub(NEWLINE_TAGS_RE) { |match| "#{match}\n" } + end +end diff --git a/app/lib/rss/serializer.rb b/app/lib/rss/serializer.rb index 7e3ed1f17..d44e94221 100644 --- a/app/lib/rss/serializer.rb +++ b/app/lib/rss/serializer.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class RSS::Serializer + include FormattingHelper + private def render_statuses(builder, statuses) @@ -9,7 +11,7 @@ class RSS::Serializer item.title(status_title(status)) .link(ActivityPub::TagManager.instance.url_for(status)) .pub_date(status.created_at) - .description(status.spoiler_text.presence || Formatter.instance.format(status, inline_poll_options: true).to_str) + .description(status_description(status)) status.ordered_media_attachments.each do |media| item.enclosure(full_asset_url(media.file.url(:original, false)), media.file.content_type, media.file.size) @@ -19,9 +21,8 @@ class RSS::Serializer end def status_title(status) - return "#{status.account.acct} deleted status" if status.destroyed? - preview = status.proper.spoiler_text.presence || status.proper.text + if preview.length > 30 || preview[0, 30].include?("\n") preview = preview[0, 30] preview = preview[0, preview.index("\n").presence || 30] + '…' @@ -35,4 +36,20 @@ class RSS::Serializer "#{status.account.acct}: #{preview}" end end + + def status_description(status) + if status.proper.spoiler_text? + status.proper.spoiler_text + else + html = status_content_format(status.proper).to_str + after_html = '' + + if status.proper.preloadable_poll + poll_options_html = status.proper.preloadable_poll.options.map { |o| "[ ] #{o}" }.join('<br />') + after_html = "<p>#{poll_options_html}</p>" + end + + "#{html}#{after_html}" + end + end end diff --git a/app/lib/text_formatter.rb b/app/lib/text_formatter.rb new file mode 100644 index 000000000..48e2fc233 --- /dev/null +++ b/app/lib/text_formatter.rb @@ -0,0 +1,158 @@ +# frozen_string_literal: true + +class TextFormatter + include ActionView::Helpers::TextHelper + include ERB::Util + include RoutingHelper + + URL_PREFIX_REGEX = /\A(https?:\/\/(www\.)?|xmpp:)/.freeze + + DEFAULT_REL = %w(nofollow noopener noreferrer).freeze + + DEFAULT_OPTIONS = { + multiline: true, + }.freeze + + attr_reader :text, :options + + # @param [String] text + # @param [Hash] options + # @option options [Boolean] :multiline + # @option options [Boolean] :with_domains + # @option options [Boolean] :with_rel_me + # @option options [Array<Account>] :preloaded_accounts + def initialize(text, options = {}) + @text = text + @options = DEFAULT_OPTIONS.merge(options) + end + + def entities + @entities ||= Extractor.extract_entities_with_indices(text, extract_url_without_protocol: false) + end + + def to_s + return ''.html_safe if text.blank? + + html = rewrite do |entity| + if entity[:url] + link_to_url(entity) + elsif entity[:hashtag] + link_to_hashtag(entity) + elsif entity[:screen_name] + link_to_mention(entity) + end + end + + html = simple_format(html, {}, sanitize: false).delete("\n") if multiline? + + html.html_safe # rubocop:disable Rails/OutputSafety + end + + private + + def rewrite + entities.sort_by! do |entity| + entity[:indices].first + end + + result = ''.dup + + last_index = entities.reduce(0) do |index, entity| + indices = entity[:indices] + result << h(text[index...indices.first]) + result << yield(entity) + indices.last + end + + result << h(text[last_index..-1]) + + result + end + + def link_to_url(entity) + url = Addressable::URI.parse(entity[:url]).to_s + rel = with_rel_me? ? (DEFAULT_REL + %w(me)) : DEFAULT_REL + + prefix = url.match(URL_PREFIX_REGEX).to_s + display_url = url[prefix.length, 30] + suffix = url[prefix.length + 30..-1] + cutoff = url[prefix.length..-1].length > 30 + + <<~HTML.squish + <a href="#{h(url)}" target="_blank" rel="#{rel.join(' ')}"><span class="invisible">#{h(prefix)}</span><span class="#{cutoff ? 'ellipsis' : ''}">#{h(display_url)}</span><span class="invisible">#{h(suffix)}</span></a> + HTML + rescue Addressable::URI::InvalidURIError, IDN::Idna::IdnaError + h(entity[:url]) + end + + def link_to_hashtag(entity) + hashtag = entity[:hashtag] + url = tag_url(hashtag) + + <<~HTML.squish + <a href="#{h(url)}" class="mention hashtag" rel="tag">#<span>#{h(hashtag)}</span></a> + HTML + end + + def link_to_mention(entity) + username, domain = entity[:screen_name].split('@') + domain = nil if local_domain?(domain) + account = nil + + if preloaded_accounts? + same_username_hits = 0 + + preloaded_accounts.each do |other_account| + same_username = other_account.username.casecmp(username).zero? + same_domain = other_account.domain.nil? ? domain.nil? : other_account.domain.casecmp(domain)&.zero? + + if same_username && !same_domain + same_username_hits += 1 + elsif same_username && same_domain + account = other_account + end + end + else + account = entity_cache.mention(username, domain) + end + + return "@#{h(entity[:screen_name])}" if account.nil? + + url = ActivityPub::TagManager.instance.url_for(account) + display_username = same_username_hits&.positive? || with_domains? ? account.pretty_acct : account.username + + <<~HTML.squish + <span class="h-card"><a href="#{h(url)}" class="u-url mention">@<span>#{h(display_username)}</span></a></span> + HTML + end + + def entity_cache + @entity_cache ||= EntityCache.instance + end + + def tag_manager + @tag_manager ||= TagManager.instance + end + + delegate :local_domain?, to: :tag_manager + + def multiline? + options[:multiline] + end + + def with_domains? + options[:with_domains] + end + + def with_rel_me? + options[:with_rel_me] + end + + def preloaded_accounts + options[:preloaded_accounts] + end + + def preloaded_accounts? + preloaded_accounts.present? + end +end |