From 94aef563b9abbc449028f44c4aac84ef2e072d89 Mon Sep 17 00:00:00 2001 From: Thibaut Girka Date: Sun, 12 May 2019 20:20:05 +0200 Subject: Add support for markdown-formatted toots --- app/lib/formatter.rb | 50 ++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 44 insertions(+), 6 deletions(-) (limited to 'app/lib') diff --git a/app/lib/formatter.rb b/app/lib/formatter.rb index 8a1aad41a..fe5b5b7b7 100644 --- a/app/lib/formatter.rb +++ b/app/lib/formatter.rb @@ -36,14 +36,52 @@ class Formatter html = raw_content html = "RT @#{prepend_reblog} #{html}" if prepend_reblog - html = encode_and_link_urls(html, linkable_accounts) + html = format_markdown(html) if status.content_type == 'text/markdown' + html = encode_and_link_urls(html, linkable_accounts, keep_html: status.content_type == 'text/markdown') html = encode_custom_emojis(html, status.emojis, options[:autoplay]) if options[:custom_emojify] - html = simple_format(html, {}, sanitize: false) + html = simple_format(html, {}, sanitize: false) unless status.content_type == 'text/markdown' html = html.delete("\n") html.html_safe # rubocop:disable Rails/OutputSafety end + def format_markdown(html) + extensions = { + autolink: false, + no_intra_emphasis: true, + fenced_code_blocks: true, + disable_indented_code_blocks: true, + strikethrough: true, + lax_spacing: true, + space_after_headers: true, + superscript: true, + underline: true, + highlight: true, + footnotes: true + } + + renderer = Redcarpet::Render::HTML.new({ + filter_html: false, + no_images: true, + no_styles: true, + safe_links_only: true, + hard_wrap: true, + link_attributes: { target: '_blank', rel: 'nofollow noopener' }, + }) + + markdown = Redcarpet::Markdown.new(renderer, extensions) + + html = reformat(markdown.render(html)) + html = html.gsub("\r\n", "\n").gsub("\r", "\n") + code_safe_strip(html) + end + + def code_safe_strip(html, char="\n") + html = html.split(/(].*?\/code>)/m) + html.each_slice(2) { |part| part[0].delete!(char) } + html.join + end + def reformat(html) sanitize(html, Sanitize::Config::MASTODON_STRICT) end @@ -116,7 +154,7 @@ class Formatter accounts = nil end - rewrite(html.dup, entities) do |entity| + rewrite(html.dup, entities, options[:keep_html]) do |entity| if entity[:url] link_to_url(entity, options) elsif entity[:hashtag] @@ -186,7 +224,7 @@ class Formatter html end - def rewrite(text, entities) + def rewrite(text, entities, keep_html = false) text = text.to_s # Sort by start index @@ -199,12 +237,12 @@ class Formatter last_index = entities.reduce(0) do |index, entity| indices = entity.respond_to?(:indices) ? entity.indices : entity[:indices] - result << encode(text[index...indices.first]) + result << (keep_html ? text[index...indices.first] : encode(text[index...indices.first])) result << yield(entity) indices.last end - result << encode(text[last_index..-1]) + result << (keep_html ? text[last_index..-1] : encode(text[last_index..-1])) result.flatten.join end -- cgit From d7520f81759e9db58a5b5ca61b9114e9ef44d92f Mon Sep 17 00:00:00 2001 From: Thibaut Girka Date: Sun, 12 May 2019 21:33:37 +0200 Subject: Add support for HTML-formatted toots --- app/lib/formatter.rb | 4 ++-- app/models/status.rb | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'app/lib') diff --git a/app/lib/formatter.rb b/app/lib/formatter.rb index fe5b5b7b7..eaece8797 100644 --- a/app/lib/formatter.rb +++ b/app/lib/formatter.rb @@ -37,9 +37,9 @@ class Formatter html = raw_content html = "RT @#{prepend_reblog} #{html}" if prepend_reblog html = format_markdown(html) if status.content_type == 'text/markdown' - html = encode_and_link_urls(html, linkable_accounts, keep_html: status.content_type == 'text/markdown') + html = encode_and_link_urls(html, linkable_accounts, keep_html: %w(text/markdown text/html).include?(status.content_type)) html = encode_custom_emojis(html, status.emojis, options[:autoplay]) if options[:custom_emojify] - html = simple_format(html, {}, sanitize: false) unless status.content_type == 'text/markdown' + html = simple_format(html, {}, sanitize: false) unless %w(text/markdown text/html).include?(status.content_type) html = html.delete("\n") html.html_safe # rubocop:disable Rails/OutputSafety diff --git a/app/models/status.rb b/app/models/status.rb index e0dc74790..6f3ba4cc3 100644 --- a/app/models/status.rb +++ b/app/models/status.rb @@ -75,7 +75,7 @@ class Status < ApplicationRecord validates_with DisallowedHashtagsValidator validates :reblog, uniqueness: { scope: :account }, if: :reblog? validates :visibility, exclusion: { in: %w(direct limited) }, if: :reblog? - validates :content_type, inclusion: { in: %w(text/plain text/markdown) }, allow_nil: true + validates :content_type, inclusion: { in: %w(text/plain text/markdown text/html) }, allow_nil: true accepts_nested_attributes_for :poll -- cgit From 0be93820f344fafc8618febde149e9c63d1ba5d4 Mon Sep 17 00:00:00 2001 From: Thibaut Girka Date: Sun, 12 May 2019 22:13:36 +0200 Subject: Fix autolinking, and newlines in code blocks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Autolinking is now performed *after* the Markdown pass, by replacing HTML tags with zero-width spaces and running the twitter-text extractor as usual, except it does not auto-link URLs to avoid links in links… --- app/lib/formatter.rb | 65 ++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 50 insertions(+), 15 deletions(-) (limited to 'app/lib') diff --git a/app/lib/formatter.rb b/app/lib/formatter.rb index eaece8797..2c509ef19 100644 --- a/app/lib/formatter.rb +++ b/app/lib/formatter.rb @@ -3,6 +3,17 @@ require 'singleton' require_relative './sanitize_config' +class HTMLRenderer < Redcarpet::Render::HTML + def block_code(code, language) + "
#{code.gsub("\n", "
")}
" + end + + def autolink(link, link_type) + return link if link_type == :email + Formatter.instance.link_url(link) + end +end + class Formatter include Singleton include RoutingHelper @@ -39,15 +50,18 @@ class Formatter html = format_markdown(html) if status.content_type == 'text/markdown' html = encode_and_link_urls(html, linkable_accounts, keep_html: %w(text/markdown text/html).include?(status.content_type)) html = encode_custom_emojis(html, status.emojis, options[:autoplay]) if options[:custom_emojify] - html = simple_format(html, {}, sanitize: false) unless %w(text/markdown text/html).include?(status.content_type) - html = html.delete("\n") + + unless %w(text/markdown text/html).include?(status.content_type) + html = simple_format(html, {}, sanitize: false) + html = html.delete("\n") + end html.html_safe # rubocop:disable Rails/OutputSafety end def format_markdown(html) extensions = { - autolink: false, + autolink: true, no_intra_emphasis: true, fenced_code_blocks: true, disable_indented_code_blocks: true, @@ -57,11 +71,12 @@ class Formatter superscript: true, underline: true, highlight: true, - footnotes: true + footnotes: false, } - renderer = Redcarpet::Render::HTML.new({ + renderer = HTMLRenderer.new({ filter_html: false, + escape_html: false, no_images: true, no_styles: true, safe_links_only: true, @@ -72,14 +87,7 @@ class Formatter markdown = Redcarpet::Markdown.new(renderer, extensions) html = reformat(markdown.render(html)) - html = html.gsub("\r\n", "\n").gsub("\r", "\n") - code_safe_strip(html) - end - - def code_safe_strip(html, char="\n") - html = html.split(/(].*?\/code>)/m) - html.each_slice(2) { |part| part[0].delete!(char) } - html.join + html.delete("\r").delete("\n") end def reformat(html) @@ -136,6 +144,10 @@ class Formatter html.html_safe # rubocop:disable Rails/OutputSafety end + def link_url(url) + "#{link_html(url)}" + end + private def html_entities @@ -147,13 +159,13 @@ class Formatter end def encode_and_link_urls(html, accounts = nil, options = {}) - entities = utf8_friendly_extractor(html, extract_url_without_protocol: false) - if accounts.is_a?(Hash) options = accounts accounts = nil end + entities = options[:keep_html] ? html_friendly_extractor(html) : utf8_friendly_extractor(html, extract_url_without_protocol: false) + rewrite(html.dup, entities, options[:keep_html]) do |entity| if entity[:url] link_to_url(entity, options) @@ -285,6 +297,29 @@ class Formatter Extractor.remove_overlapping_entities(special + standard) end + def html_friendly_extractor(html, options = {}) + gaps = [] + total_offset = 0 + + escaped = html.gsub(/<[^>]*>/) do |match| + total_offset += match.length - 1 + end_offset = Regexp.last_match.end(0) + gaps << [end_offset - total_offset, total_offset] + "\u200b" + end + + entities = Extractor.extract_hashtags_with_indices(escaped, :check_url_overlap => false) + + Extractor.extract_mentions_or_lists_with_indices(escaped) + Extractor.remove_overlapping_entities(entities).map do |extract| + pos = extract[:indices].first + offset_idx = gaps.rindex { |gap| gap.first <= pos } + offset = offset_idx.nil? ? 0 : gaps[offset_idx].last + next extract.merge( + :indices => [extract[:indices].first + offset, extract[:indices].last + offset] + ) + end + end + def link_to_url(entity, options = {}) url = Addressable::URI.parse(entity[:url]) html_attrs = { target: '_blank', rel: 'nofollow noopener' } -- cgit From a6b7c23f6fd33c209f83562fffb46211e062312e Mon Sep 17 00:00:00 2001 From: Thibaut Girka Date: Mon, 13 May 2019 14:42:39 +0200 Subject: Add option for default toot content-type --- app/controllers/settings/preferences_controller.rb | 1 + app/lib/user_settings_decorator.rb | 5 +++++ app/models/user.rb | 2 +- app/services/post_status_service.rb | 2 +- app/views/settings/preferences/show.html.haml | 2 ++ config/locales/simple_form.en.yml | 7 +++++++ config/settings.yml | 1 + 7 files changed, 18 insertions(+), 2 deletions(-) (limited to 'app/lib') diff --git a/app/controllers/settings/preferences_controller.rb b/app/controllers/settings/preferences_controller.rb index eb7a0eb4a..3d98d583c 100644 --- a/app/controllers/settings/preferences_controller.rb +++ b/app/controllers/settings/preferences_controller.rb @@ -46,6 +46,7 @@ class Settings::PreferencesController < Settings::BaseController :setting_hide_followers_count, :setting_aggregate_reblogs, :setting_show_application, + :setting_default_content_type, notification_emails: %i(follow follow_request reblog favourite mention digest report pending_account), interactions: %i(must_be_follower must_be_following) ) diff --git a/app/lib/user_settings_decorator.rb b/app/lib/user_settings_decorator.rb index 367ba9a83..802ca71fe 100644 --- a/app/lib/user_settings_decorator.rb +++ b/app/lib/user_settings_decorator.rb @@ -36,6 +36,7 @@ class UserSettingsDecorator user.settings['hide_network'] = hide_network_preference if change?('setting_hide_network') user.settings['aggregate_reblogs'] = aggregate_reblogs_preference if change?('setting_aggregate_reblogs') user.settings['show_application'] = show_application_preference if change?('setting_show_application') + user.settings['default_content_type']= default_content_type_preference if change?('setting_default_content_type') end def merged_notification_emails @@ -122,6 +123,10 @@ class UserSettingsDecorator boolean_cast_setting 'setting_aggregate_reblogs' end + def default_content_type_preference + settings['setting_default_content_type'] + end + def boolean_cast_setting(key) ActiveModel::Type::Boolean.new.cast(settings[key]) end diff --git a/app/models/user.rb b/app/models/user.rb index 8985ebf53..a245d3eb2 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -104,7 +104,7 @@ class User < ApplicationRecord delegate :auto_play_gif, :default_sensitive, :unfollow_modal, :boost_modal, :favourite_modal, :delete_modal, :reduce_motion, :system_font_ui, :noindex, :flavour, :skin, :display_media, :hide_network, :hide_followers_count, - :expand_spoilers, :default_language, :aggregate_reblogs, :show_application, to: :settings, prefix: :setting, allow_nil: false + :expand_spoilers, :default_language, :aggregate_reblogs, :show_application, :default_content_type, to: :settings, prefix: :setting, allow_nil: false attr_reader :invite_code attr_writer :external diff --git a/app/services/post_status_service.rb b/app/services/post_status_service.rb index 25aa6629c..c2584e090 100644 --- a/app/services/post_status_service.rb +++ b/app/services/post_status_service.rb @@ -168,7 +168,7 @@ class PostStatusService < BaseService visibility: @visibility, language: language_from_option(@options[:language]) || @account.user&.setting_default_language&.presence || LanguageDetector.instance.detect(@text, @account), application: @options[:application], - content_type: @options[:content_type], + content_type: @options[:content_type] || @account.user&.setting_default_content_type, }.compact end diff --git a/app/views/settings/preferences/show.html.haml b/app/views/settings/preferences/show.html.haml index a50f33517..cd5bf9be2 100644 --- a/app/views/settings/preferences/show.html.haml +++ b/app/views/settings/preferences/show.html.haml @@ -25,6 +25,8 @@ .fields-group = f.input :setting_default_privacy, collection: Status.selectable_visibilities, wrapper: :with_floating_label, include_blank: false, label_method: lambda { |visibility| safe_join([I18n.t("statuses.visibilities.#{visibility}"), content_tag(:span, I18n.t("statuses.visibilities.#{visibility}_long"), class: 'hint')]) }, required: false, as: :radio_buttons, collection_wrapper_tag: 'ul', item_wrapper_tag: 'li' + = f.input :setting_default_content_type, collection: ['text/plain', 'text/markdown', 'text/html'], wrapper: :with_label, include_blank: false, label_method: lambda { |item| safe_join([t("simple_form.labels.defaults.setting_default_content_type_#{item.split('/')[1]}"), content_tag(:span, t("simple_form.hints.defaults.setting_default_content_type_#{item.split('/')[1]}"), class: 'hint')]) }, required: false, as: :radio_buttons, collection_wrapper_tag: 'ul', item_wrapper_tag: 'li' + = f.input :setting_default_sensitive, as: :boolean, wrapper: :with_label %hr#settings_other/ diff --git a/config/locales/simple_form.en.yml b/config/locales/simple_form.en.yml index ba0e403e4..6fad7f73a 100644 --- a/config/locales/simple_form.en.yml +++ b/config/locales/simple_form.en.yml @@ -27,6 +27,9 @@ en: phrase: Will be matched regardless of casing in text or content warning of a toot scopes: Which APIs the application will be allowed to access. If you select a top-level scope, you don't need to select individual ones. setting_aggregate_reblogs: Do not show new boosts for toots that have been recently boosted (only affects newly-received boosts) + setting_default_content_type_html: When writing toots, assume they are written in raw HTML, unless specified otherwise + setting_default_content_type_markdown: When writing toots, assume they are using Markdown for rich text formatting, unless specified otherwise + setting_default_content_type_plain: When writing toots, assume they are plain text with no special formatting, unless specified otherwise (default Mastodon behavior) setting_default_language: The language of your toots can be detected automatically, but it's not always accurate setting_display_media_default: Hide media marked as sensitive setting_display_media_hide_all: Always hide all media @@ -93,6 +96,10 @@ en: setting_aggregate_reblogs: Group boosts in timelines setting_auto_play_gif: Auto-play animated GIFs setting_boost_modal: Show confirmation dialog before boosting + setting_default_content_type: Default format for toots + setting_default_content_type_html: HTML + setting_default_content_type_markdown: Markdown + setting_default_content_type_plain: Plain text setting_default_language: Posting language setting_default_privacy: Post privacy setting_default_sensitive: Always mark media as sensitive diff --git a/config/settings.yml b/config/settings.yml index c3aeab551..69996af25 100644 --- a/config/settings.yml +++ b/config/settings.yml @@ -64,6 +64,7 @@ defaults: &defaults show_known_fediverse_at_about_page: true show_reblogs_in_public_timelines: false show_replies_in_public_timelines: false + default_content_type: 'text/plain' development: <<: *defaults -- cgit From dd5bf40b97d42daae855cd05ac13c6efa6cda4f6 Mon Sep 17 00:00:00 2001 From: Thibaut Girka Date: Fri, 17 May 2019 10:43:17 +0200 Subject: Properly escape HTML in code blocks --- app/lib/formatter.rb | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) (limited to 'app/lib') diff --git a/app/lib/formatter.rb b/app/lib/formatter.rb index 2c509ef19..ccebf4353 100644 --- a/app/lib/formatter.rb +++ b/app/lib/formatter.rb @@ -5,13 +5,23 @@ require_relative './sanitize_config' class HTMLRenderer < Redcarpet::Render::HTML def block_code(code, language) - "
#{code.gsub("\n", "
")}
" + "
#{encode(code).gsub("\n", "
")}
" end def autolink(link, link_type) return link if link_type == :email Formatter.instance.link_url(link) end + + private + + def html_entities + @html_entities ||= HTMLEntities.new + end + + def encode(html) + html_entities.encode(html) + end end class Formatter -- cgit From 9ca21e93cc1506d0a3c0cfe450636933c3d2388a Mon Sep 17 00:00:00 2001 From: Thibaut Girka Date: Fri, 17 May 2019 11:14:09 +0200 Subject: Minor optimization --- app/lib/formatter.rb | 58 ++++++++++++++++++++++++++++------------------------ 1 file changed, 31 insertions(+), 27 deletions(-) (limited to 'app/lib') diff --git a/app/lib/formatter.rb b/app/lib/formatter.rb index ccebf4353..78a0e9f25 100644 --- a/app/lib/formatter.rb +++ b/app/lib/formatter.rb @@ -70,33 +70,7 @@ class Formatter end def format_markdown(html) - extensions = { - autolink: true, - no_intra_emphasis: true, - fenced_code_blocks: true, - disable_indented_code_blocks: true, - strikethrough: true, - lax_spacing: true, - space_after_headers: true, - superscript: true, - underline: true, - highlight: true, - footnotes: false, - } - - renderer = HTMLRenderer.new({ - filter_html: false, - escape_html: false, - no_images: true, - no_styles: true, - safe_links_only: true, - hard_wrap: true, - link_attributes: { target: '_blank', rel: 'nofollow noopener' }, - }) - - markdown = Redcarpet::Markdown.new(renderer, extensions) - - html = reformat(markdown.render(html)) + html = reformat(markdown_formatter.render(html)) html.delete("\r").delete("\n") end @@ -160,6 +134,36 @@ class Formatter private + def markdown_formatter + return @markdown_formatter if defined?(@markdown_formatter) + + extensions = { + autolink: true, + no_intra_emphasis: true, + fenced_code_blocks: true, + disable_indented_code_blocks: true, + strikethrough: true, + lax_spacing: true, + space_after_headers: true, + superscript: true, + underline: true, + highlight: true, + footnotes: false, + } + + renderer = HTMLRenderer.new({ + filter_html: false, + escape_html: false, + no_images: true, + no_styles: true, + safe_links_only: true, + hard_wrap: true, + link_attributes: { target: '_blank', rel: 'nofollow noopener' }, + }) + + @markdown_formatter = Redcarpet::Markdown.new(renderer, extensions) + end + def html_entities @html_entities ||= HTMLEntities.new end -- cgit From 2332b3f146b0d879daba8a99bd35c8bf425edea3 Mon Sep 17 00:00:00 2001 From: Thibaut Girka Date: Tue, 21 May 2019 22:57:59 +0200 Subject: Fix local text/html toots not being sanitized --- app/lib/formatter.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'app/lib') diff --git a/app/lib/formatter.rb b/app/lib/formatter.rb index 78a0e9f25..a099ff728 100644 --- a/app/lib/formatter.rb +++ b/app/lib/formatter.rb @@ -61,7 +61,9 @@ class Formatter html = encode_and_link_urls(html, linkable_accounts, keep_html: %w(text/markdown text/html).include?(status.content_type)) html = encode_custom_emojis(html, status.emojis, options[:autoplay]) if options[:custom_emojify] - unless %w(text/markdown text/html).include?(status.content_type) + if %w(text/markdown text/html).include?(status.content_type) + html = reformat(html) + else html = simple_format(html, {}, sanitize: false) html = html.delete("\n") end @@ -70,7 +72,7 @@ class Formatter end def format_markdown(html) - html = reformat(markdown_formatter.render(html)) + html = markdown_formatter.render(html) html.delete("\r").delete("\n") end -- cgit