8 files changed, 293 insertions, 6 deletions
diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index e2355bfbc..eca446243 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -152,7 +152,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
       # If there is at least one silent mention, then the status can be considered
       # as a limited-audience status, and not strictly a direct message, but only
       # if we considered a direct message in the first place
-      @params[:visibility] = :limited if @params[:visibility] == :direct
+      @params[:visibility] = :limited if @params[:visibility] == :direct && !@object['directMessage']
     end
 
     # Accounts that are tagged but are not in the audience are not
@@ -164,7 +164,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     return if @status.mentions.find_by(account_id: @options[:delivered_to_account_id])
 
     @status.mentions.create(account: delivered_to_account, silent: true)
-    @status.update(visibility: :limited) if @status.direct_visibility?
+    @status.update(visibility: :limited) if @status.direct_visibility? && !@object['directMessage']
 
     return unless delivered_to_account.following?(@account)
 
diff --git a/app/lib/activitypub/parser/status_parser.rb b/app/lib/activitypub/parser/status_parser.rb
index 3ba154d01..75b8f3d5c 100644
--- a/app/lib/activitypub/parser/status_parser.rb
+++ b/app/lib/activitypub/parser/status_parser.rb
@@ -79,6 +79,8 @@ class ActivityPub::Parser::StatusParser
       :unlisted
     elsif audience_to.include?(@magic_values[:followers_collection])
       :private
+    elsif direct_message == false
+      :limited
     else
       :direct
     end
@@ -94,6 +96,10 @@ class ActivityPub::Parser::StatusParser
     end
   end
 
+  def direct_message
+    @object['directMessage']
+  end
+
   private
 
   def audience_to
diff --git a/app/lib/advanced_text_formatter.rb b/app/lib/advanced_text_formatter.rb
new file mode 100644
index 000000000..21e81d4d1
--- /dev/null
+++ b/app/lib/advanced_text_formatter.rb
@@ -0,0 +1,133 @@
+# frozen_string_literal: true
+
+class AdvancedTextFormatter < TextFormatter
+  class HTMLRenderer < Redcarpet::Render::HTML
+    def initialize(options, &block)
+      super(options)
+      @format_link = block
+    end
+
+    def block_code(code, _language)
+      <<~HTML
+        <pre><code>#{ERB::Util.h(code).gsub("\n", '<br/>')}</code></pre>
+      HTML
+    end
+
+    def autolink(link, link_type)
+      return link if link_type == :email
+      @format_link.call(link)
+    end
+  end
+
+  attr_reader :content_type
+
+  # @param [String] text
+  # @param [Hash] options
+  # @option options [Boolean] :multiline
+  # @option options [Boolean] :with_domains
+  # @option options [Boolean] :with_rel_me
+  # @option options [Array<Account>] :preloaded_accounts
+  # @option options [String] :content_type
+  def initialize(text, options = {})
+    @content_type = options.delete(:content_type)
+    super(text, options)
+
+    @text = format_markdown(text) if content_type == 'text/markdown'
+  end
+
+  # Differs from TextFormatter by not messing with newline after parsing
+  def to_s
+    return ''.html_safe if text.blank?
+
+    html = rewrite do |entity|
+      if entity[:url]
+        link_to_url(entity)
+      elsif entity[:hashtag]
+        link_to_hashtag(entity)
+      elsif entity[:screen_name]
+        link_to_mention(entity)
+      end
+    end
+
+    html.html_safe # rubocop:disable Rails/OutputSafety
+  end
+
+  # Differs from TextFormatter by operating on the parsed HTML tree
+  def rewrite
+    if @tree.nil?
+      src = text.gsub(Sanitize::REGEX_UNSUITABLE_CHARS, '')
+      @tree = Nokogiri::HTML5.fragment(src)
+      document = @tree.document
+
+      @tree.xpath('.//text()[not(ancestor::a | ancestor::code)]').each do |text_node|
+        # Iterate over text elements and build up their replacements.
+        content = text_node.content
+        replacement = Nokogiri::XML::NodeSet.new(document)
+        processed_index = 0
+        Extractor.extract_entities_with_indices(
+          content,
+          extract_url_without_protocol: false
+        ) do |entity|
+          # Iterate over entities in this text node.
+          advance = entity[:indices].first - processed_index
+          if advance.positive?
+            # Text node for content which precedes entity.
+            replacement << Nokogiri::XML::Text.new(
+              content[processed_index, advance],
+              document
+            )
+          end
+          replacement << Nokogiri::HTML5.fragment(yield(entity))
+          processed_index = entity[:indices].last
+        end
+        if processed_index < content.size
+          # Text node for remaining content.
+          replacement << Nokogiri::XML::Text.new(
+            content[processed_index, content.size - processed_index],
+            document
+          )
+        end
+        text_node.replace(replacement)
+      end
+    end
+
+    Sanitize.node!(@tree, Sanitize::Config::MASTODON_OUTGOING).to_html
+  end
+
+  private
+
+  def format_markdown(html)
+    html = markdown_formatter.render(html)
+    html.delete("\r").delete("\n")
+  end
+
+  def markdown_formatter
+    extensions = {
+      autolink: true,
+      no_intra_emphasis: true,
+      fenced_code_blocks: true,
+      disable_indented_code_blocks: true,
+      strikethrough: true,
+      lax_spacing: true,
+      space_after_headers: true,
+      superscript: true,
+      underline: true,
+      highlight: true,
+      footnotes: false,
+    }
+
+    renderer = HTMLRenderer.new({
+      filter_html: false,
+      escape_html: false,
+      no_images: true,
+      no_styles: true,
+      safe_links_only: true,
+      hard_wrap: true,
+      link_attributes: { target: '_blank', rel: 'nofollow noopener' },
+    }) do |url|
+      link_to_url({ url: url })
+    end
+
+    Redcarpet::Markdown.new(renderer, extensions)
+  end
+end
diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb
index 7dda6b185..4ce888fc9 100644
--- a/app/lib/feed_manager.rb
+++ b/app/lib/feed_manager.rb
@@ -45,6 +45,8 @@ class FeedManager
       filter_from_list?(status, receiver) || filter_from_home?(status, receiver.account_id, build_crutches(receiver.account_id, [status]))
     when :mentions
       filter_from_mentions?(status, receiver.id)
+    when :direct
+      filter_from_direct?(status, receiver.id)
     when :tags
       filter_from_tags?(status, receiver.id, build_crutches(receiver.id, [status]))
     else
@@ -102,6 +104,29 @@ class FeedManager
     true
   end
 
+  # Add a status to a linear direct message feed and send a streaming API update
+  # @param [Account] account
+  # @param [Status] status
+  # @return [Boolean]
+  def push_to_direct(account, status, update: false)
+    return false unless add_to_feed(:direct, account.id, status)
+
+    trim(:direct, account.id)
+    PushUpdateWorker.perform_async(account.id, status.id, "timeline:direct:#{account.id}") unless update
+    true
+  end
+
+  # Remove a status from a linear direct message feed and send a streaming API update
+  # @param [List] list
+  # @param [Status] status
+  # @return [Boolean]
+  def unpush_from_direct(account, status, update: false)
+    return false unless remove_from_feed(:direct, account.id, status)
+
+    redis.publish("timeline:direct:#{account.id}", Oj.dump(event: :delete, payload: status.id.to_s)) unless update
+    true
+  end
+
   # Fill a home feed with an account's statuses
   # @param [Account] from_account
   # @param [Account] into_account
@@ -266,6 +291,30 @@ class FeedManager
     end
   end
 
+  # Populate direct feed of account from scratch
+  # @param [Account] account
+  # @return [void]
+  def populate_direct_feed(account)
+    added  = 0
+    limit  = FeedManager::MAX_ITEMS / 2
+    max_id = nil
+
+    loop do
+      statuses = Status.as_direct_timeline(account, limit, max_id)
+
+      break if statuses.empty?
+
+      statuses.each do |status|
+        next if filter_from_direct?(status, account)
+        added += 1 if add_to_feed(:direct, account.id, status)
+      end
+
+      break unless added.zero?
+
+      max_id = statuses.last.id
+    end
+  end
+
   # Completely clear multiple feeds at once
   # @param [Symbol] type
   # @param [Array<Integer>] ids
@@ -404,6 +453,15 @@ class FeedManager
     should_filter
   end
 
+  # Check if status should not be added to the linear direct message feed
+  # @param [Status] status
+  # @param [Integer] receiver_id
+  # @return [Boolean]
+  def filter_from_direct?(status, receiver_id)
+    return false if receiver_id == status.account_id
+    filter_from_mentions?(status, receiver_id)
+  end
+
   # Check if status should not be added to the list feed
   # @param [Status] status
   # @param [List] list
diff --git a/app/lib/html_aware_formatter.rb b/app/lib/html_aware_formatter.rb
index 64edba09b..8766c5ee0 100644
--- a/app/lib/html_aware_formatter.rb
+++ b/app/lib/html_aware_formatter.rb
@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 class HtmlAwareFormatter
+  STATUS_MIME_TYPES = %w(text/plain text/markdown text/html).freeze
+
   attr_reader :text, :local, :options
 
   alias local? local
@@ -33,6 +35,10 @@ class HtmlAwareFormatter
   end
 
   def linkify
-    TextFormatter.new(text, options).to_s
+    if %w(text/markdown text/html).include?(@options[:content_type])
+      AdvancedTextFormatter.new(text, options).to_s
+    else
+      TextFormatter.new(text, options).to_s
+    end
   end
 end
diff --git a/app/lib/settings/scoped_settings.rb b/app/lib/settings/scoped_settings.rb
index 3ad57cc1e..983865a68 100644
--- a/app/lib/settings/scoped_settings.rb
+++ b/app/lib/settings/scoped_settings.rb
@@ -3,7 +3,8 @@
 module Settings
   class ScopedSettings
     DEFAULTING_TO_UNSCOPED = %w(
-      theme
+      flavour
+      skin
       noindex
     ).freeze
 
diff --git a/app/lib/themes.rb b/app/lib/themes.rb
index 243ffb9ab..81e016d4a 100644
--- a/app/lib/themes.rb
+++ b/app/lib/themes.rb
@@ -7,10 +7,86 @@ class Themes
   include Singleton
 
   def initialize
-    @conf = YAML.load_file(Rails.root.join('config', 'themes.yml'))
+
+    core = YAML.load_file(Rails.root.join('app', 'javascript', 'core', 'theme.yml'))
+    core['pack'] = Hash.new unless core['pack']
+
+    result = Hash.new
+    Dir.glob(Rails.root.join('app', 'javascript', 'flavours', '*', 'theme.yml')) do |path|
+      data = YAML.load_file(path)
+      next unless data['pack']
+
+      dir = File.dirname(path)
+      name = File.basename(dir)
+      locales = []
+      screenshots = []
+
+      if data['locales']
+        Dir.glob(File.join(dir, data['locales'], '*.{js,json}')) do |locale|
+          localeName = File.basename(locale, File.extname(locale))
+          locales.push(localeName) unless localeName.match(/defaultMessages|whitelist|index/)
+        end
+      end
+
+      if data['screenshot']
+        if data['screenshot'].is_a? Array
+          screenshots = data['screenshot']
+        else
+          screenshots.push(data['screenshot'])
+        end
+      end
+
+      data['name'] = name
+      data['locales'] = locales
+      data['screenshot'] = screenshots
+      data['skin'] = { 'default' => [] }
+      result[name] = data
+    end
+
+    Dir.glob(Rails.root.join('app', 'javascript', 'skins', '*', '*')) do |path|
+      ext = File.extname(path)
+      skin = File.basename(path)
+      name = File.basename(File.dirname(path))
+      next unless result[name]
+
+      if File.directory?(path)
+        pack = []
+        Dir.glob(File.join(path, '*.{css,scss}')) do |sheet|
+          pack.push(File.basename(sheet, File.extname(sheet)))
+        end
+      elsif ext.match(/^\.s?css$/i)
+        skin = File.basename(path, ext)
+        pack = ['common']
+      end
+
+      if skin != 'default'
+        result[name]['skin'][skin] = pack
+      end
+    end
+
+    @core = core
+    @conf = result
+  end
+
+  def core
+    @core
+  end
+
+  def flavour(name)
+    @conf[name]
   end
 
-  def names
+  def flavours
     @conf.keys
   end
+
+  def skins_for(name)
+    @conf[name]['skin'].keys
+  end
+
+  def flavours_and_skins
+    flavours.map do |flavour|
+      [flavour, skins_for(flavour).map{ |skin| [flavour, skin] }]
+    end
+  end
 end
diff --git a/app/lib/vacuum/feeds_vacuum.rb b/app/lib/vacuum/feeds_vacuum.rb
index fb0b8a847..b0246bc0d 100644
--- a/app/lib/vacuum/feeds_vacuum.rb
+++ b/app/lib/vacuum/feeds_vacuum.rb
@@ -4,6 +4,7 @@ class Vacuum::FeedsVacuum
   def perform
     vacuum_inactive_home_feeds!
     vacuum_inactive_list_feeds!
+    vacuum_inactive_direct_feeds!
   end
 
   private
@@ -20,6 +21,12 @@ class Vacuum::FeedsVacuum
     end
   end
 
+  def vacuum_inactive_direct_feeds!
+    inactive_users_lists.select(:id).find_in_batches do |lists|
+      feed_manager.clean_feeds!(:direct, lists.map(&:id))
+    end
+  end
+
   def inactive_users
     User.confirmed.inactive
   end