Merge branch 'master' into glitch-soc/merge-upstream

Conflicts:
- app/controllers/oauth/authorized_applications_controller.rb
  Two changes too close to each other
- app/controllers/settings/sessions_controller.rb
- app/lib/user_settings_decorator.rb
  Two changes too close to each other
- app/models/media_attachment.rb
  New changes too close to glitch-soc only changes.
- app/models/user.rb
  Two changes too close to each other.
- app/services/remove_status_service.rb
  Kept direct timeline code which had been removed upstream.
- app/views/settings/preferences/show.html.haml
  Two changes too close to each other.
- config/locales/en.yml
  Introduction of a new string too close to glitch-soc-only's “flavour”
- config/locales/ja.yml
  Introduction of a new string too close to glitch-soc-only's “flavour”
- config/locales/pl.yml
  Introduction of a new string too close to glitch-soc-only's “flavour”
- config/locales/simple_form.en.yml
  Introduction of a new string too close to glitch-soc-only's “skin”
- config/locales/simple_form.pl.yml
  Introduction of a new string too close to glitch-soc-only's “skin”
- config/settings.yml
  Reverted upstream's decision of enabling posting application by default.

6 files changed, 58 insertions, 25 deletions

diff --git a/app/lib/activity_tracker.rb b/app/lib/activity_tracker.rb
index 5b4972674..ae3c11b6a 100644
--- a/app/lib/activity_tracker.rb
+++ b/app/lib/activity_tracker.rb
@@ -4,6 +4,8 @@ class ActivityTracker
   EXPIRE_AFTER = 90.days.seconds
 
   class << self
+    include Redisable
+
     def increment(prefix)
       key = [prefix, current_week].join(':')
 
@@ -20,10 +22,6 @@ class ActivityTracker
 
     private
 
-    def redis
-      Redis.current
-    end
-
     def current_week
       Time.zone.today.cweek
     end
diff --git a/app/lib/activitypub/activity.rb b/app/lib/activitypub/activity.rb
index 87318fb1c..919678618 100644
--- a/app/lib/activitypub/activity.rb
+++ b/app/lib/activitypub/activity.rb
@@ -2,6 +2,7 @@
 
 class ActivityPub::Activity
   include JsonLdHelper
+  include Redisable
 
   def initialize(json, account, **options)
     @json    = json
@@ -70,10 +71,6 @@ class ActivityPub::Activity
     @object_uri ||= value_or_id(@object)
   end
 
-  def redis
-    Redis.current
-  end
-
   def distribute(status)
     crawl_links(status)
 
diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb
index a1b186f1c..4bc75dae8 100644
--- a/app/lib/feed_manager.rb
+++ b/app/lib/feed_manager.rb
@@ -4,6 +4,7 @@ require 'singleton'
 
 class FeedManager
   include Singleton
+  include Redisable
 
   MAX_ITEMS = 400
 
@@ -35,7 +36,7 @@ class FeedManager
 
   def unpush_from_home(account, status)
     return false unless remove_from_feed(:home, account.id, status)
-    Redis.current.publish("timeline:#{account.id}", Oj.dump(event: :delete, payload: status.id.to_s))
+    redis.publish("timeline:#{account.id}", Oj.dump(event: :delete, payload: status.id.to_s))
     true
   end
 
@@ -54,7 +55,7 @@ class FeedManager
 
   def unpush_from_list(list, status)
     return false unless remove_from_feed(:list, list.id, status)
-    Redis.current.publish("timeline:list:#{list.id}", Oj.dump(event: :delete, payload: status.id.to_s))
+    redis.publish("timeline:list:#{list.id}", Oj.dump(event: :delete, payload: status.id.to_s))
     true
   end
 
@@ -143,10 +144,6 @@ class FeedManager
 
   private
 
-  def redis
-    Redis.current
-  end
-
   def push_update_required?(timeline_id)
     redis.exists("subscribed:#{timeline_id}")
   end
diff --git a/app/lib/formatter.rb b/app/lib/formatter.rb
index 05fd9eeb1..0653214f5 100644
--- a/app/lib/formatter.rb
+++ b/app/lib/formatter.rb
@@ -99,7 +99,7 @@ class Formatter
   end
 
   def encode_and_link_urls(html, accounts = nil, options = {})
-    entities = Extractor.extract_entities_with_indices(html, extract_url_without_protocol: false)
+    entities = utf8_friendly_extractor(html, extract_url_without_protocol: false)
 
     if accounts.is_a?(Hash)
       options  = accounts
@@ -199,6 +199,53 @@ class Formatter
     result.flatten.join
   end
 
+  UNICODE_ESCAPE_BLACKLIST_RE = /\p{Z}|\p{P}/
+
+  def utf8_friendly_extractor(text, options = {})
+    old_to_new_index = [0]
+
+    escaped = text.chars.map do |c|
+      output = begin
+        if c.ord.to_s(16).length > 2 && UNICODE_ESCAPE_BLACKLIST_RE.match(c).nil?
+          CGI.escape(c)
+        else
+          c
+        end
+      end
+
+      old_to_new_index << old_to_new_index.last + output.length
+
+      output
+    end.join
+
+    # Note: I couldn't obtain list_slug with @user/list-name format
+    # for mention so this requires additional check
+    special = Extractor.extract_urls_with_indices(escaped, options).map do |extract|
+      # exactly one of :url, :hashtag, :screen_name, :cashtag keys is present
+      key = (extract.keys & [:url, :hashtag, :screen_name, :cashtag]).first
+
+      new_indices = [
+        old_to_new_index.find_index(extract[:indices].first),
+        old_to_new_index.find_index(extract[:indices].last),
+      ]
+
+      has_prefix_char = [:hashtag, :screen_name, :cashtag].include?(key)
+      value_indices = [
+        new_indices.first + (has_prefix_char ? 1 : 0), # account for #, @ or $
+        new_indices.last - 1,
+      ]
+
+      next extract.merge(
+        :indices => new_indices,
+        key => text[value_indices.first..value_indices.last]
+      )
+    end
+
+    standard = Extractor.extract_entities_with_indices(text, options)
+
+    Extractor.remove_overlapping_entities(special + standard)
+  end
+
   def link_to_url(entity, options = {})
     url        = Addressable::URI.parse(entity[:url])
     html_attrs = { target: '_blank', rel: 'nofollow noopener' }
diff --git a/app/lib/ostatus/activity/base.rb b/app/lib/ostatus/activity/base.rb
index c5933f3ad..db70f1998 100644
--- a/app/lib/ostatus/activity/base.rb
+++ b/app/lib/ostatus/activity/base.rb
@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 class OStatus::Activity::Base
+  include Redisable
+
   def initialize(xml, account = nil, **options)
     @xml     = xml
     @account = account
@@ -66,8 +68,4 @@ class OStatus::Activity::Base
       Status.find_by(uri: uri)
     end
   end
-
-  def redis
-    Redis.current
-  end
 end
diff --git a/app/lib/potential_friendship_tracker.rb b/app/lib/potential_friendship_tracker.rb
index 017a9748d..188aa4a27 100644
--- a/app/lib/potential_friendship_tracker.rb
+++ b/app/lib/potential_friendship_tracker.rb
@@ -11,6 +11,8 @@ class PotentialFriendshipTracker
   }.freeze
 
   class << self
+    include Redisable
+
     def record(account_id, target_account_id, action)
       return if account_id == target_account_id
 
@@ -31,11 +33,5 @@ class PotentialFriendshipTracker
       return [] if account_ids.empty?
       Account.searchable.where(id: account_ids)
     end
-
-    private
-
-    def redis
-      Redis.current
-    end
   end
 end