Fix IDN mentions not being processed, IDN domains not being rendered (#12715)

This changes the REST API to return unicode domains in the `acct`
attribute instead of punycode, and to render unicode instead of
punycode on public HTML pages as well.

Fix #7812, fix #12246

4 files changed, 20 insertions, 3 deletions

diff --git a/app/helpers/accounts_helper.rb b/app/helpers/accounts_helper.rb
index 99815be7b..53939adfc 100644
--- a/app/helpers/accounts_helper.rb
+++ b/app/helpers/accounts_helper.rb
@@ -13,7 +13,7 @@ module AccountsHelper
     if account.local?
       "@#{account.acct}@#{Rails.configuration.x.local_domain}"
     else
-      "@#{account.acct}"
+      "@#{account.pretty_acct}"
     end
   end
 
diff --git a/app/models/account.rb b/app/models/account.rb
index 884332e5a..feaf273c1 100644
--- a/app/models/account.rb
+++ b/app/models/account.rb
@@ -50,7 +50,7 @@
 
 class Account < ApplicationRecord
   USERNAME_RE = /[a-z0-9_]+([a-z0-9_\.-]+[a-z0-9_]+)?/i
-  MENTION_RE  = /(?<=^|[^\/[:word:]])@((#{USERNAME_RE})(?:@[a-z0-9\.\-]+[a-z0-9]+)?)/i
+  MENTION_RE  = /(?<=^|[^\/[:word:]])@((#{USERNAME_RE})(?:@[[:word:]\.\-]+[a-z0-9]+)?)/i
 
   include AccountAssociations
   include AccountAvatar
@@ -164,6 +164,10 @@ class Account < ApplicationRecord
     local? ? username : "#{username}@#{domain}"
   end
 
+  def pretty_acct
+    local? ? username : "#{username}@#{Addressable::IDNA.to_unicode(domain)}"
+  end
+
   def local_username_and_domain
     "#{username}@#{Rails.configuration.x.local_domain}"
   end
diff --git a/app/serializers/rest/account_serializer.rb b/app/serializers/rest/account_serializer.rb
index 5fec75673..fd361a34c 100644
--- a/app/serializers/rest/account_serializer.rb
+++ b/app/serializers/rest/account_serializer.rb
@@ -24,6 +24,10 @@ class REST::AccountSerializer < ActiveModel::Serializer
     object.id.to_s
   end
 
+  def acct
+    object.pretty_acct
+  end
+
   def note
     Formatter.instance.simplified_format(object)
   end
diff --git a/app/services/process_mentions_service.rb b/app/services/process_mentions_service.rb
index 2f7a9e985..b2d868165 100644
--- a/app/services/process_mentions_service.rb
+++ b/app/services/process_mentions_service.rb
@@ -14,7 +14,16 @@ class ProcessMentionsService < BaseService
     mentions = []
 
     status.text = status.text.gsub(Account::MENTION_RE) do |match|
-      username, domain  = Regexp.last_match(1).split('@')
+      username, domain = Regexp.last_match(1).split('@')
+
+      domain = begin
+        if TagManager.instance.local_domain?(domain)
+          nil
+        else
+          TagManager.instance.normalize_domain(domain)
+        end
+      end
+
       mentioned_account = Account.find_remote(username, domain)
 
       if mention_undeliverable?(mentioned_account)