Fix emojify() by generating a mapping to existing Twemoji files (#5080)

A new rake task emojis:generate downloads a full list of valid
unicode sequences from unicode.org and checks it against existing
Twemoji files, finally generating a map from each sequence to the
existing file (e.g. when there's multiple ways an emoji can be
expressed). The map is dumped into app/javascript/mastodon/emoji_map.json

That file is loaded by emojione_light.js (now a misnomer) which
decorates it further with shortcodes taken from emoji-mart's index.

1 files changed, 56 insertions, 0 deletions

diff --git a/lib/tasks/emojis.rake b/lib/tasks/emojis.rake
new file mode 100644
index 000000000..cd5e30e96
--- /dev/null
+++ b/lib/tasks/emojis.rake
@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+def codepoints_to_filename(codepoints)
+  codepoints.downcase.gsub(/\A[0]+/, '').tr(' ', '-')
+end
+
+def codepoints_to_unicode(codepoints)
+  if codepoints.include?(' ')
+    codepoints.split(' ').map(&:hex).pack('U*')
+  else
+    [codepoints.hex].pack('U')
+  end
+end
+
+namespace :emojis do
+  desc 'Generate a unicode to filename mapping'
+  task :generate do
+    source = 'http://www.unicode.org/Public/emoji/5.0/emoji-test.txt'
+    codes  = []
+    dest   = Rails.root.join('app', 'javascript', 'mastodon', 'emoji_map.json')
+
+    puts "Downloading emojos from source... (#{source})"
+
+    HTTP.get(source).to_s.split("\n").each do |line|
+      next if line.start_with? '#'
+      parts = line.split(';').map(&:strip)
+      next if parts.size < 2
+      codes << [parts[0], parts[1].start_with?('fully-qualified')]
+    end
+
+    grouped_codes = codes.reduce([]) do |agg, current|
+      if current[1]
+        agg << [current[0]]
+      else
+        agg.last << current[0]
+        agg
+      end
+    end
+
+    existence_maps = grouped_codes.map { |c| c.map { |cc| [cc, File.exist?(Rails.root.join('public', 'emoji', codepoints_to_filename(cc) + '.svg'))] }.to_h }
+    map = {}
+
+    existence_maps.each do |group|
+      existing_one = group.key(true)
+
+      group.each_key do |key|
+        map[codepoints_to_unicode(key)] = codepoints_to_filename(existing_one)
+      end
+    end
+
+    map = map.sort { |a, b| a[0].size <=> b[0].size }.to_h
+
+    File.write(dest, Oj.dump(map))
+    puts "Wrote emojo to destination! (#{dest})"
+  end
+end