Change how hashtags are normalized (#18795)

* Change how hashtags are normalized

* Fix tests

1 files changed, 29 insertions, 0 deletions

diff --git a/spec/lib/hashtag_normalizer_spec.rb b/spec/lib/hashtag_normalizer_spec.rb
new file mode 100644
index 000000000..fbb9f37c0
--- /dev/null
+++ b/spec/lib/hashtag_normalizer_spec.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+describe HashtagNormalizer do
+  subject { described_class.new }
+
+  describe '#normalize' do
+    it 'converts full-width Latin characters into basic Latin characters' do
+      expect(subject.normalize('Ｓｙｎｔｈｗａｖｅ')).to eq 'synthwave'
+    end
+
+    it 'converts half-width Katakana into Kana characters' do
+      expect(subject.normalize('ｼｰｻｲﾄﾞﾗｲﾅｰ')).to eq 'シーサイドライナー'
+    end
+
+    it 'converts modified Latin characters into basic Latin characters' do
+      expect(subject.normalize('BLÅHAJ')).to eq 'blahaj'
+    end
+
+    it 'strips out invalid characters' do
+      expect(subject.normalize('#foo')).to eq 'foo'
+    end
+
+    it 'keeps valid characters' do
+      expect(subject.normalize('a·b')).to eq 'a·b'
+    end
+  end
+end