From e7aa2be828f6a632dadd5c41e2364cea91ddbb2c Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Wed, 13 Jul 2022 15:03:28 +0200
Subject: Change how hashtags are normalized (#18795)

* Change how hashtags are normalized

* Fix tests
---
 spec/lib/hashtag_normalizer_spec.rb | 29 +++++++++++++++++++++++++++++
 spec/models/tag_spec.rb             |  8 ++++----
 2 files changed, 33 insertions(+), 4 deletions(-)
 create mode 100644 spec/lib/hashtag_normalizer_spec.rb

(limited to 'spec')

diff --git a/spec/lib/hashtag_normalizer_spec.rb b/spec/lib/hashtag_normalizer_spec.rb
new file mode 100644
index 000000000..fbb9f37c0
--- /dev/null
+++ b/spec/lib/hashtag_normalizer_spec.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require 'rails_helper'
+
+describe HashtagNormalizer do
+  subject { described_class.new }
+
+  describe '#normalize' do
+    it 'converts full-width Latin characters into basic Latin characters' do
+      expect(subject.normalize('Ｓｙｎｔｈｗａｖｅ')).to eq 'synthwave'
+    end
+
+    it 'converts half-width Katakana into Kana characters' do
+      expect(subject.normalize('ｼｰｻｲﾄﾞﾗｲﾅｰ')).to eq 'シーサイドライナー'
+    end
+
+    it 'converts modified Latin characters into basic Latin characters' do
+      expect(subject.normalize('BLÅHAJ')).to eq 'blahaj'
+    end
+
+    it 'strips out invalid characters' do
+      expect(subject.normalize('#foo')).to eq 'foo'
+    end
+
+    it 'keeps valid characters' do
+      expect(subject.normalize('a·b')).to eq 'a·b'
+    end
+  end
+end
diff --git a/spec/models/tag_spec.rb b/spec/models/tag_spec.rb
index 3949dbce5..b16f99a79 100644
--- a/spec/models/tag_spec.rb
+++ b/spec/models/tag_spec.rb
@@ -91,7 +91,7 @@ RSpec.describe Tag, type: :model do
       upcase_string   = 'abcABCａｂｃＡＢＣやゆよ'
       downcase_string = 'abcabcａｂｃａｂｃやゆよ';
 
-      tag = Fabricate(:tag, name: downcase_string)
+      tag = Fabricate(:tag, name: HashtagNormalizer.new.normalize(downcase_string))
       expect(Tag.find_normalized(upcase_string)).to eq tag
     end
   end
@@ -101,12 +101,12 @@ RSpec.describe Tag, type: :model do
       upcase_string   = 'abcABCａｂｃＡＢＣやゆよ'
       downcase_string = 'abcabcａｂｃａｂｃやゆよ';
 
-      tag = Fabricate(:tag, name: downcase_string)
+      tag = Fabricate(:tag, name: HashtagNormalizer.new.normalize(downcase_string))
       expect(Tag.matches_name(upcase_string)).to eq [tag]
     end
 
     it 'uses the LIKE operator' do
-      expect(Tag.matches_name('100%abc').to_sql).to eq %q[SELECT "tags".* FROM "tags" WHERE LOWER("tags"."name") LIKE LOWER('100\\%abc%')]
+      expect(Tag.matches_name('100%abc').to_sql).to eq %q[SELECT "tags".* FROM "tags" WHERE LOWER("tags"."name") LIKE LOWER('100abc%')]
     end
   end
 
@@ -115,7 +115,7 @@ RSpec.describe Tag, type: :model do
       upcase_string   = 'abcABCａｂｃＡＢＣやゆよ'
       downcase_string = 'abcabcａｂｃａｂｃやゆよ';
 
-      tag = Fabricate(:tag, name: downcase_string)
+      tag = Fabricate(:tag, name: HashtagNormalizer.new.normalize(downcase_string))
       expect(Tag.matching_name(upcase_string)).to eq [tag]
     end
   end
-- 
cgit