diff options
Diffstat (limited to 'app/lib')
| -rw-r--r-- | app/lib/activitypub/activity/create.rb | 2 | ||||
| -rw-r--r-- | app/lib/extractor.rb | 3 | ||||
| -rw-r--r-- | app/lib/feed_manager.rb | 73 | ||||
| -rw-r--r-- | app/lib/formatter.rb | 26 | ||||
| -rw-r--r-- | app/lib/language_detector.rb | 31 | ||||
| -rw-r--r-- | app/lib/themes.rb | 9 |
6 files changed, 96 insertions, 48 deletions
diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index 376684c00..66e4f7c5e 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -173,7 +173,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity end def language_from_content - return nil unless language_map? + return LanguageDetector.instance.detect(text_from_content, @account) unless language_map? @object['contentMap'].keys.first end diff --git a/app/lib/extractor.rb b/app/lib/extractor.rb index 957364293..738ec89a0 100644 --- a/app/lib/extractor.rb +++ b/app/lib/extractor.rb @@ -5,7 +5,8 @@ module Extractor module_function - def extract_mentions_or_lists_with_indices(text) # :yields: username, list_slug, start, end + # :yields: username, list_slug, start, end + def extract_mentions_or_lists_with_indices(text) return [] unless text =~ Twitter::Regex[:at_signs] possible_entries = [] diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb index a99606e1b..5d7f47c6f 100644 --- a/app/lib/feed_manager.rb +++ b/app/lib/feed_manager.rb @@ -26,34 +26,42 @@ class FeedManager end end - def push(timeline_type, account, status) - return false unless add_to_feed(timeline_type, account, status) - - trim(timeline_type, account.id) - - PushUpdateWorker.perform_async(account.id, status.id) if push_update_required?(timeline_type, account.id) - + def push_to_home(account, status) + return false unless add_to_feed(:home, account.id, status) + trim(:home, account.id) + PushUpdateWorker.perform_async(account.id, status.id, "timeline:#{account.id}") if push_update_required?("timeline:#{account.id}") true end - def unpush(timeline_type, account, status) - return false unless remove_from_feed(timeline_type, account, status) + def unpush_from_home(account, status) + return false unless remove_from_feed(:home, account.id, status) + Redis.current.publish("timeline:#{account.id}", Oj.dump(event: :delete, payload: status.id.to_s)) + true + end - payload = Oj.dump(event: :delete, payload: status.id.to_s) - Redis.current.publish("timeline:#{account.id}", payload) + def push_to_list(list, status) + return false unless add_to_feed(:list, list.id, status) + trim(:list, list.id) + PushUpdateWorker.perform_async(list.account_id, status.id, "timeline:list:#{list.id}") if push_update_required?("timeline:list:#{list.id}") + true + end + def unpush_from_list(list, status) + return false unless remove_from_feed(:list, list.id, status) + Redis.current.publish("timeline:list:#{list.id}", Oj.dump(event: :delete, payload: status.id.to_s)) true end def trim(type, account_id) timeline_key = key(type, account_id) - reblog_key = key(type, account_id, 'reblogs') + reblog_key = key(type, account_id, 'reblogs') + # Remove any items past the MAX_ITEMS'th entry in our feed redis.zremrangebyrank(timeline_key, '0', (-(FeedManager::MAX_ITEMS + 1)).to_s) # Get the score of the REBLOG_FALLOFF'th item in our feed, and stop # tracking anything after it for deduplication purposes. - falloff_rank = FeedManager::REBLOG_FALLOFF - 1 + falloff_rank = FeedManager::REBLOG_FALLOFF - 1 falloff_range = redis.zrevrange(timeline_key, falloff_rank, falloff_rank, with_scores: true) falloff_score = falloff_range&.first&.last&.to_i || 0 @@ -69,10 +77,6 @@ class FeedManager end end - def push_update_required?(timeline_type, account_id) - timeline_type != :home || redis.get("subscribed:timeline:#{account_id}").present? - end - def merge_into_timeline(from_account, into_account) timeline_key = key(:home, into_account.id) query = from_account.statuses.limit(FeedManager::MAX_ITEMS / 4) @@ -84,28 +88,28 @@ class FeedManager query.each do |status| next if status.direct_visibility? || filter?(:home, status, into_account) - add_to_feed(:home, into_account, status) + add_to_feed(:home, into_account.id, status) end trim(:home, into_account.id) end def unmerge_from_timeline(from_account, into_account) - timeline_key = key(:home, into_account.id) + timeline_key = key(:home, into_account.id) oldest_home_score = redis.zrange(timeline_key, 0, 0, with_scores: true)&.first&.last&.to_i || 0 from_account.statuses.select('id, reblog_of_id').where('id > ?', oldest_home_score).reorder(nil).find_each do |status| - remove_from_feed(:home, into_account, status) + remove_from_feed(:home, into_account.id, status) end end def clear_from_timeline(account, target_account) - timeline_key = key(:home, account.id) + timeline_key = key(:home, account.id) timeline_status_ids = redis.zrange(timeline_key, 0, -1) - target_statuses = Status.where(id: timeline_status_ids, account: target_account) + target_statuses = Status.where(id: timeline_status_ids, account: target_account) target_statuses.each do |status| - unpush(:home, account, status) + unpush_from_home(account, status) end end @@ -122,7 +126,7 @@ class FeedManager statuses.each do |status| next if filter_from_home?(status, account) - added += 1 if add_to_feed(:home, account, status) + added += 1 if add_to_feed(:home, account.id, status) end break unless added.zero? @@ -137,6 +141,10 @@ class FeedManager Redis.current end + def push_update_required?(timeline_id) + redis.exists("subscribed:#{timeline_id}") + end + def filter_from_home?(status, receiver_id) return false if receiver_id == status.account_id return true if status.reply? && (status.in_reply_to_id.nil? || status.in_reply_to_account_id.nil?) @@ -207,9 +215,9 @@ class FeedManager # added, and false if it was not added to the feed. Note that this is # an internal helper: callers must call trim or push updates if # either action is appropriate. - def add_to_feed(timeline_type, account, status) - timeline_key = key(timeline_type, account.id) - reblog_key = key(timeline_type, account.id, 'reblogs') + def add_to_feed(timeline_type, account_id, status) + timeline_key = key(timeline_type, account_id) + reblog_key = key(timeline_type, account_id, 'reblogs') if status.reblog? # If the original status or a reblog of it is within @@ -220,6 +228,7 @@ class FeedManager return false if !rank.nil? && rank < FeedManager::REBLOG_FALLOFF reblog_rank = redis.zrevrank(reblog_key, status.reblog_of_id) + if reblog_rank.nil? # This is not something we've already seen reblogged, so we # can just add it to the feed (and note that we're @@ -230,7 +239,7 @@ class FeedManager # Another reblog of the same status was already in the # REBLOG_FALLOFF most recent statuses, so we note that this # is an "extra" reblog, by storing it in reblog_set_key. - reblog_set_key = key(timeline_type, account.id, "reblogs:#{status.reblog_of_id}") + reblog_set_key = key(timeline_type, account_id, "reblogs:#{status.reblog_of_id}") redis.sadd(reblog_set_key, status.id) return false end @@ -245,8 +254,8 @@ class FeedManager # with reblogs, and returning true if a status was removed. As with # `add_to_feed`, this does not trigger push updates, so callers must # do so if appropriate. - def remove_from_feed(timeline_type, account, status) - timeline_key = key(timeline_type, account.id) + def remove_from_feed(timeline_type, account_id, status) + timeline_key = key(timeline_type, account_id) if status.reblog? # 1. If the reblogging status is not in the feed, stop. @@ -254,7 +263,7 @@ class FeedManager return false if status_rank.nil? # 2. Remove reblog from set of this status's reblogs. - reblog_set_key = key(timeline_type, account.id, "reblogs:#{status.reblog_of_id}") + reblog_set_key = key(timeline_type, account_id, "reblogs:#{status.reblog_of_id}") redis.srem(reblog_set_key, status.id) # 3. Re-insert another reblog or original into the feed if one @@ -269,7 +278,7 @@ class FeedManager # (outside conditional) else # If the original is getting deleted, no use for reblog references - redis.del(key(timeline_type, account.id, "reblogs:#{status.id}")) + redis.del(key(timeline_type, account_id, "reblogs:#{status.id}")) end redis.zrem(timeline_key, status.id) diff --git a/app/lib/formatter.rb b/app/lib/formatter.rb index 57f105da7..733a1c4b7 100644 --- a/app/lib/formatter.rb +++ b/app/lib/formatter.rb @@ -89,20 +89,28 @@ class Formatter end end + def count_tag_nesting(tag) + if tag[1] == '/' then -1 + elsif tag[-2] == '/' then 0 + else 1 + end + end + def encode_custom_emojis(html, emojis) return html if emojis.empty? emoji_map = emojis.map { |e| [e.shortcode, full_asset_url(e.image.url(:static))] }.to_h i = -1 - inside_tag = false + tag_open_index = nil inside_shortname = false shortname_start_index = -1 + invisible_depth = 0 while i + 1 < html.size i += 1 - if inside_shortname && html[i] == ':' + if invisible_depth.zero? && inside_shortname && html[i] == ':' shortcode = html[shortname_start_index + 1..i - 1] emoji = emoji_map[shortcode] @@ -116,12 +124,18 @@ class Formatter end inside_shortname = false - elsif inside_tag && html[i] == '>' - inside_tag = false + elsif tag_open_index && html[i] == '>' + tag = html[tag_open_index..i] + tag_open_index = nil + if invisible_depth.positive? + invisible_depth += count_tag_nesting(tag) + elsif tag == '<span class="invisible">' + invisible_depth = 1 + end elsif html[i] == '<' - inside_tag = true + tag_open_index = i inside_shortname = false - elsif !inside_tag && html[i] == ':' + elsif !tag_open_index && html[i] == ':' inside_shortname = true shortname_start_index = i end diff --git a/app/lib/language_detector.rb b/app/lib/language_detector.rb index a42460e10..c6f52f0c7 100644 --- a/app/lib/language_detector.rb +++ b/app/lib/language_detector.rb @@ -38,12 +38,31 @@ class LanguageDetector end def simplify_text(text) - text.dup.tap do |new_text| - new_text.gsub!(FetchLinkCardService::URL_PATTERN, '') - new_text.gsub!(Account::MENTION_RE, '') - new_text.gsub!(Tag::HASHTAG_RE, '') - new_text.gsub!(/\s+/, ' ') - end + new_text = remove_html(text) + new_text.gsub!(FetchLinkCardService::URL_PATTERN, '') + new_text.gsub!(Account::MENTION_RE, '') + new_text.gsub!(Tag::HASHTAG_RE, '') + new_text.gsub!(/:#{CustomEmoji::SHORTCODE_RE_FRAGMENT}:/, '') + new_text.gsub!(/\s+/, ' ') + new_text + end + + def new_scrubber + scrubber = Rails::Html::PermitScrubber.new + scrubber.tags = %w(br p) + scrubber + end + + def scrubber + @scrubber ||= new_scrubber + end + + def remove_html(text) + text = Loofah.fragment(text).scrub!(scrubber).to_s + text.gsub!('<br>', "\n") + text.gsub!('</p><p>', "\n\n") + text.gsub!(/(^<p>|<\/p>$)/, '') + text end def default_locale(account) diff --git a/app/lib/themes.rb b/app/lib/themes.rb index 2dd188297..f7ec22fd2 100644 --- a/app/lib/themes.rb +++ b/app/lib/themes.rb @@ -10,13 +10,18 @@ class Themes result = Hash.new Dir.glob(Rails.root.join('app', 'javascript', 'themes', '*', 'theme.yml')) do |path| data = YAML.load_file(path) - if data['pack'] && data['name'] - result[data['name']] = data + name = File.basename(File.dirname(path)) + if data['pack'] + result[name] = data end end @conf = result end + def get(name) + @conf[name] + end + def names @conf.keys end |