diff options
Diffstat (limited to 'app/lib')
| -rw-r--r-- | app/lib/account_reach_finder.rb | 25 | ||||
| -rw-r--r-- | app/lib/activitypub/activity.rb | 14 | ||||
| -rw-r--r-- | app/lib/activitypub/activity/announce.rb | 38 | ||||
| -rw-r--r-- | app/lib/activitypub/activity/create.rb | 43 | ||||
| -rw-r--r-- | app/lib/activitypub/activity/delete.rb | 62 | ||||
| -rw-r--r-- | app/lib/activitypub/activity/flag.rb | 2 | ||||
| -rw-r--r-- | app/lib/admin/system_check/sidekiq_process_check.rb | 1 | ||||
| -rw-r--r-- | app/lib/application_extension.rb | 4 | ||||
| -rw-r--r-- | app/lib/delivery_failure_tracker.rb | 26 | ||||
| -rw-r--r-- | app/lib/feed_manager.rb | 30 | ||||
| -rw-r--r-- | app/lib/formatter.rb | 35 | ||||
| -rw-r--r-- | app/lib/potential_friendship_tracker.rb | 6 | ||||
| -rw-r--r-- | app/lib/spam_check.rb | 198 | ||||
| -rw-r--r-- | app/lib/status_reach_finder.rb | 29 | ||||
| -rw-r--r-- | app/lib/tag_manager.rb | 8 | ||||
| -rw-r--r-- | app/lib/video_metadata_extractor.rb | 54 |
16 files changed, 266 insertions, 309 deletions
diff --git a/app/lib/account_reach_finder.rb b/app/lib/account_reach_finder.rb new file mode 100644 index 000000000..706ce8c1f --- /dev/null +++ b/app/lib/account_reach_finder.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +class AccountReachFinder + def initialize(account) + @account = account + end + + def inboxes + (followers_inboxes + reporters_inboxes + relay_inboxes).uniq + end + + private + + def followers_inboxes + @account.followers.inboxes + end + + def reporters_inboxes + Account.where(id: @account.targeted_reports.select(:account_id)).inboxes + end + + def relay_inboxes + Relay.enabled.pluck(:inbox_url) + end +end diff --git a/app/lib/activitypub/activity.rb b/app/lib/activitypub/activity.rb index 2b5d3ffc2..3baee4ca4 100644 --- a/app/lib/activitypub/activity.rb +++ b/app/lib/activitypub/activity.rb @@ -144,7 +144,7 @@ class ActivityPub::Activity end def delete_later!(uri) - redis.setex("delete_upon_arrival:#{@account.id}:#{uri}", 6.hours.seconds, uri) + redis.setex("delete_upon_arrival:#{@account.id}:#{uri}", 6.hours.seconds, true) end def status_from_object @@ -210,12 +210,22 @@ class ActivityPub::Activity end end - def lock_or_return(key, expire_after = 7.days.seconds) + def lock_or_return(key, expire_after = 2.hours.seconds) yield if redis.set(key, true, nx: true, ex: expire_after) ensure redis.del(key) end + def lock_or_fail(key) + RedisLock.acquire({ redis: Redis.current, key: key }) do |lock| + if lock.acquired? + yield + else + raise Mastodon::RaceConditionError + end + end + end + def fetch? !@options[:delivery] end diff --git a/app/lib/activitypub/activity/announce.rb b/app/lib/activitypub/activity/announce.rb index ae8b2db75..9f778ffb9 100644 --- a/app/lib/activitypub/activity/announce.rb +++ b/app/lib/activitypub/activity/announce.rb @@ -4,29 +4,29 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity def perform return reject_payload! if delete_arrived_first?(@json['id']) || !related_to_local_activity? - RedisLock.acquire(lock_options) do |lock| - if lock.acquired? - original_status = status_from_object + lock_or_fail("announce:#{@object['id']}") do + original_status = status_from_object - return reject_payload! if original_status.nil? || !announceable?(original_status) + return reject_payload! if original_status.nil? || !announceable?(original_status) - @status = Status.find_by(account: @account, reblog: original_status) + @status = Status.find_by(account: @account, reblog: original_status) - return @status unless @status.nil? + return @status unless @status.nil? - @status = Status.create!( - account: @account, - reblog: original_status, - uri: @json['id'], - created_at: @json['published'], - override_timestamps: @options[:override_timestamps], - visibility: visibility_from_audience - ) + @status = Status.create!( + account: @account, + reblog: original_status, + uri: @json['id'], + created_at: @json['published'], + override_timestamps: @options[:override_timestamps], + visibility: visibility_from_audience + ) - distribute(@status) - else - raise Mastodon::RaceConditionError + original_status.tags.each do |tag| + tag.use!(@account) end + + distribute(@status) end @status @@ -69,8 +69,4 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity def reblog_of_local_status? status_from_uri(object_uri)&.account&.local? end - - def lock_options - { redis: Redis.current, key: "announce:#{@object['id']}" } - end end diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index f10fc5f43..763c417f9 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -45,19 +45,15 @@ class ActivityPub::Activity::Create < ActivityPub::Activity def create_status return reject_payload! if unsupported_object_type? || invalid_origin?(object_uri) || tombstone_exists? || !related_to_local_activity? - RedisLock.acquire(lock_options) do |lock| - if lock.acquired? - return if delete_arrived_first?(object_uri) || poll_vote? # rubocop:disable Lint/NonLocalExitFromIterator + lock_or_fail("create:#{object_uri}") do + return if delete_arrived_first?(object_uri) || poll_vote? # rubocop:disable Lint/NonLocalExitFromIterator - @status = find_existing_status + @status = find_existing_status - if @status.nil? - process_status - elsif @options[:delivered_to_account_id].present? - postprocess_audience_and_deliver - end - else - raise Mastodon::RaceConditionError + if @status.nil? + process_status + elsif @options[:delivered_to_account_id].present? + postprocess_audience_and_deliver end end @@ -88,7 +84,6 @@ class ActivityPub::Activity::Create < ActivityPub::Activity resolve_thread(@status) fetch_replies(@status) - check_for_spam distribute(@status) forward_for_reply end @@ -169,7 +164,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity def attach_tags(status) @tags.each do |tag| status.tags << tag - TrendingTags.record_use!(tag, status.account, status.created_at) if status.public_visibility? + tag.use!(@account, status: status, at_time: status.created_at) if status.public_visibility? end @mentions.each do |mention| @@ -314,13 +309,9 @@ class ActivityPub::Activity::Create < ActivityPub::Activity poll = replied_to_status.preloadable_poll already_voted = true - RedisLock.acquire(poll_lock_options) do |lock| - if lock.acquired? - already_voted = poll.votes.where(account: @account).exists? - poll.votes.create!(account: @account, choice: poll.options.index(@object['name']), uri: object_uri) - else - raise Mastodon::RaceConditionError - end + lock_or_fail("vote:#{replied_to_status.poll_id}:#{@account.id}") do + already_voted = poll.votes.where(account: @account).exists? + poll.votes.create!(account: @account, choice: poll.options.index(@object['name']), uri: object_uri) end increment_voters_count! unless already_voted @@ -498,10 +489,6 @@ class ActivityPub::Activity::Create < ActivityPub::Activity Tombstone.exists?(uri: object_uri) end - def check_for_spam - SpamCheck.perform(@status) - end - def forward_for_reply return unless @status.distributable? && @json['signature'].present? && reply_to_local? @@ -519,12 +506,4 @@ class ActivityPub::Activity::Create < ActivityPub::Activity poll.reload retry end - - def lock_options - { redis: Redis.current, key: "create:#{object_uri}" } - end - - def poll_lock_options - { redis: Redis.current, key: "vote:#{replied_to_status.poll_id}:#{@account.id}" } - end end diff --git a/app/lib/activitypub/activity/delete.rb b/app/lib/activitypub/activity/delete.rb index 2e5293b83..801647cf7 100644 --- a/app/lib/activitypub/activity/delete.rb +++ b/app/lib/activitypub/activity/delete.rb @@ -20,33 +20,35 @@ class ActivityPub::Activity::Delete < ActivityPub::Activity def delete_note return if object_uri.nil? - unless invalid_origin?(object_uri) - RedisLock.acquire(lock_options) { |_lock| delete_later!(object_uri) } - Tombstone.find_or_create_by(uri: object_uri, account: @account) - end + lock_or_return("delete_status_in_progress:#{object_uri}", 5.minutes.seconds) do + unless invalid_origin?(object_uri) + # This lock ensures a concurrent `ActivityPub::Activity::Create` either + # does not create a status at all, or has finished saving it to the + # database before we try to load it. + # Without the lock, `delete_later!` could be called after `delete_arrived_first?` + # and `Status.find` before `Status.create!` + lock_or_fail("create:#{object_uri}") { delete_later!(object_uri) } - @status = Status.find_by(uri: object_uri, account: @account) - @status ||= Status.find_by(uri: @object['atomUri'], account: @account) if @object.is_a?(Hash) && @object['atomUri'].present? + Tombstone.find_or_create_by(uri: object_uri, account: @account) + end - return if @status.nil? + @status = Status.find_by(uri: object_uri, account: @account) + @status ||= Status.find_by(uri: @object['atomUri'], account: @account) if @object.is_a?(Hash) && @object['atomUri'].present? - if @status.distributable? - forward_for_reply - forward_for_reblogs - end + return if @status.nil? - delete_now! + forward! if @json['signature'].present? && @status.distributable? + delete_now! + end end - def forward_for_reblogs - return if @json['signature'].blank? - - rebloggers_ids = @status.reblogs.includes(:account).references(:account).merge(Account.local).pluck(:account_id) - inboxes = Account.where(id: ::Follow.where(target_account_id: rebloggers_ids).select(:account_id)).inboxes - [@account.preferred_inbox_url] + def rebloggers_ids + return @rebloggers_ids if defined?(@rebloggers_ids) + @rebloggers_ids = @status.reblogs.includes(:account).references(:account).merge(Account.local).pluck(:account_id) + end - ActivityPub::LowPriorityDeliveryWorker.push_bulk(inboxes) do |inbox_url| - [payload, rebloggers_ids.first, inbox_url] - end + def inboxes_for_reblogs + Account.where(id: ::Follow.where(target_account_id: rebloggers_ids).select(:account_id)).inboxes end def replied_to_status @@ -58,13 +60,19 @@ class ActivityPub::Activity::Delete < ActivityPub::Activity !replied_to_status.nil? && replied_to_status.account.local? end - def forward_for_reply - return unless @json['signature'].present? && reply_to_local? + def inboxes_for_reply + replied_to_status.account.followers.inboxes + end + + def forward! + inboxes = inboxes_for_reblogs + inboxes += inboxes_for_reply if reply_to_local? + inboxes -= [@account.preferred_inbox_url] - inboxes = replied_to_status.account.followers.inboxes - [@account.preferred_inbox_url] + sender_id = reply_to_local? ? replied_to_status.account_id : rebloggers_ids.first - ActivityPub::LowPriorityDeliveryWorker.push_bulk(inboxes) do |inbox_url| - [payload, replied_to_status.account_id, inbox_url] + ActivityPub::LowPriorityDeliveryWorker.push_bulk(inboxes.uniq) do |inbox_url| + [payload, sender_id, inbox_url] end end @@ -75,8 +83,4 @@ class ActivityPub::Activity::Delete < ActivityPub::Activity def payload @payload ||= Oj.dump(@json) end - - def lock_options - { redis: Redis.current, key: "create:#{object_uri}" } - end end diff --git a/app/lib/activitypub/activity/flag.rb b/app/lib/activitypub/activity/flag.rb index 8dfc76f0a..b0443849a 100644 --- a/app/lib/activitypub/activity/flag.rb +++ b/app/lib/activitypub/activity/flag.rb @@ -10,6 +10,8 @@ class ActivityPub::Activity::Flag < ActivityPub::Activity target_accounts.each do |target_account| target_statuses = target_statuses_by_account[target_account.id] + next if target_account.suspended? + ReportService.new.call( @account, target_account, diff --git a/app/lib/admin/system_check/sidekiq_process_check.rb b/app/lib/admin/system_check/sidekiq_process_check.rb index c44d86c44..22446edaf 100644 --- a/app/lib/admin/system_check/sidekiq_process_check.rb +++ b/app/lib/admin/system_check/sidekiq_process_check.rb @@ -7,7 +7,6 @@ class Admin::SystemCheck::SidekiqProcessCheck < Admin::SystemCheck::BaseCheck mailers pull scheduler - ingress ).freeze def pass? diff --git a/app/lib/application_extension.rb b/app/lib/application_extension.rb index 1d80b8c6d..e61cd0721 100644 --- a/app/lib/application_extension.rb +++ b/app/lib/application_extension.rb @@ -4,6 +4,8 @@ module ApplicationExtension extend ActiveSupport::Concern included do - validates :website, url: true, if: :website? + validates :name, length: { maximum: 60 } + validates :website, url: true, length: { maximum: 2_000 }, if: :website? + validates :redirect_uri, length: { maximum: 2_000 } end end diff --git a/app/lib/delivery_failure_tracker.rb b/app/lib/delivery_failure_tracker.rb index 2cd6ef7ad..8907ade4c 100644 --- a/app/lib/delivery_failure_tracker.rb +++ b/app/lib/delivery_failure_tracker.rb @@ -17,6 +17,10 @@ class DeliveryFailureTracker UnavailableDomain.find_by(domain: @host)&.destroy end + def clear_failures! + Redis.current.del(exhausted_deliveries_key) + end + def days Redis.current.scard(exhausted_deliveries_key) || 0 end @@ -25,6 +29,10 @@ class DeliveryFailureTracker !UnavailableDomain.where(domain: @host).exists? end + def exhausted_deliveries_days + Redis.current.smembers(exhausted_deliveries_key).sort.map { |date| Date.new(date.slice(0, 4).to_i, date.slice(4, 2).to_i, date.slice(6, 2).to_i) } + end + alias reset! track_success! class << self @@ -44,6 +52,24 @@ class DeliveryFailureTracker def reset!(url) new(url).reset! end + + def warning_domains + domains = Redis.current.keys(exhausted_deliveries_key_by('*')).map do |key| + key.delete_prefix(exhausted_deliveries_key_by('')) + end + + domains - UnavailableDomain.all.pluck(:domain) + end + + def warning_domains_map + warning_domains.index_with { |domain| Redis.current.scard(exhausted_deliveries_key_by(domain)) } + end + + private + + def exhausted_deliveries_key_by(host) + "exhausted_deliveries:#{host}" + end end private diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb index 90e6652a9..d57508ef9 100644 --- a/app/lib/feed_manager.rb +++ b/app/lib/feed_manager.rb @@ -219,6 +219,36 @@ class FeedManager end end + # Clear all statuses from or mentioning target_account from a list feed + # @param [List] list + # @param [Account] target_account + # @return [void] + def clear_from_list(list, target_account) + timeline_key = key(:list, list.id) + timeline_status_ids = redis.zrange(timeline_key, 0, -1) + statuses = Status.where(id: timeline_status_ids).select(:id, :reblog_of_id, :account_id).to_a + reblogged_ids = Status.where(id: statuses.map(&:reblog_of_id).compact, account: target_account).pluck(:id) + with_mentions_ids = Mention.active.where(status_id: statuses.flat_map { |s| [s.id, s.reblog_of_id] }.compact, account: target_account).pluck(:status_id) + + target_statuses = statuses.select do |status| + status.account_id == target_account.id || reblogged_ids.include?(status.reblog_of_id) || with_mentions_ids.include?(status.id) || with_mentions_ids.include?(status.reblog_of_id) + end + + target_statuses.each do |status| + unpush_from_list(list, status) + end + end + + # Clear all statuses from or mentioning target_account from an account's lists + # @param [Account] account + # @param [Account] target_account + # @return [void] + def clear_from_lists(account, target_account) + List.where(account: account).each do |list| + clear_from_list(list, target_account) + end + end + # Populate home feed of account from scratch # @param [Account] account # @return [void] diff --git a/app/lib/formatter.rb b/app/lib/formatter.rb index 02ebe6f89..b26138642 100644 --- a/app/lib/formatter.rb +++ b/app/lib/formatter.rb @@ -118,7 +118,7 @@ class Formatter end def format_field(account, str, **options) - html = account.local? ? encode_and_link_urls(str, me: true) : reformat(str) + html = account.local? ? encode_and_link_urls(str, me: true, with_domain: true) : reformat(str) html = encode_custom_emojis(html, account.emojis, options[:autoplay]) if options[:custom_emojify] html.html_safe # rubocop:disable Rails/OutputSafety end @@ -187,7 +187,7 @@ class Formatter elsif entity[:hashtag] link_to_hashtag(entity) elsif entity[:screen_name] - link_to_mention(entity, accounts) + link_to_mention(entity, accounts, options) end end end @@ -352,22 +352,37 @@ class Formatter encode(entity[:url]) end - def link_to_mention(entity, linkable_accounts) + def link_to_mention(entity, linkable_accounts, options = {}) acct = entity[:screen_name] - return link_to_account(acct) unless linkable_accounts + return link_to_account(acct, options) unless linkable_accounts - account = linkable_accounts.find { |item| TagManager.instance.same_acct?(item.acct, acct) } - account ? mention_html(account) : "@#{encode(acct)}" + same_username_hits = 0 + account = nil + username, domain = acct.split('@') + domain = nil if TagManager.instance.local_domain?(domain) + + linkable_accounts.each do |item| + same_username = item.username.casecmp(username).zero? + same_domain = item.domain.nil? ? domain.nil? : item.domain.casecmp(domain)&.zero? + + if same_username && !same_domain + same_username_hits += 1 + elsif same_username && same_domain + account = item + end + end + + account ? mention_html(account, with_domain: same_username_hits.positive? || options[:with_domain]) : "@#{encode(acct)}" end - def link_to_account(acct) + def link_to_account(acct, options = {}) username, domain = acct.split('@') domain = nil if TagManager.instance.local_domain?(domain) account = EntityCache.instance.mention(username, domain) - account ? mention_html(account) : "@#{encode(acct)}" + account ? mention_html(account, with_domain: options[:with_domain]) : "@#{encode(acct)}" end def link_to_hashtag(entity) @@ -388,7 +403,7 @@ class Formatter "<a href=\"#{encode(tag_url(tag))}\" class=\"mention hashtag\" rel=\"tag\">#<span>#{encode(tag)}</span></a>" end - def mention_html(account) - "<span class=\"h-card\"><a href=\"#{encode(ActivityPub::TagManager.instance.url_for(account))}\" class=\"u-url mention\">@<span>#{encode(account.username)}</span></a></span>" + def mention_html(account, with_domain: false) + "<span class=\"h-card\"><a href=\"#{encode(ActivityPub::TagManager.instance.url_for(account))}\" class=\"u-url mention\">@<span>#{encode(with_domain ? account.pretty_acct : account.username)}</span></a></span>" end end diff --git a/app/lib/potential_friendship_tracker.rb b/app/lib/potential_friendship_tracker.rb index 188aa4a27..f5bc20346 100644 --- a/app/lib/potential_friendship_tracker.rb +++ b/app/lib/potential_friendship_tracker.rb @@ -27,11 +27,5 @@ class PotentialFriendshipTracker def remove(account_id, target_account_id) redis.zrem("interactions:#{account_id}", target_account_id) end - - def get(account_id, limit: 20, offset: 0) - account_ids = redis.zrevrange("interactions:#{account_id}", offset, limit) - return [] if account_ids.empty? - Account.searchable.where(id: account_ids) - end end end diff --git a/app/lib/spam_check.rb b/app/lib/spam_check.rb deleted file mode 100644 index dcb2db9ca..000000000 --- a/app/lib/spam_check.rb +++ /dev/null @@ -1,198 +0,0 @@ -# frozen_string_literal: true - -class SpamCheck - include Redisable - include ActionView::Helpers::TextHelper - - # Threshold over which two Nilsimsa values are considered - # to refer to the same text - NILSIMSA_COMPARE_THRESHOLD = 95 - - # Nilsimsa doesn't work well on small inputs, so below - # this size, we check only for exact matches with MD5 - NILSIMSA_MIN_SIZE = 10 - - # How long to keep the trail of digests between updates, - # there is no reason to store it forever - EXPIRE_SET_AFTER = 1.week.seconds - - # How many digests to keep in an account's trail. If it's - # too small, spam could rotate around different message templates - MAX_TRAIL_SIZE = 10 - - # How many detected duplicates to allow through before - # considering the message as spam - THRESHOLD = 5 - - def initialize(status) - @account = status.account - @status = status - end - - def skip? - disabled? || already_flagged? || trusted? || no_unsolicited_mentions? || solicited_reply? - end - - def spam? - if insufficient_data? - false - elsif nilsimsa? - digests_over_threshold?('nilsimsa') { |_, other_digest| nilsimsa_compare_value(digest, other_digest) >= NILSIMSA_COMPARE_THRESHOLD } - else - digests_over_threshold?('md5') { |_, other_digest| other_digest == digest } - end - end - - def flag! - auto_report_status! - end - - def remember! - # The scores in sorted sets don't actually have enough bits to hold an exact - # value of our snowflake IDs, so we use it only for its ordering property. To - # get the correct status ID back, we have to save it in the string value - - redis.zadd(redis_key, @status.id, digest_with_algorithm) - redis.zremrangebyrank(redis_key, 0, -(MAX_TRAIL_SIZE + 1)) - redis.expire(redis_key, EXPIRE_SET_AFTER) - end - - def reset! - redis.del(redis_key) - end - - def hashable_text - return @hashable_text if defined?(@hashable_text) - - @hashable_text = @status.text - @hashable_text = remove_mentions(@hashable_text) - @hashable_text = strip_tags(@hashable_text) unless @status.local? - @hashable_text = normalize_unicode(@status.spoiler_text + ' ' + @hashable_text) - @hashable_text = remove_whitespace(@hashable_text) - end - - def insufficient_data? - hashable_text.blank? - end - - def digest - @digest ||= begin - if nilsimsa? - Nilsimsa.new(hashable_text).hexdigest - else - Digest::MD5.hexdigest(hashable_text) - end - end - end - - def digest_with_algorithm - if nilsimsa? - ['nilsimsa', digest, @status.id].join(':') - else - ['md5', digest, @status.id].join(':') - end - end - - class << self - def perform(status) - spam_check = new(status) - - return if spam_check.skip? - - if spam_check.spam? - spam_check.flag! - else - spam_check.remember! - end - end - end - - private - - def disabled? - !Setting.spam_check_enabled - end - - def remove_mentions(text) - return text.gsub(Account::MENTION_RE, '') if @status.local? - - Nokogiri::HTML.fragment(text).tap do |html| - mentions = @status.mentions.map { |mention| ActivityPub::TagManager.instance.url_for(mention.account) } - - html.traverse do |element| - element.unlink if element.name == 'a' && mentions.include?(element['href']) - end - end.to_s - end - - def normalize_unicode(text) - text.unicode_normalize(:nfkc).downcase - end - - def remove_whitespace(text) - text.gsub(/\s+/, ' ').strip - end - - def auto_report_status! - status_ids = Status.where(visibility: %i(public unlisted)).where(id: matching_status_ids).pluck(:id) + [@status.id] if @status.distributable? - ReportService.new.call(Account.representative, @account, status_ids: status_ids, comment: I18n.t('spam_check.spam_detected')) - end - - def already_flagged? - @account.silenced? || @account.targeted_reports.unresolved.where(account_id: -99).exists? - end - - def trusted? - @account.trust_level > Account::TRUST_LEVELS[:untrusted] || (@account.local? && @account.user_staff?) - end - - def no_unsolicited_mentions? - @status.mentions.all? { |mention| mention.silent? || (!@account.local? && !mention.account.local?) || mention.account.following?(@account) } - end - - def solicited_reply? - !@status.thread.nil? && @status.thread.mentions.where(account: @account).exists? - end - - def nilsimsa_compare_value(first, second) - first = [first].pack('H*') - second = [second].pack('H*') - bits = 0 - - 0.upto(31) do |i| - bits += Nilsimsa::POPC[255 & (first[i].ord ^ second[i].ord)].ord - end - - 128 - bits # -128 <= Nilsimsa Compare Value <= 128 - end - - def nilsimsa? - hashable_text.size > NILSIMSA_MIN_SIZE - end - - def other_digests - redis.zrange(redis_key, 0, -1) - end - - def digests_over_threshold?(filter_algorithm) - other_digests.select do |record| - algorithm, other_digest, status_id = record.split(':') - - next unless algorithm == filter_algorithm - - yield algorithm, other_digest, status_id - end.size >= THRESHOLD - end - - def matching_status_ids - if nilsimsa? - other_digests.filter_map { |record| record.split(':')[2] if record.start_with?('nilsimsa') && nilsimsa_compare_value(digest, record.split(':')[1]) >= NILSIMSA_COMPARE_THRESHOLD } - else - other_digests.filter_map { |record| record.split(':')[2] if record.start_with?('md5') && record.split(':')[1] == digest } - end - end - - def redis_key - @redis_key ||= "spam_check:#{@account.id}" - end -end diff --git a/app/lib/status_reach_finder.rb b/app/lib/status_reach_finder.rb index 35b191dad..735d66a4f 100644 --- a/app/lib/status_reach_finder.rb +++ b/app/lib/status_reach_finder.rb @@ -6,11 +6,22 @@ class StatusReachFinder end def inboxes - Account.where(id: reached_account_ids).inboxes + (reached_account_inboxes + followers_inboxes + relay_inboxes).uniq end private + def reached_account_inboxes + # When the status is a reblog, there are no interactions with it + # directly, we assume all interactions are with the original one + + if @status.reblog? + [] + else + Account.where(id: reached_account_ids).inboxes + end + end + def reached_account_ids [ replied_to_account_id, @@ -49,4 +60,20 @@ class StatusReachFinder def replies_account_ids @status.replies.pluck(:account_id) end + + def followers_inboxes + if @status.in_reply_to_local_account? && @status.distributable? + @status.account.followers.or(@status.thread.account.followers).inboxes + else + @status.account.followers.inboxes + end + end + + def relay_inboxes + if @status.public_visibility? + Relay.enabled.pluck(:inbox_url) + else + [] + end + end end diff --git a/app/lib/tag_manager.rb b/app/lib/tag_manager.rb index 29dde128c..a1d12a654 100644 --- a/app/lib/tag_manager.rb +++ b/app/lib/tag_manager.rb @@ -22,14 +22,6 @@ class TagManager uri.normalized_host end - def same_acct?(canonical, needle) - return true if canonical.casecmp(needle).zero? - - username, domain = needle.split('@') - - local_domain?(domain) && canonical.casecmp(username).zero? - end - def local_url?(url) uri = Addressable::URI.parse(url).normalize return false unless uri.host diff --git a/app/lib/video_metadata_extractor.rb b/app/lib/video_metadata_extractor.rb new file mode 100644 index 000000000..03e40f923 --- /dev/null +++ b/app/lib/video_metadata_extractor.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +class VideoMetadataExtractor + attr_reader :duration, :bitrate, :video_codec, :audio_codec, + :colorspace, :width, :height, :frame_rate + + def initialize(path) + @path = path + @metadata = Oj.load(ffmpeg_command_output, mode: :strict, symbol_keys: true) + + parse_metadata + rescue Terrapin::ExitStatusError, Oj::ParseError + @invalid = true + rescue Terrapin::CommandNotFoundError + raise Paperclip::Errors::CommandNotFoundError, 'Could not run the `ffprobe` command. Please install ffmpeg.' + end + + def valid? + !@invalid + end + + private + + def ffmpeg_command_output + command = Terrapin::CommandLine.new('ffprobe', '-i :path -print_format :format -show_format -show_streams -show_error -loglevel :loglevel') + command.run(path: @path, format: 'json', loglevel: 'fatal') + end + + def parse_metadata + if @metadata.key?(:format) + @duration = @metadata[:format][:duration].to_f + @bitrate = @metadata[:format][:bit_rate].to_i + end + + if @metadata.key?(:streams) + video_streams = @metadata[:streams].select { |stream| stream[:codec_type] == 'video' } + audio_streams = @metadata[:streams].select { |stream| stream[:codec_type] == 'audio' } + + if (video_stream = video_streams.first) + @video_codec = video_stream[:codec_name] + @colorspace = video_stream[:pix_fmt] + @width = video_stream[:width] + @height = video_stream[:height] + @frame_rate = video_stream[:avg_frame_rate] == '0/0' ? nil : Rational(video_stream[:avg_frame_rate]) + end + + if (audio_stream = audio_streams.first) + @audio_codec = audio_stream[:codec_name] + end + end + + @invalid = true if @metadata.key?(:error) + end +end |