diff options
| -rw-r--r-- | app/services/process_feed_service.rb | 393 |
1 files changed, 188 insertions, 205 deletions
diff --git a/app/services/process_feed_service.rb b/app/services/process_feed_service.rb index 19ab0c858..c335d2159 100644 --- a/app/services/process_feed_service.rb +++ b/app/services/process_feed_service.rb @@ -22,285 +22,268 @@ class ProcessFeedService < BaseService class ProcessEntry def call(xml, account) @account = account - @fetched = Activity.new(xml) - - return unless [:activity, :note, :comment].include?(@fetched.type) - - klass = case @fetched.verb - when :post - PostActivity - when :share - ShareActivity - when :delete - DeletionActivity - else - return - end - - @fetched = klass.new(xml, account) - @fetched.perform + @xml = xml + + return if skip_unsupported_type? + + case verb + when :post, :share + return create_status + when :delete + return delete_status + end rescue ActiveRecord::RecordInvalid => e Rails.logger.debug "Nothing was saved for #{id} because: #{e}" nil end - class Activity - def initialize(xml, account = nil) - @xml = xml - @account = account - end + private - def verb - raw = @xml.at_xpath('./activity:verb', activity: TagManager::AS_XMLNS).content - TagManager::VERBS.key(raw) - rescue - :post + def create_status + if redis.exists("delete_upon_arrival:#{@account.id}:#{id}") + Rails.logger.debug "Delete for status #{id} was queued, ignoring" + return end - def type - raw = @xml.at_xpath('./activity:object-type', activity: TagManager::AS_XMLNS).content - TagManager::TYPES.key(raw) - rescue - :activity - end + status, just_created = nil - def id - @xml.at_xpath('./xmlns:id', xmlns: TagManager::XMLNS).content - end + Rails.logger.debug "Creating remote status #{id}" - def url - link = @xml.at_xpath('./xmlns:link[@rel="alternate"]', xmlns: TagManager::XMLNS) - link.nil? ? nil : link['href'] + if verb == :share + original_status = shared_status_from_xml(@xml.at_xpath('.//activity:object', activity: TagManager::AS_XMLNS)) + return nil if original_status.nil? end - private + ApplicationRecord.transaction do + status, just_created = status_from_xml(@xml) - def find_status(uri) - if TagManager.instance.local_id?(uri) - local_id = TagManager.instance.unique_tag_to_local_id(uri, 'Status') - return Status.find_by(id: local_id) + return if status.nil? + return status unless just_created + + if verb == :share + status.reblog = original_status.reblog? ? original_status.reblog : original_status end - Status.find_by(uri: uri) + status.save! end - def redis - Redis.current + if thread?(@xml) && status.thread.nil? + Rails.logger.debug "Trying to attach #{status.id} (#{id(@xml)}) to #{thread(@xml).first}" + ThreadResolveWorker.perform_async(status.id, thread(@xml).second) end - end - class CreationActivity < Activity - def perform - if redis.exists("delete_upon_arrival:#{@account.id}:#{id}") - Rails.logger.debug "Delete for status #{id} was queued, ignoring" - return - end + notify_about_mentions!(status) unless status.reblog? + notify_about_reblog!(status) if status.reblog? && status.reblog.account.local? - Rails.logger.debug "Creating remote status #{id}" - # Return early if status already exists in db - status = find_status(id) - - return [status, false] unless status.nil? - - return [nil, false] if @account.suspended? - - status = Status.create!( - uri: id, - url: url, - account: @account, - reblog: reblog, - text: content, - spoiler_text: content_warning, - created_at: published, - reply: thread?, - language: content_language, - visibility: visibility_scope, - conversation: converstation_to_persistent&.first, - thread: thread? ? find_status(thread.first) : nil - ) - - save_mentions(status) - save_hashtags(status) - save_media(status) - - if thread? && status.thread.nil? - Rails.logger.debug "Trying to attach #{status.id} (#{id}) to #{thread.first}" - ThreadResolveWorker.perform_async(status.id, thread.second) - end + Rails.logger.debug "Queuing remote status #{status.id} (#{id}) for distribution" - Rails.logger.debug "Queuing remote status #{status.id} (#{id}) for distribution" + LinkCrawlWorker.perform_async(status.id) unless status.spoiler_text? + DistributionWorker.perform_async(status.id) - LinkCrawlWorker.perform_async(status.id) unless status.spoiler_text? - DistributionWorker.perform_async(status.id) + status + end - [status, true] + def notify_about_mentions!(status) + status.mentions.includes(:account).each do |mention| + mentioned_account = mention.account + next unless mentioned_account.local? + NotifyService.new.call(mentioned_account, mention) end + end - def content - @xml.at_xpath('./xmlns:content', xmlns: TagManager::XMLNS).content - end + def notify_about_reblog!(status) + NotifyService.new.call(status.reblog.account, status) + end - def content_language - @xml.at_xpath('./xmlns:content', xmlns: TagManager::XMLNS)['xml:lang']&.presence || 'en' - end + def delete_status + Rails.logger.debug "Deleting remote status #{id}" + status = Status.find_by(uri: id, account: @account) - def content_warning - @xml.at_xpath('./xmlns:summary', xmlns: TagManager::XMLNS)&.content || '' - end - - def visibility_scope - @xml.at_xpath('./mastodon:scope', mastodon: TagManager::MTDN_XMLNS)&.content&.to_sym || :public + if status.nil? + redis.setex("delete_upon_arrival:#{@account.id}:#{id}", 6 * 3_600, id) + else + RemoveStatusService.new.call(status) end + end - def published - @xml.at_xpath('./xmlns:published', xmlns: TagManager::XMLNS).content - end + def skip_unsupported_type? + !([:post, :share, :delete].include?(verb) && [:activity, :note, :comment].include?(type)) + end - def thread? - !@xml.at_xpath('./thr:in-reply-to', thr: TagManager::THR_XMLNS).nil? - end + def shared_status_from_xml(entry) + status = find_status(id(entry)) - def thread - thr = @xml.at_xpath('./thr:in-reply-to', thr: TagManager::THR_XMLNS) - [thr['ref'], thr['href']] - end + return status unless status.nil? - private + FetchRemoteStatusService.new.call(url(entry)) + end - def converstation_to_persistent - uri = @xml.at_xpath('./ostatus:conversation', ostatus: TagManager::OS_XMLNS)&.attribute('ref')&.content - return if uri.nil? + def status_from_xml(entry) + # Return early if status already exists in db + status = find_status(id(entry)) - if TagManager.instance.local_id?(uri) - local_id = TagManager.instance.unique_tag_to_local_id(uri, 'Conversation') - return Conversation.find_by(id: local_id) - end + return [status, false] unless status.nil? - found = Conversation.find_by(uri: uri) - found ? [found, false] : [Conversation.create!(uri: uri), true] - end + account = @account - def save_mentions(parent) - processed_account_ids = [] + return [nil, false] if account.suspended? - @xml.xpath('./xmlns:link[@rel="mentioned"]', xmlns: TagManager::XMLNS).each do |link| - next if [TagManager::TYPES[:group], TagManager::TYPES[:collection]].include? link['ostatus:object-type'] + status = Status.create!( + uri: id(entry), + url: url(entry), + account: account, + text: content(entry), + spoiler_text: content_warning(entry), + created_at: published(entry), + reply: thread?(entry), + language: content_language(entry), + visibility: visibility_scope(entry), + conversation: find_or_create_conversation(entry), + thread: thread?(entry) ? find_status(thread(entry).first) : nil + ) - mentioned_account = account_from_href(link['href']) + mentions_from_xml(status, entry) + hashtags_from_xml(status, entry) + media_from_xml(status, entry) - next if mentioned_account.nil? || processed_account_ids.include?(mentioned_account.id) + [status, true] + end - mentioned_account.mentions.where(status: parent).first_or_create(status: parent) + def find_or_create_conversation(xml) + uri = xml.at_xpath('./ostatus:conversation', ostatus: TagManager::OS_XMLNS)&.attribute('ref')&.content + return if uri.nil? - # So we can skip duplicate mentions - processed_account_ids << mentioned_account.id - end + if TagManager.instance.local_id?(uri) + local_id = TagManager.instance.unique_tag_to_local_id(uri, 'Conversation') + return Conversation.find_by(id: local_id) end - def save_hashtags(parent) - tags = @xml.xpath('./xmlns:category', xmlns: TagManager::XMLNS).map { |category| category['term'] }.select(&:present?) - ProcessHashtagsService.new.call(parent, tags) + Conversation.find_by(uri: uri) || Conversation.create!(uri: uri) + end + + def find_status(uri) + if TagManager.instance.local_id?(uri) + local_id = TagManager.instance.unique_tag_to_local_id(uri, 'Status') + return Status.find_by(id: local_id) end - def save_media(parent) - do_not_download = DomainBlock.find_by(domain: parent.account.domain)&.reject_media? + Status.find_by(uri: uri) + end - @xml.xpath('./xmlns:link[@rel="enclosure"]', xmlns: TagManager::XMLNS).each do |link| - next unless link['href'] + def mentions_from_xml(parent, xml) + processed_account_ids = [] - media = MediaAttachment.where(status: parent, remote_url: link['href']).first_or_initialize(account: parent.account, status: parent, remote_url: link['href']) - parsed_url = Addressable::URI.parse(link['href']).normalize + xml.xpath('./xmlns:link[@rel="mentioned"]', xmlns: TagManager::XMLNS).each do |link| + next if [TagManager::TYPES[:group], TagManager::TYPES[:collection]].include? link['ostatus:object-type'] - next if !%w(http https).include?(parsed_url.scheme) || parsed_url.host.empty? + mentioned_account = account_from_href(link['href']) - media.save + next if mentioned_account.nil? || processed_account_ids.include?(mentioned_account.id) - next if do_not_download + mentioned_account.mentions.where(status: parent).first_or_create(status: parent) - begin - media.file_remote_url = link['href'] - media.save! - rescue ActiveRecord::RecordInvalid - next - end - end + # So we can skip duplicate mentions + processed_account_ids << mentioned_account.id end + end - def account_from_href(href) - url = Addressable::URI.parse(href).normalize + def account_from_href(href) + url = Addressable::URI.parse(href).normalize - if TagManager.instance.web_domain?(url.host) - Account.find_local(url.path.gsub('/users/', '')) - else - Account.where(uri: href).or(Account.where(url: href)).first || FetchRemoteAccountService.new.call(href) - end + if TagManager.instance.web_domain?(url.host) + Account.find_local(url.path.gsub('/users/', '')) + else + Account.where(uri: href).or(Account.where(url: href)).first || FetchRemoteAccountService.new.call(href) end end - class ShareActivity < CreationActivity - def perform - status, just_created = super - NotifyService.new.call(reblog.account, status) if reblog&.account&.local? && just_created - status - end + def hashtags_from_xml(parent, xml) + tags = xml.xpath('./xmlns:category', xmlns: TagManager::XMLNS).map { |category| category['term'] }.select(&:present?) + ProcessHashtagsService.new.call(parent, tags) + end - def object - @xml.at_xpath('.//activity:object', activity: TagManager::AS_XMLNS) - end + def media_from_xml(parent, xml) + do_not_download = DomainBlock.find_by(domain: parent.account.domain)&.reject_media? - private + xml.xpath('./xmlns:link[@rel="enclosure"]', xmlns: TagManager::XMLNS).each do |link| + next unless link['href'] - def status - reblog && super - end + media = MediaAttachment.where(status: parent, remote_url: link['href']).first_or_initialize(account: parent.account, status: parent, remote_url: link['href']) + parsed_url = Addressable::URI.parse(link['href']).normalize + + next if !%w(http https).include?(parsed_url.scheme) || parsed_url.host.empty? + + media.save - def reblog - return @reblog if defined? @reblog + next if do_not_download - original_status = RemoteActivity.new(object).perform - @reblog = original_status.reblog? ? original_status.reblog : original_status + begin + media.file_remote_url = link['href'] + media.save! + rescue ActiveRecord::RecordInvalid + next + end end end - class PostActivity < CreationActivity - def perform - status, just_created = super + def id(xml = @xml) + xml.at_xpath('./xmlns:id', xmlns: TagManager::XMLNS).content + end - if just_created - status.mentions.includes(:account).each do |mention| - mentioned_account = mention.account - next unless mentioned_account.local? - NotifyService.new.call(mentioned_account, mention) - end - end + def verb(xml = @xml) + raw = xml.at_xpath('./activity:verb', activity: TagManager::AS_XMLNS).content + TagManager::VERBS.key(raw) + rescue + :post + end - status - end + def type(xml = @xml) + raw = xml.at_xpath('./activity:object-type', activity: TagManager::AS_XMLNS).content + TagManager::TYPES.key(raw) + rescue + :activity + end + + def url(xml = @xml) + link = xml.at_xpath('./xmlns:link[@rel="alternate"]', xmlns: TagManager::XMLNS) + link.nil? ? nil : link['href'] + end - private + def content(xml = @xml) + xml.at_xpath('./xmlns:content', xmlns: TagManager::XMLNS).content + end - def reblog - nil - end + def content_language(xml = @xml) + xml.at_xpath('./xmlns:content', xmlns: TagManager::XMLNS)['xml:lang']&.presence || 'en' end - class DeletionActivity < Activity - def perform - Rails.logger.debug "Deleting remote status #{id}" - status = Status.find_by(uri: id, account: @account) + def content_warning(xml = @xml) + xml.at_xpath('./xmlns:summary', xmlns: TagManager::XMLNS)&.content || '' + end - if status.nil? - redis.setex("delete_upon_arrival:#{@account.id}:#{id}", 6 * 3_600, id) - else - RemoveStatusService.new.call(status) - end - end + def visibility_scope(xml = @xml) + xml.at_xpath('./mastodon:scope', mastodon: TagManager::MTDN_XMLNS)&.content&.to_sym || :public end - class RemoteActivity < Activity - def perform - find_status(id) || FetchRemoteStatusService.new.call(url) - end + def published(xml = @xml) + xml.at_xpath('./xmlns:published', xmlns: TagManager::XMLNS).content + end + + def thread?(xml = @xml) + !xml.at_xpath('./thr:in-reply-to', thr: TagManager::THR_XMLNS).nil? + end + + def thread(xml = @xml) + thr = xml.at_xpath('./thr:in-reply-to', thr: TagManager::THR_XMLNS) + [thr['ref'], thr['href']] + end + + def account?(xml = @xml) + !xml.at_xpath('./xmlns:author', xmlns: TagManager::XMLNS).nil? + end + + def redis + Redis.current end end end |