about summary refs log tree commit diff
path: root/app/services/process_feed_service.rb
blob: 19ab0c858b82d23b5d5bea15afce0036c0c776a1 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
# frozen_string_literal: true

class ProcessFeedService < BaseService
  def call(body, account)
    xml = Nokogiri::XML(body)
    xml.encoding = 'utf-8'

    update_author(body, account)
    process_entries(xml, account)
  end

  private

  def update_author(body, account)
    RemoteProfileUpdateWorker.perform_async(account.id, body.force_encoding('UTF-8'), true)
  end

  def process_entries(xml, account)
    xml.xpath('//xmlns:entry', xmlns: TagManager::XMLNS).reverse_each.map { |entry| ProcessEntry.new.call(entry, account) }.compact
  end

  class ProcessEntry
    def call(xml, account)
      @account = account
      @fetched = Activity.new(xml)

      return unless [:activity, :note, :comment].include?(@fetched.type)

      klass = case @fetched.verb
              when :post
                PostActivity
              when :share
                ShareActivity
              when :delete
                DeletionActivity
              else
                return
              end

      @fetched = klass.new(xml, account)
      @fetched.perform
    rescue ActiveRecord::RecordInvalid => e
      Rails.logger.debug "Nothing was saved for #{id} because: #{e}"
      nil
    end

    class Activity
      def initialize(xml, account = nil)
        @xml = xml
        @account = account
      end

      def verb
        raw = @xml.at_xpath('./activity:verb', activity: TagManager::AS_XMLNS).content
        TagManager::VERBS.key(raw)
      rescue
        :post
      end

      def type
        raw = @xml.at_xpath('./activity:object-type', activity: TagManager::AS_XMLNS).content
        TagManager::TYPES.key(raw)
      rescue
        :activity
      end

      def id
        @xml.at_xpath('./xmlns:id', xmlns: TagManager::XMLNS).content
      end

      def url
        link = @xml.at_xpath('./xmlns:link[@rel="alternate"]', xmlns: TagManager::XMLNS)
        link.nil? ? nil : link['href']
      end

      private

      def find_status(uri)
        if TagManager.instance.local_id?(uri)
          local_id = TagManager.instance.unique_tag_to_local_id(uri, 'Status')
          return Status.find_by(id: local_id)
        end

        Status.find_by(uri: uri)
      end

      def redis
        Redis.current
      end
    end

    class CreationActivity < Activity
      def perform
        if redis.exists("delete_upon_arrival:#{@account.id}:#{id}")
          Rails.logger.debug "Delete for status #{id} was queued, ignoring"
          return
        end

        Rails.logger.debug "Creating remote status #{id}"
        # Return early if status already exists in db
        status = find_status(id)

        return [status, false] unless status.nil?

        return [nil, false] if @account.suspended?

        status = Status.create!(
          uri: id,
          url: url,
          account: @account,
          reblog: reblog,
          text: content,
          spoiler_text: content_warning,
          created_at: published,
          reply: thread?,
          language: content_language,
          visibility: visibility_scope,
          conversation: converstation_to_persistent&.first,
          thread: thread? ? find_status(thread.first) : nil
        )

        save_mentions(status)
        save_hashtags(status)
        save_media(status)

        if thread? && status.thread.nil?
          Rails.logger.debug "Trying to attach #{status.id} (#{id}) to #{thread.first}"
          ThreadResolveWorker.perform_async(status.id, thread.second)
        end

        Rails.logger.debug "Queuing remote status #{status.id} (#{id}) for distribution"

        LinkCrawlWorker.perform_async(status.id) unless status.spoiler_text?
        DistributionWorker.perform_async(status.id)

        [status, true]
      end

      def content
        @xml.at_xpath('./xmlns:content', xmlns: TagManager::XMLNS).content
      end

      def content_language
        @xml.at_xpath('./xmlns:content', xmlns: TagManager::XMLNS)['xml:lang']&.presence || 'en'
      end

      def content_warning
        @xml.at_xpath('./xmlns:summary', xmlns: TagManager::XMLNS)&.content || ''
      end

      def visibility_scope
        @xml.at_xpath('./mastodon:scope', mastodon: TagManager::MTDN_XMLNS)&.content&.to_sym || :public
      end

      def published
        @xml.at_xpath('./xmlns:published', xmlns: TagManager::XMLNS).content
      end

      def thread?
        !@xml.at_xpath('./thr:in-reply-to', thr: TagManager::THR_XMLNS).nil?
      end

      def thread
        thr = @xml.at_xpath('./thr:in-reply-to', thr: TagManager::THR_XMLNS)
        [thr['ref'], thr['href']]
      end

      private

      def converstation_to_persistent
        uri = @xml.at_xpath('./ostatus:conversation', ostatus: TagManager::OS_XMLNS)&.attribute('ref')&.content
        return if uri.nil?

        if TagManager.instance.local_id?(uri)
          local_id = TagManager.instance.unique_tag_to_local_id(uri, 'Conversation')
          return Conversation.find_by(id: local_id)
        end

        found = Conversation.find_by(uri: uri)
        found ? [found, false] : [Conversation.create!(uri: uri), true]
      end

      def save_mentions(parent)
        processed_account_ids = []

        @xml.xpath('./xmlns:link[@rel="mentioned"]', xmlns: TagManager::XMLNS).each do |link|
          next if [TagManager::TYPES[:group], TagManager::TYPES[:collection]].include? link['ostatus:object-type']

          mentioned_account = account_from_href(link['href'])

          next if mentioned_account.nil? || processed_account_ids.include?(mentioned_account.id)

          mentioned_account.mentions.where(status: parent).first_or_create(status: parent)

          # So we can skip duplicate mentions
          processed_account_ids << mentioned_account.id
        end
      end

      def save_hashtags(parent)
        tags = @xml.xpath('./xmlns:category', xmlns: TagManager::XMLNS).map { |category| category['term'] }.select(&:present?)
        ProcessHashtagsService.new.call(parent, tags)
      end

      def save_media(parent)
        do_not_download = DomainBlock.find_by(domain: parent.account.domain)&.reject_media?

        @xml.xpath('./xmlns:link[@rel="enclosure"]', xmlns: TagManager::XMLNS).each do |link|
          next unless link['href']

          media = MediaAttachment.where(status: parent, remote_url: link['href']).first_or_initialize(account: parent.account, status: parent, remote_url: link['href'])
          parsed_url = Addressable::URI.parse(link['href']).normalize

          next if !%w(http https).include?(parsed_url.scheme) || parsed_url.host.empty?

          media.save

          next if do_not_download

          begin
            media.file_remote_url = link['href']
            media.save!
          rescue ActiveRecord::RecordInvalid
            next
          end
        end
      end

      def account_from_href(href)
        url = Addressable::URI.parse(href).normalize

        if TagManager.instance.web_domain?(url.host)
          Account.find_local(url.path.gsub('/users/', ''))
        else
          Account.where(uri: href).or(Account.where(url: href)).first || FetchRemoteAccountService.new.call(href)
        end
      end
    end

    class ShareActivity < CreationActivity
      def perform
        status, just_created = super
        NotifyService.new.call(reblog.account, status) if reblog&.account&.local? && just_created
        status
      end

      def object
        @xml.at_xpath('.//activity:object', activity: TagManager::AS_XMLNS)
      end

      private

      def status
        reblog && super
      end

      def reblog
        return @reblog if defined? @reblog

        original_status = RemoteActivity.new(object).perform
        @reblog = original_status.reblog? ? original_status.reblog : original_status
      end
    end

    class PostActivity < CreationActivity
      def perform
        status, just_created = super

        if just_created
          status.mentions.includes(:account).each do |mention|
            mentioned_account = mention.account
            next unless mentioned_account.local?
            NotifyService.new.call(mentioned_account, mention)
          end
        end

        status
      end

      private

      def reblog
        nil
      end
    end

    class DeletionActivity < Activity
      def perform
        Rails.logger.debug "Deleting remote status #{id}"
        status = Status.find_by(uri: id, account: @account)

        if status.nil?
          redis.setex("delete_upon_arrival:#{@account.id}:#{id}", 6 * 3_600, id)
        else
          RemoveStatusService.new.call(status)
        end
      end
    end

    class RemoteActivity < Activity
      def perform
        find_status(id) || FetchRemoteStatusService.new.call(url)
      end
    end
  end
end