Merge upstream 2.0ish #165

13 files changed, 204 insertions, 68 deletions

diff --git a/app/lib/activitypub/activity.rb b/app/lib/activitypub/activity.rb
index b06dd6194..9688f57a6 100644
--- a/app/lib/activitypub/activity.rb
+++ b/app/lib/activitypub/activity.rb
@@ -3,10 +3,11 @@
 class ActivityPub::Activity
   include JsonLdHelper
 
-  def initialize(json, account)
+  def initialize(json, account, options = {})
     @json    = json
     @account = account
     @object  = @json['object']
+    @options = options
   end
 
   def perform
@@ -14,9 +15,9 @@ class ActivityPub::Activity
   end
 
   class << self
-    def factory(json, account)
+    def factory(json, account, options = {})
       @json = json
-      klass&.new(json, account)
+      klass&.new(json, account, options)
     end
 
     private
diff --git a/app/lib/activitypub/activity/announce.rb b/app/lib/activitypub/activity/announce.rb
index 4516454e1..b84098933 100644
--- a/app/lib/activitypub/activity/announce.rb
+++ b/app/lib/activitypub/activity/announce.rb
@@ -15,8 +15,9 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity
       account: @account,
       reblog: original_status,
       uri: @json['id'],
-      created_at: @json['published'] || Time.now.utc
+      created_at: @options[:override_timestamps] ? nil : @json['published']
     )
+
     distribute(status)
     status
   end
@@ -27,7 +28,7 @@ class ActivityPub::Activity::Announce < ActivityPub::Activity
     if object_uri.start_with?('http')
       return if ActivityPub::TagManager.instance.local_uri?(object_uri)
 
-      ActivityPub::FetchRemoteStatusService.new.call(object_uri)
+      ActivityPub::FetchRemoteStatusService.new.call(object_uri, id: true)
     elsif @object['url'].present?
       ::FetchRemoteStatusService.new.call(@object['url'])
     end
diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb
index 4e19b3096..d6e9bc1de 100644
--- a/app/lib/activitypub/activity/create.rb
+++ b/app/lib/activitypub/activity/create.rb
@@ -43,7 +43,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
       text: text_from_content || '',
       language: language_from_content,
       spoiler_text: @object['summary'] || '',
-      created_at: @object['published'] || Time.now.utc,
+      created_at: @options[:override_timestamps] ? nil : @object['published'],
       reply: @object['inReplyTo'].present?,
       sensitive: @object['sensitive'] || false,
       visibility: visibility_from_audience,
@@ -80,21 +80,25 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
     return if tag['href'].blank?
 
     account = account_from_uri(tag['href'])
-    account = FetchRemoteAccountService.new.call(tag['href']) if account.nil?
+    account = FetchRemoteAccountService.new.call(tag['href'], id: false) if account.nil?
     return if account.nil?
     account.mentions.create(status: status)
   end
 
   def process_emoji(tag, _status)
-    return if tag['name'].blank? || tag['href'].blank?
+    return if skip_download?
+    return if tag['name'].blank? || tag['icon'].blank? || tag['icon']['url'].blank?
 
     shortcode = tag['name'].delete(':')
+    image_url = tag['icon']['url']
+    uri       = tag['id']
+    updated   = tag['updated']
     emoji     = CustomEmoji.find_by(shortcode: shortcode, domain: @account.domain)
 
-    return if !emoji.nil? || skip_download?
+    return unless emoji.nil? || emoji.updated_at >= updated
 
-    emoji = CustomEmoji.new(domain: @account.domain, shortcode: shortcode)
-    emoji.image_remote_url = tag['href']
+    emoji ||= CustomEmoji.new(domain: @account.domain, shortcode: shortcode, uri: uri)
+    emoji.image_remote_url = image_url
     emoji.save
   end
 
@@ -105,7 +109,7 @@ class ActivityPub::Activity::Create < ActivityPub::Activity
       next if unsupported_media_type?(attachment['mediaType']) || attachment['url'].blank?
 
       href             = Addressable::URI.parse(attachment['url']).normalize.to_s
-      media_attachment = MediaAttachment.create(status: status, account: status.account, remote_url: href)
+      media_attachment = MediaAttachment.create(status: status, account: status.account, remote_url: href, description: attachment['name'].presence)
 
       next if skip_download?
 
diff --git a/app/lib/activitypub/linked_data_signature.rb b/app/lib/activitypub/linked_data_signature.rb
index adb8b6cdf..16142a6ff 100644
--- a/app/lib/activitypub/linked_data_signature.rb
+++ b/app/lib/activitypub/linked_data_signature.rb
@@ -19,7 +19,7 @@ class ActivityPub::LinkedDataSignature
     return unless type == 'RsaSignature2017'
 
     creator   = ActivityPub::TagManager.instance.uri_to_resource(creator_uri, Account)
-    creator ||= ActivityPub::FetchRemoteKeyService.new.call(creator_uri)
+    creator ||= ActivityPub::FetchRemoteKeyService.new.call(creator_uri, id: false)
 
     return if creator.nil?
 
diff --git a/app/lib/activitypub/tag_manager.rb b/app/lib/activitypub/tag_manager.rb
index 4ec3b8c56..0708713e6 100644
--- a/app/lib/activitypub/tag_manager.rb
+++ b/app/lib/activitypub/tag_manager.rb
@@ -33,6 +33,8 @@ class ActivityPub::TagManager
     when :note, :comment, :activity
       return activity_account_status_url(target.account, target) if target.reblog?
       account_status_url(target.account, target)
+    when :emoji
+      emoji_url(target)
     end
   end
 
diff --git a/app/lib/delivery_failure_tracker.rb b/app/lib/delivery_failure_tracker.rb
new file mode 100644
index 000000000..8d3be35de
--- /dev/null
+++ b/app/lib/delivery_failure_tracker.rb
@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+class DeliveryFailureTracker
+  FAILURE_DAYS_THRESHOLD = 7
+
+  def initialize(inbox_url)
+    @inbox_url = inbox_url
+  end
+
+  def track_failure!
+    Redis.current.sadd(exhausted_deliveries_key, today)
+    Redis.current.sadd('unavailable_inboxes', @inbox_url) if reached_failure_threshold?
+  end
+
+  def track_success!
+    Redis.current.del(exhausted_deliveries_key)
+    Redis.current.srem('unavailable_inboxes', @inbox_url)
+  end
+
+  def days
+    Redis.current.scard(exhausted_deliveries_key) || 0
+  end
+
+  class << self
+    def filter(arr)
+      arr.reject(&method(:unavailable?))
+    end
+
+    def unavailable?(url)
+      Redis.current.sismember('unavailable_inboxes', url)
+    end
+
+    def available?(url)
+      !unavailable?(url)
+    end
+
+    def track_inverse_success!(from_account)
+      new(from_account.inbox_url).track_success! if from_account.inbox_url.present?
+      new(from_account.shared_inbox_url).track_success! if from_account.shared_inbox_url.present?
+    end
+  end
+
+  private
+
+  def exhausted_deliveries_key
+    "exhausted_deliveries:#{@inbox_url}"
+  end
+
+  def today
+    Time.now.utc.strftime('%Y%m%d')
+  end
+
+  def reached_failure_threshold?
+    days >= FAILURE_DAYS_THRESHOLD
+  end
+end
diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb
index 3b6796142..f6a694135 100644
--- a/app/lib/feed_manager.rb
+++ b/app/lib/feed_manager.rb
@@ -7,8 +7,13 @@ class FeedManager
 
   MAX_ITEMS = 400
 
-  def key(type, id)
-    "feed:#{type}:#{id}"
+  # Must be <= MAX_ITEMS or the tracking sets will grow forever
+  REBLOG_FALLOFF = 40
+
+  def key(type, id, subtype = nil)
+    return "feed:#{type}:#{id}" unless subtype
+
+    "feed:#{type}:#{id}:#{subtype}"
   end
 
   def filter?(timeline_type, status, receiver_id)
@@ -22,23 +27,36 @@ class FeedManager
   end
 
   def push(timeline_type, account, status)
-    timeline_key = key(timeline_type, account.id)
+    return false unless add_to_feed(timeline_type, account, status)
 
-    if status.reblog?
-      # If the original status is within 40 statuses from top, do not re-insert it into the feed
-      rank = redis.zrevrank(timeline_key, status.reblog_of_id)
-      return if !rank.nil? && rank < 40
-      redis.zadd(timeline_key, status.id, status.reblog_of_id)
-    else
-      redis.zadd(timeline_key, status.id, status.id)
-      trim(timeline_type, account.id)
-    end
+    trim(timeline_type, account.id)
 
     PushUpdateWorker.perform_async(account.id, status.id) if push_update_required?(timeline_type, account.id)
+
+    true
+  end
+
+  def unpush(timeline_type, account, status)
+    return false unless remove_from_feed(timeline_type, account, status)
+
+    payload = Oj.dump(event: :delete, payload: status.id.to_s)
+    Redis.current.publish("timeline:#{account.id}", payload)
+
+    true
   end
 
   def trim(type, account_id)
-    redis.zremrangebyrank(key(type, account_id), '0', (-(FeedManager::MAX_ITEMS + 1)).to_s)
+    timeline_key = key(type, account_id)
+    reblog_key = key(type, account_id, 'reblogs')
+    # Remove any items past the MAX_ITEMS'th entry in our feed
+    redis.zremrangebyrank(timeline_key, '0', (-(FeedManager::MAX_ITEMS + 1)).to_s)
+
+    # Get the score of the REBLOG_FALLOFF'th item in our feed, and stop
+    # tracking anything after it for deduplication purposes.
+    falloff_rank = FeedManager::REBLOG_FALLOFF - 1
+    falloff_range = redis.zrevrange(timeline_key, falloff_rank, falloff_rank, with_scores: true)
+    falloff_score = falloff_range&.first&.last&.to_i || 0
+    redis.zremrangebyscore(reblog_key, 0, falloff_score)
   end
 
   def push_update_required?(timeline_type, account_id)
@@ -54,11 +72,9 @@ class FeedManager
       query = query.where('id > ?', oldest_home_score)
     end
 
-    redis.pipelined do
-      query.each do |status|
-        next if status.direct_visibility? || filter?(:home, status, into_account)
-        redis.zadd(timeline_key, status.id, status.id)
-      end
+    query.each do |status|
+      next if status.direct_visibility? || filter?(:home, status, into_account)
+      add_to_feed(:home, into_account, status)
     end
 
     trim(:home, into_account.id)
@@ -68,22 +84,28 @@ class FeedManager
     timeline_key = key(:home, into_account.id)
     oldest_home_score = redis.zrange(timeline_key, 0, 0, with_scores: true)&.first&.last&.to_i || 0
 
-    from_account.statuses.select('id').where('id > ?', oldest_home_score).reorder(nil).find_in_batches do |statuses|
-      redis.pipelined do
-        statuses.each do |status|
-          redis.zrem(timeline_key, status.id)
-          redis.zremrangebyscore(timeline_key, status.id, status.id)
-        end
-      end
+    from_account.statuses.select('id, reblog_of_id').where('id > ?', oldest_home_score).reorder(nil).find_each do |status|
+      unpush(:home, into_account, status)
     end
   end
 
   def clear_from_timeline(account, target_account)
     timeline_key = key(:home, account.id)
     timeline_status_ids = redis.zrange(timeline_key, 0, -1)
-    target_status_ids = Status.where(id: timeline_status_ids, account: target_account).ids
+    target_statuses = Status.where(id: timeline_status_ids, account: target_account)
+
+    target_statuses.each do |status|
+      unpush(:home, account, status)
+    end
+  end
 
-    redis.zrem(timeline_key, target_status_ids) if target_status_ids.present?
+  def populate_feed(account)
+    prepopulate_limit = FeedManager::MAX_ITEMS / 4
+    statuses = Status.as_home_timeline(account).order(account_id: :desc).limit(prepopulate_limit)
+    statuses.reverse_each do |status|
+      next if filter_from_home?(status, account)
+      add_to_feed(:home, account, status)
+    end
   end
 
   private
@@ -137,4 +159,58 @@ class FeedManager
 
     should_filter
   end
+
+  # Adds a status to an account's feed, returning true if a status was
+  # added, and false if it was not added to the feed. Note that this is
+  # an internal helper: callers must call trim or push updates if
+  # either action is appropriate.
+  def add_to_feed(timeline_type, account, status)
+    timeline_key = key(timeline_type, account.id)
+    reblog_key = key(timeline_type, account.id, 'reblogs')
+
+    if status.reblog?
+      # If the original status or a reblog of it is within
+      # REBLOG_FALLOFF statuses from the top, do not re-insert it into
+      # the feed
+      rank = redis.zrevrank(timeline_key, status.reblog_of_id)
+      return false if !rank.nil? && rank < FeedManager::REBLOG_FALLOFF
+
+      reblog_rank = redis.zrevrank(reblog_key, status.reblog_of_id)
+      return false unless reblog_rank.nil?
+
+      redis.zadd(timeline_key, status.id, status.id)
+      redis.zadd(reblog_key, status.id, status.reblog_of_id)
+    else
+      redis.zadd(timeline_key, status.id, status.id)
+    end
+
+    true
+  end
+
+  # Removes an individual status from a feed, correctly handling cases
+  # with reblogs, and returning true if a status was removed. As with
+  # `add_to_feed`, this does not trigger push updates, so callers must
+  # do so if appropriate.
+  def remove_from_feed(timeline_type, account, status)
+    timeline_key = key(timeline_type, account.id)
+    reblog_key = key(timeline_type, account.id, 'reblogs')
+
+    if status.reblog?
+      # 1. If the reblogging status is not in the feed, stop.
+      status_rank = redis.zrevrank(timeline_key, status.id)
+      return false if status_rank.nil?
+
+      # 2. Remove the reblogged status from the `:reblogs` zset.
+      redis.zrem(reblog_key, status.reblog_of_id)
+
+      # 3. Add the reblogged status to the feed using the reblogging
+      # status' ID as its score, and the reblogged status' ID as its
+      # value.
+      redis.zadd(timeline_key, status.id, status.reblog_of_id)
+
+      # 4. Remove the reblogging status from the feed (as normal)
+    end
+
+    redis.zrem(timeline_key, status.id)
+  end
 end
diff --git a/app/lib/formatter.rb b/app/lib/formatter.rb
index 42cd72990..57f105da7 100644
--- a/app/lib/formatter.rb
+++ b/app/lib/formatter.rb
@@ -50,7 +50,7 @@ class Formatter
   end
 
   def simplified_format(account)
-    return reformat(account.note) unless account.local?
+    return reformat(account.note).html_safe unless account.local? # rubocop:disable Rails/OutputSafety
 
     html = encode_and_link_urls(account.note)
     html = simple_format(html, {}, sanitize: false)
@@ -92,7 +92,7 @@ class Formatter
   def encode_custom_emojis(html, emojis)
     return html if emojis.empty?
 
-    emoji_map = emojis.map { |e| [e.shortcode, full_asset_url(e.image.url)] }.to_h
+    emoji_map = emojis.map { |e| [e.shortcode, full_asset_url(e.image.url(:static))] }.to_h
 
     i                     = -1
     inside_tag            = false
diff --git a/app/lib/ostatus/activity/base.rb b/app/lib/ostatus/activity/base.rb
index 039381397..8b27b124f 100644
--- a/app/lib/ostatus/activity/base.rb
+++ b/app/lib/ostatus/activity/base.rb
@@ -1,9 +1,10 @@
 # frozen_string_literal: true
 
 class OStatus::Activity::Base
-  def initialize(xml, account = nil)
-    @xml = xml
+  def initialize(xml, account = nil, options = {})
+    @xml     = xml
     @account = account
+    @options = options
   end
 
   def status?
diff --git a/app/lib/ostatus/activity/creation.rb b/app/lib/ostatus/activity/creation.rb
index 2687776f9..a1ab522e2 100644
--- a/app/lib/ostatus/activity/creation.rb
+++ b/app/lib/ostatus/activity/creation.rb
@@ -9,11 +9,6 @@ class OStatus::Activity::Creation < OStatus::Activity::Base
 
     return [nil, false] if @account.suspended?
 
-    if activitypub_uri? && [:public, :unlisted].include?(visibility_scope)
-      result = perform_via_activitypub
-      return result if result.first.present?
-    end
-
     RedisLock.acquire(lock_options) do |lock|
       if lock.acquired?
         # Return early if status already exists in db
@@ -39,7 +34,7 @@ class OStatus::Activity::Creation < OStatus::Activity::Base
         reblog: cached_reblog,
         text: content,
         spoiler_text: content_warning,
-        created_at: published,
+        created_at: @options[:override_timestamps] ? nil : published,
         reply: thread?,
         language: content_language,
         visibility: visibility_scope,
@@ -66,10 +61,6 @@ class OStatus::Activity::Creation < OStatus::Activity::Base
     status
   end
 
-  def perform_via_activitypub
-    [find_status(activitypub_uri) || ActivityPub::FetchRemoteStatusService.new.call(activitypub_uri), false]
-  end
-
   def content
     @xml.at_xpath('./xmlns:content', xmlns: OStatus::TagManager::XMLNS).content
   end
diff --git a/app/lib/ostatus/activity/general.rb b/app/lib/ostatus/activity/general.rb
index b3bef9861..8a6aabc33 100644
--- a/app/lib/ostatus/activity/general.rb
+++ b/app/lib/ostatus/activity/general.rb
@@ -2,7 +2,7 @@
 
 class OStatus::Activity::General < OStatus::Activity::Base
   def specialize
-    special_class&.new(@xml, @account)
+    special_class&.new(@xml, @account, @options)
   end
 
   private
diff --git a/app/lib/request.rb b/app/lib/request.rb
index b083edaf7..30ea0e7ee 100644
--- a/app/lib/request.rb
+++ b/app/lib/request.rb
@@ -32,7 +32,7 @@ class Request
   def perform
     http_client.headers(headers).public_send(@verb, @url.to_s, @options)
   rescue => e
-    raise e.class, "#{e.message} on #{@url}"
+    raise e.class, "#{e.message} on #{@url}", e.backtrace[0]
   end
 
   def headers
@@ -85,6 +85,6 @@ class Request
   end
 
   def http_client
-    HTTP.timeout(:per_operation, timeout).follow
+    HTTP.timeout(:per_operation, timeout).follow(max_hops: 2)
   end
 end
diff --git a/app/lib/user_settings_decorator.rb b/app/lib/user_settings_decorator.rb
index 3b156b98c..3b7a856ee 100644
--- a/app/lib/user_settings_decorator.rb
+++ b/app/lib/user_settings_decorator.rb
@@ -15,17 +15,17 @@ class UserSettingsDecorator
   private
 
   def process_update
-    user.settings['notification_emails'] = merged_notification_emails
-    user.settings['interactions'] = merged_interactions
-    user.settings['default_privacy'] = default_privacy_preference
-    user.settings['default_sensitive'] = default_sensitive_preference
-    user.settings['unfollow_modal'] = unfollow_modal_preference
-    user.settings['boost_modal'] = boost_modal_preference
-    user.settings['delete_modal'] = delete_modal_preference
-    user.settings['auto_play_gif'] = auto_play_gif_preference
-    user.settings['system_font_ui'] = system_font_ui_preference
-    user.settings['noindex'] = noindex_preference
-    user.settings['theme'] = theme_preference
+    user.settings['notification_emails'] = merged_notification_emails if change?('notification_emails')
+    user.settings['interactions']        = merged_interactions if change?('interactions')
+    user.settings['default_privacy']     = default_privacy_preference if change?('setting_default_privacy')
+    user.settings['default_sensitive']   = default_sensitive_preference if change?('setting_default_sensitive')
+    user.settings['unfollow_modal']      = unfollow_modal_preference if change?('setting_unfollow_modal')
+    user.settings['boost_modal']         = boost_modal_preference if change?('setting_boost_modal')
+    user.settings['delete_modal']        = delete_modal_preference if change?('setting_delete_modal')
+    user.settings['auto_play_gif']       = auto_play_gif_preference if change?('setting_auto_play_gif')
+    user.settings['system_font_ui']      = system_font_ui_preference if change?('setting_system_font_ui')
+    user.settings['noindex']             = noindex_preference if change?('setting_noindex')
+    user.settings['theme']               = theme_preference if change?('setting_theme')
   end
 
   def merged_notification_emails
@@ -83,4 +83,8 @@ class UserSettingsDecorator
   def coerce_values(params_hash)
     params_hash.transform_values { |x| x == '1' }
   end
+
+  def change?(key)
+    !settings[key].nil?
+  end
 end