From f722bd2387df9163760014e9555928ec487ae95f Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Tue, 4 Apr 2017 00:53:20 +0200
Subject: Separate background jobs into different queues. ATTENTION: new queue
 "pull" must be added to the Sidekiq invokation in your systemd file

The pull queue will handle link crawling, thread resolving, and OStatus
processing. Such tasks are more likely to hang for a longer time (due to
network requests) so it is more sensible to not make the "in-house" tasks
wait for them.
---
 app/workers/after_remote_follow_request_worker.rb | 2 +-
 app/workers/after_remote_follow_worker.rb         | 2 +-
 app/workers/import_worker.rb                      | 2 +-
 app/workers/link_crawl_worker.rb                  | 2 +-
 app/workers/merge_worker.rb                       | 2 ++
 app/workers/notification_worker.rb                | 2 +-
 app/workers/processing_worker.rb                  | 2 +-
 app/workers/regeneration_worker.rb                | 2 ++
 app/workers/salmon_worker.rb                      | 2 +-
 app/workers/thread_resolve_worker.rb              | 2 +-
 app/workers/unmerge_worker.rb                     | 2 ++
 11 files changed, 14 insertions(+), 8 deletions(-)

(limited to 'app')

diff --git a/app/workers/after_remote_follow_request_worker.rb b/app/workers/after_remote_follow_request_worker.rb
index f1d6869cc..1f2db3061 100644
--- a/app/workers/after_remote_follow_request_worker.rb
+++ b/app/workers/after_remote_follow_request_worker.rb
@@ -3,7 +3,7 @@
 class AfterRemoteFollowRequestWorker
   include Sidekiq::Worker
 
-  sidekiq_options retry: 5
+  sidekiq_options queue: 'pull', retry: 5
 
   def perform(follow_request_id)
     follow_request  = FollowRequest.find(follow_request_id)
diff --git a/app/workers/after_remote_follow_worker.rb b/app/workers/after_remote_follow_worker.rb
index 0d04456a9..bdd2c2a91 100644
--- a/app/workers/after_remote_follow_worker.rb
+++ b/app/workers/after_remote_follow_worker.rb
@@ -3,7 +3,7 @@
 class AfterRemoteFollowWorker
   include Sidekiq::Worker
 
-  sidekiq_options retry: 5
+  sidekiq_options queue: 'pull', retry: 5
 
   def perform(follow_id)
     follow          = Follow.find(follow_id)
diff --git a/app/workers/import_worker.rb b/app/workers/import_worker.rb
index a3ae2a85a..7cf29fb53 100644
--- a/app/workers/import_worker.rb
+++ b/app/workers/import_worker.rb
@@ -5,7 +5,7 @@ require 'csv'
 class ImportWorker
   include Sidekiq::Worker
 
-  sidekiq_options retry: false
+  sidekiq_options queue: 'pull', retry: false
 
   def perform(import_id)
     import = Import.find(import_id)
diff --git a/app/workers/link_crawl_worker.rb b/app/workers/link_crawl_worker.rb
index af3394b8b..834b0088b 100644
--- a/app/workers/link_crawl_worker.rb
+++ b/app/workers/link_crawl_worker.rb
@@ -3,7 +3,7 @@
 class LinkCrawlWorker
   include Sidekiq::Worker
 
-  sidekiq_options retry: false
+  sidekiq_options queue: 'pull', retry: false
 
   def perform(status_id)
     FetchLinkCardService.new.call(Status.find(status_id))
diff --git a/app/workers/merge_worker.rb b/app/workers/merge_worker.rb
index 0f288f43f..d745cb99c 100644
--- a/app/workers/merge_worker.rb
+++ b/app/workers/merge_worker.rb
@@ -3,6 +3,8 @@
 class MergeWorker
   include Sidekiq::Worker
 
+  sidekiq_options queue: 'pull'
+
   def perform(from_account_id, into_account_id)
     FeedManager.instance.merge_into_timeline(Account.find(from_account_id), Account.find(into_account_id))
   end
diff --git a/app/workers/notification_worker.rb b/app/workers/notification_worker.rb
index 1a2faefd8..da1d6ab45 100644
--- a/app/workers/notification_worker.rb
+++ b/app/workers/notification_worker.rb
@@ -3,7 +3,7 @@
 class NotificationWorker
   include Sidekiq::Worker
 
-  sidekiq_options retry: 5
+  sidekiq_options queue: 'push', retry: 5
 
   def perform(xml, source_account_id, target_account_id)
     SendInteractionService.new.call(xml, Account.find(source_account_id), Account.find(target_account_id))
diff --git a/app/workers/processing_worker.rb b/app/workers/processing_worker.rb
index 5df404bcc..4a467d924 100644
--- a/app/workers/processing_worker.rb
+++ b/app/workers/processing_worker.rb
@@ -3,7 +3,7 @@
 class ProcessingWorker
   include Sidekiq::Worker
 
-  sidekiq_options backtrace: true
+  sidekiq_options queue: 'pull', backtrace: true
 
   def perform(account_id, body)
     ProcessFeedService.new.call(body, Account.find(account_id))
diff --git a/app/workers/regeneration_worker.rb b/app/workers/regeneration_worker.rb
index 3aece0ba2..289b63d84 100644
--- a/app/workers/regeneration_worker.rb
+++ b/app/workers/regeneration_worker.rb
@@ -3,6 +3,8 @@
 class RegenerationWorker
   include Sidekiq::Worker
 
+  sidekiq_options queue: 'pull', backtrace: true
+
   def perform(account_id, timeline_type)
     PrecomputeFeedService.new.call(timeline_type, Account.find(account_id))
   end
diff --git a/app/workers/salmon_worker.rb b/app/workers/salmon_worker.rb
index fc95ce47f..2888b574b 100644
--- a/app/workers/salmon_worker.rb
+++ b/app/workers/salmon_worker.rb
@@ -3,7 +3,7 @@
 class SalmonWorker
   include Sidekiq::Worker
 
-  sidekiq_options backtrace: true
+  sidekiq_options queue: 'pull', backtrace: true
 
   def perform(account_id, body)
     ProcessInteractionService.new.call(body, Account.find(account_id))
diff --git a/app/workers/thread_resolve_worker.rb b/app/workers/thread_resolve_worker.rb
index 593edd032..38287e8e6 100644
--- a/app/workers/thread_resolve_worker.rb
+++ b/app/workers/thread_resolve_worker.rb
@@ -3,7 +3,7 @@
 class ThreadResolveWorker
   include Sidekiq::Worker
 
-  sidekiq_options retry: false
+  sidekiq_options queue: 'pull', retry: false
 
   def perform(child_status_id, parent_url)
     child_status  = Status.find(child_status_id)
diff --git a/app/workers/unmerge_worker.rb b/app/workers/unmerge_worker.rb
index dbf7243de..ea6aacebf 100644
--- a/app/workers/unmerge_worker.rb
+++ b/app/workers/unmerge_worker.rb
@@ -3,6 +3,8 @@
 class UnmergeWorker
   include Sidekiq::Worker
 
+  sidekiq_options queue: 'pull'
+
   def perform(from_account_id, into_account_id)
     FeedManager.instance.unmerge_from_timeline(Account.find(from_account_id), Account.find(into_account_id))
   end
-- 
cgit 


From 4c53af64f0b10bc11473df5e3fd1cd7a11b755f6 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Tue, 4 Apr 2017 01:33:34 +0200
Subject: Fix ActionController::Parameters in API issue

---
 app/controllers/api/v1/apps_controller.rb     |  8 +++++++-
 app/controllers/api/v1/follows_controller.rb  |  8 ++++++--
 app/controllers/api/v1/media_controller.rb    |  8 +++++++-
 app/controllers/api/v1/reports_controller.rb  | 12 +++++++++---
 app/controllers/api/v1/statuses_controller.rb | 14 +++++++++-----
 app/models/status.rb                          |  2 +-
 6 files changed, 39 insertions(+), 13 deletions(-)

(limited to 'app')

diff --git a/app/controllers/api/v1/apps_controller.rb b/app/controllers/api/v1/apps_controller.rb
index ca9dd0b7e..2ec7280af 100644
--- a/app/controllers/api/v1/apps_controller.rb
+++ b/app/controllers/api/v1/apps_controller.rb
@@ -4,6 +4,12 @@ class Api::V1::AppsController < ApiController
   respond_to :json
 
   def create
-    @app = Doorkeeper::Application.create!(name: params[:client_name], redirect_uri: params[:redirect_uris], scopes: (params[:scopes] || Doorkeeper.configuration.default_scopes), website: params[:website])
+    @app = Doorkeeper::Application.create!(name: app_params[:client_name], redirect_uri: app_params[:redirect_uris], scopes: (app_params[:scopes] || Doorkeeper.configuration.default_scopes), website: app_params[:website])
+  end
+
+  private
+
+  def app_params
+    params.permit(:client_name, :redirect_uris, :scopes, :website)
   end
 end
diff --git a/app/controllers/api/v1/follows_controller.rb b/app/controllers/api/v1/follows_controller.rb
index c22dacbaa..7c0f44f03 100644
--- a/app/controllers/api/v1/follows_controller.rb
+++ b/app/controllers/api/v1/follows_controller.rb
@@ -7,7 +7,7 @@ class Api::V1::FollowsController < ApiController
   respond_to :json
 
   def create
-    raise ActiveRecord::RecordNotFound if params[:uri].blank?
+    raise ActiveRecord::RecordNotFound if follow_params[:uri].blank?
 
     @account = FollowService.new.call(current_user.account, target_uri).try(:target_account)
     render action: :show
@@ -16,6 +16,10 @@ class Api::V1::FollowsController < ApiController
   private
 
   def target_uri
-    params[:uri].strip.gsub(/\A@/, '')
+    follow_params[:uri].strip.gsub(/\A@/, '')
+  end
+
+  def follow_params
+    params.permit(:uri)
   end
 end
diff --git a/app/controllers/api/v1/media_controller.rb b/app/controllers/api/v1/media_controller.rb
index f8139ade7..aed3578d7 100644
--- a/app/controllers/api/v1/media_controller.rb
+++ b/app/controllers/api/v1/media_controller.rb
@@ -10,10 +10,16 @@ class Api::V1::MediaController < ApiController
   respond_to :json
 
   def create
-    @media = MediaAttachment.create!(account: current_user.account, file: params[:file])
+    @media = MediaAttachment.create!(account: current_user.account, file: media_params[:file])
   rescue Paperclip::Errors::NotIdentifiedByImageMagickError
     render json: { error: 'File type of uploaded media could not be verified' }, status: 422
   rescue Paperclip::Error
     render json: { error: 'Error processing thumbnail for uploaded media' }, status: 500
   end
+
+  private
+
+  def media_params
+    params.permit(:file)
+  end
 end
diff --git a/app/controllers/api/v1/reports_controller.rb b/app/controllers/api/v1/reports_controller.rb
index 46bdddbc1..f83c573cb 100644
--- a/app/controllers/api/v1/reports_controller.rb
+++ b/app/controllers/api/v1/reports_controller.rb
@@ -12,13 +12,19 @@ class Api::V1::ReportsController < ApiController
   end
 
   def create
-    status_ids = params[:status_ids].is_a?(Enumerable) ? params[:status_ids] : [params[:status_ids]]
+    status_ids = report_params[:status_ids].is_a?(Enumerable) ? report_params[:status_ids] : [report_params[:status_ids]]
 
     @report = Report.create!(account: current_account,
-                             target_account: Account.find(params[:account_id]),
+                             target_account: Account.find(report_params[:account_id]),
                              status_ids: Status.find(status_ids).pluck(:id),
-                             comment: params[:comment])
+                             comment: report_params[:comment])
 
     render :show
   end
+
+  private
+
+  def report_params
+    params.permit(:account_id, :comment, status_ids: [])
+  end
 end
diff --git a/app/controllers/api/v1/statuses_controller.rb b/app/controllers/api/v1/statuses_controller.rb
index 024258c0e..4ece7e702 100644
--- a/app/controllers/api/v1/statuses_controller.rb
+++ b/app/controllers/api/v1/statuses_controller.rb
@@ -62,11 +62,11 @@ class Api::V1::StatusesController < ApiController
   end
 
   def create
-    @status = PostStatusService.new.call(current_user.account, params[:status], params[:in_reply_to_id].blank? ? nil : Status.find(params[:in_reply_to_id]), media_ids: params[:media_ids],
-                                                                                                                                                             sensitive: params[:sensitive],
-                                                                                                                                                             spoiler_text: params[:spoiler_text],
-                                                                                                                                                             visibility: params[:visibility],
-                                                                                                                                                             application: doorkeeper_token.application)
+    @status = PostStatusService.new.call(current_user.account, status_params[:status], status_params[:in_reply_to_id].blank? ? nil : Status.find(status_params[:in_reply_to_id]), media_ids: status_params[:media_ids],
+                                                                                                                                                                                  sensitive: status_params[:sensitive],
+                                                                                                                                                                                  spoiler_text: status_params[:spoiler_text],
+                                                                                                                                                                                  visibility: status_params[:visibility],
+                                                                                                                                                                                  application: doorkeeper_token.application)
     render action: :show
   end
 
@@ -111,4 +111,8 @@ class Api::V1::StatusesController < ApiController
     @status = Status.find(params[:id])
     raise ActiveRecord::RecordNotFound unless @status.permitted?(current_account)
   end
+
+  def status_params
+    params.permit(:status, :in_reply_to_id, :sensitive, :spoiler_text, :visibility, media_ids: [])
+  end
 end
diff --git a/app/models/status.rb b/app/models/status.rb
index 81b26fd14..daf128572 100644
--- a/app/models/status.rb
+++ b/app/models/status.rb
@@ -188,7 +188,7 @@ class Status < ApplicationRecord
   end
 
   before_validation do
-    text.strip!
+    text&.strip!
     spoiler_text&.strip!
 
     self.reply                  = !(in_reply_to_id.nil? && thread.nil?) unless reply
-- 
cgit 


From b510a56c0c3d8c1a48bb295a85b688af94466723 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Tue, 4 Apr 2017 02:00:10 +0200
Subject: Only call regeneration worker after first login after a 14 day break

---
 app/controllers/application_controller.rb          |  9 ++++++++-
 app/controllers/oauth/authorizations_controller.rb |  7 +++++++
 app/models/feed.rb                                 | 12 ++----------
 app/workers/regeneration_worker.rb                 |  4 ++--
 4 files changed, 19 insertions(+), 13 deletions(-)

(limited to 'app')

diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb
index ef9364897..c06142fd4 100644
--- a/app/controllers/application_controller.rb
+++ b/app/controllers/application_controller.rb
@@ -39,7 +39,14 @@ class ApplicationController < ActionController::Base
   end
 
   def set_user_activity
-    current_user.touch(:current_sign_in_at) if !current_user.nil? && (current_user.current_sign_in_at.nil? || current_user.current_sign_in_at < 24.hours.ago)
+    return unless !current_user.nil? && (current_user.current_sign_in_at.nil? || current_user.current_sign_in_at < 24.hours.ago)
+
+    # Mark user as signed-in today
+    current_user.update_tracked_fields(request)
+
+    # If the sign in is after a two week break, we need to regenerate their feed
+    RegenerationWorker.perform_async(current_user.account_id) if current_user.last_sign_in_at < 14.days.ago
+    return
   end
 
   def check_suspension
diff --git a/app/controllers/oauth/authorizations_controller.rb b/app/controllers/oauth/authorizations_controller.rb
index feaad04f6..7c25266d8 100644
--- a/app/controllers/oauth/authorizations_controller.rb
+++ b/app/controllers/oauth/authorizations_controller.rb
@@ -3,6 +3,7 @@
 class Oauth::AuthorizationsController < Doorkeeper::AuthorizationsController
   skip_before_action :authenticate_resource_owner!
 
+  before_action :set_locale
   before_action :store_current_location
   before_action :authenticate_resource_owner!
 
@@ -11,4 +12,10 @@ class Oauth::AuthorizationsController < Doorkeeper::AuthorizationsController
   def store_current_location
     store_location_for(:user, request.url)
   end
+
+  def set_locale
+    I18n.locale = current_user.try(:locale) || I18n.default_locale
+  rescue I18n::InvalidLocale
+    I18n.locale = I18n.default_locale
+  end
 end
diff --git a/app/models/feed.rb b/app/models/feed.rb
index 5e1905e15..3cbc160a0 100644
--- a/app/models/feed.rb
+++ b/app/models/feed.rb
@@ -10,17 +10,9 @@ class Feed
     max_id     = '+inf' if max_id.blank?
     since_id   = '-inf' if since_id.blank?
     unhydrated = redis.zrevrangebyscore(key, "(#{max_id}", "(#{since_id}", limit: [0, limit], with_scores: true).map(&:last).map(&:to_i)
+    status_map = Status.where(id: unhydrated).cache_ids.map { |s| [s.id, s] }.to_h
 
-    # If we're after most recent items and none are there, we need to precompute the feed
-    if unhydrated.empty? && max_id == '+inf' && since_id == '-inf'
-      RegenerationWorker.perform_async(@account.id, @type)
-      @statuses = Status.send("as_#{@type}_timeline", @account).cache_ids.paginate_by_max_id(limit, nil, nil)
-    else
-      status_map = Status.where(id: unhydrated).cache_ids.map { |s| [s.id, s] }.to_h
-      @statuses  = unhydrated.map { |id| status_map[id] }.compact
-    end
-
-    @statuses
+    unhydrated.map { |id| status_map[id] }.compact
   end
 
   private
diff --git a/app/workers/regeneration_worker.rb b/app/workers/regeneration_worker.rb
index 289b63d84..82665b581 100644
--- a/app/workers/regeneration_worker.rb
+++ b/app/workers/regeneration_worker.rb
@@ -5,7 +5,7 @@ class RegenerationWorker
 
   sidekiq_options queue: 'pull', backtrace: true
 
-  def perform(account_id, timeline_type)
-    PrecomputeFeedService.new.call(timeline_type, Account.find(account_id))
+  def perform(account_id, _ = :home)
+    PrecomputeFeedService.new.call(:home, Account.find(account_id))
   end
 end
-- 
cgit 


From eb023beb4975a019d6a3b3091483c91c2c837bbd Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Tue, 4 Apr 2017 02:03:16 +0200
Subject: Fix #808 - smaller elephant friend PNG for frontpage

---
 app/assets/images/fluffy-elephant-friend.png | Bin 1101408 -> 60667 bytes
 1 file changed, 0 insertions(+), 0 deletions(-)

(limited to 'app')

diff --git a/app/assets/images/fluffy-elephant-friend.png b/app/assets/images/fluffy-elephant-friend.png
index 11787e936..f0df29927 100644
Binary files a/app/assets/images/fluffy-elephant-friend.png and b/app/assets/images/fluffy-elephant-friend.png differ
-- 
cgit 


From ce9df2fa8295a3bdd0da583ba5d0d90251e1d448 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Tue, 4 Apr 2017 13:01:14 +0200
Subject: Optimize filter methods in FeedManager a bit, use redis pipelining on
 merge/unmerge feed methods, do not re-create a dynamic class on each feed
 push call, make sure redis-rb uses hiredis

---
 Gemfile                      |  2 +-
 app/lib/feed_manager.rb      | 72 ++++++++++++++++++++------------------------
 app/lib/inline_rabl_scope.rb | 17 +++++++++++
 3 files changed, 50 insertions(+), 41 deletions(-)
 create mode 100644 app/lib/inline_rabl_scope.rb

(limited to 'app')

diff --git a/Gemfile b/Gemfile
index 46baed307..cb9824131 100644
--- a/Gemfile
+++ b/Gemfile
@@ -38,7 +38,7 @@ gem 'rqrcode'
 gem 'twitter-text'
 gem 'oj'
 gem 'hiredis'
-gem 'redis', '~>3.2'
+gem 'redis', '~>3.2', require: ['redis', 'redis/connection/hiredis']
 gem 'fast_blank'
 gem 'htmlentities'
 gem 'simple_form'
diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb
index cd6ca1291..2c29275c8 100644
--- a/app/lib/feed_manager.rb
+++ b/app/lib/feed_manager.rb
@@ -51,9 +51,11 @@ class FeedManager
   def merge_into_timeline(from_account, into_account)
     timeline_key = key(:home, into_account.id)
 
-    from_account.statuses.limit(MAX_ITEMS).each do |status|
-      next if status.direct_visibility? || filter?(:home, status, into_account)
-      redis.zadd(timeline_key, status.id, status.id)
+    redis.pipelined do
+      from_account.statuses.limit(MAX_ITEMS).each do |status|
+        next if status.direct_visibility? || filter?(:home, status, into_account)
+        redis.zadd(timeline_key, status.id, status.id)
+      end
     end
 
     trim(:home, into_account.id)
@@ -62,30 +64,18 @@ class FeedManager
   def unmerge_from_timeline(from_account, into_account)
     timeline_key = key(:home, into_account.id)
 
-    from_account.statuses.select('id').find_each do |status|
-      redis.zrem(timeline_key, status.id)
-      redis.zremrangebyscore(timeline_key, status.id, status.id)
+    from_account.statuses.select('id').find_in_batches do |statuses|
+      redis.pipelined do
+        statuses.each do |status|
+          redis.zrem(timeline_key, status.id)
+          redis.zremrangebyscore(timeline_key, status.id, status.id)
+        end
+      end
     end
   end
 
   def inline_render(target_account, template, object)
-    rabl_scope = Class.new do
-      include RoutingHelper
-
-      def initialize(account)
-        @account = account
-      end
-
-      def current_user
-        @account.try(:user)
-      end
-
-      def current_account
-        @account
-      end
-    end
-
-    Rabl::Renderer.new(template, object, view_path: 'app/views', format: :json, scope: rabl_scope.new(target_account)).render
+    Rabl::Renderer.new(template, object, view_path: 'app/views', format: :json, scope: InlineRablScope.new(target_account)).render
   end
 
   private
@@ -95,37 +85,39 @@ class FeedManager
   end
 
   def filter_from_home?(status, receiver)
-    return true if receiver.muting?(status.account)
+    return true if status.reply? && status.in_reply_to_id.nil?
+
+    check_for_mutes = [status.account_id]
+    check_for_mutes.concat([status.reblog.account_id]) if status.reblog?
+
+    return true if receiver.muting?(check_for_mutes)
 
-    should_filter = false
+    check_for_blocks = status.mentions.map(&:account_id)
+    check_for_blocks.concat([status.reblog.account_id]) if status.reblog?
 
-    if status.reply? && status.in_reply_to_id.nil?
-      should_filter = true
-    elsif status.reply? && !status.in_reply_to_account_id.nil?                # Filter out if it's a reply
+    return true if receiver.blocking?(check_for_blocks)
+
+    if status.reply? && !status.in_reply_to_account_id.nil?                   # Filter out if it's a reply
       should_filter   = !receiver.following?(status.in_reply_to_account)      # and I'm not following the person it's a reply to
       should_filter &&= !(receiver.id == status.in_reply_to_account_id)       # and it's not a reply to me
       should_filter &&= !(status.account_id == status.in_reply_to_account_id) # and it's not a self-reply
+      return should_filter
     elsif status.reblog?                                                      # Filter out a reblog
-      should_filter = receiver.blocking?(status.reblog.account)               # if I'm blocking the reblogged person
-      should_filter ||= receiver.muting?(status.reblog.account)               # or muting that person
-      should_filter ||= status.reblog.account.blocking?(receiver)             # or if the author of the reblogged status is blocking me
+      return status.reblog.account.blocking?(receiver)                        # or if the author of the reblogged status is blocking me
     end
 
-    should_filter ||= receiver.blocking?(status.mentions.map(&:account_id))   # or if it mentions someone I blocked
-
-    should_filter
+    false
   end
 
   def filter_from_mentions?(status, receiver)
+    check_for_blocks = [status.account_id]
+    check_for_blocks.concat(status.mentions.select('account_id').map(&:account_id))
+    check_for_blocks.concat([status.in_reply_to_account]) if status.reply? && !status.in_reply_to_account_id.nil?
+
     should_filter   = receiver.id == status.account_id                                      # Filter if I'm mentioning myself
-    should_filter ||= receiver.blocking?(status.account)                                    # or it's from someone I blocked
-    should_filter ||= receiver.blocking?(status.mentions.includes(:account).map(&:account)) # or if it mentions someone I blocked
+    should_filter ||= receiver.blocking?(check_for_blocks)                                  # or it's from someone I blocked, in reply to someone I blocked, or mentioning someone I blocked
     should_filter ||= (status.account.silenced? && !receiver.following?(status.account))    # of if the account is silenced and I'm not following them
 
-    if status.reply? && !status.in_reply_to_account_id.nil?                                 # or it's a reply
-      should_filter ||= receiver.blocking?(status.in_reply_to_account)                      # to a user I blocked
-    end
-
     should_filter
   end
 end
diff --git a/app/lib/inline_rabl_scope.rb b/app/lib/inline_rabl_scope.rb
new file mode 100644
index 000000000..26adcb03a
--- /dev/null
+++ b/app/lib/inline_rabl_scope.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+class InlineRablScope
+  include RoutingHelper
+
+  def initialize(account)
+    @account = account
+  end
+
+  def current_user
+    @account.try(:user)
+  end
+
+  def current_account
+    @account
+  end
+end
-- 
cgit 


From b21f7c28f6832817d5de616ab0c4c2d3c28d90b0 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Tue, 4 Apr 2017 13:02:49 +0200
Subject: Move OStatus processing back into default queue

---
 app/workers/processing_worker.rb | 2 +-
 app/workers/salmon_worker.rb     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

(limited to 'app')

diff --git a/app/workers/processing_worker.rb b/app/workers/processing_worker.rb
index 4a467d924..5df404bcc 100644
--- a/app/workers/processing_worker.rb
+++ b/app/workers/processing_worker.rb
@@ -3,7 +3,7 @@
 class ProcessingWorker
   include Sidekiq::Worker
 
-  sidekiq_options queue: 'pull', backtrace: true
+  sidekiq_options backtrace: true
 
   def perform(account_id, body)
     ProcessFeedService.new.call(body, Account.find(account_id))
diff --git a/app/workers/salmon_worker.rb b/app/workers/salmon_worker.rb
index 2888b574b..fc95ce47f 100644
--- a/app/workers/salmon_worker.rb
+++ b/app/workers/salmon_worker.rb
@@ -3,7 +3,7 @@
 class SalmonWorker
   include Sidekiq::Worker
 
-  sidekiq_options queue: 'pull', backtrace: true
+  sidekiq_options backtrace: true
 
   def perform(account_id, body)
     ProcessInteractionService.new.call(body, Account.find(account_id))
-- 
cgit 


From b1f3499c3806682375a0496f99b4bc908d89cd84 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Tue, 4 Apr 2017 13:43:36 +0200
Subject: Optimize FeedManager#unmerge, and slightly optimize FeedManager#merge

---
 app/lib/feed_manager.rb | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

(limited to 'app')

diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb
index 2c29275c8..919bc3df9 100644
--- a/app/lib/feed_manager.rb
+++ b/app/lib/feed_manager.rb
@@ -50,9 +50,15 @@ class FeedManager
 
   def merge_into_timeline(from_account, into_account)
     timeline_key = key(:home, into_account.id)
+    query        = from_account.statuses.limit(MAX_ITEMS)
+
+    if redis.zcard(timeline_key) >= FeedManager::MAX_ITEMS
+      oldest_home_score = redis.zrange(timeline_key, 0, 0, with_scores: true)&.first&.last&.to_i || 0
+      query = query.where('id > ?', oldest_home_score)
+    end
 
     redis.pipelined do
-      from_account.statuses.limit(MAX_ITEMS).each do |status|
+      query.each do |status|
         next if status.direct_visibility? || filter?(:home, status, into_account)
         redis.zadd(timeline_key, status.id, status.id)
       end
@@ -63,8 +69,9 @@ class FeedManager
 
   def unmerge_from_timeline(from_account, into_account)
     timeline_key = key(:home, into_account.id)
+    oldest_home_score = redis.zrange(timeline_key, 0, 0, with_scores: true)&.first&.last&.to_i || 0
 
-    from_account.statuses.select('id').find_in_batches do |statuses|
+    from_account.statuses.select('id').where('id > ?', oldest_home_score).find_in_batches do |statuses|
       redis.pipelined do
         statuses.each do |status|
           redis.zrem(timeline_key, status.id)
-- 
cgit 


From 82aaedec467815c2947a11651d5216bb88ce4038 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Tue, 4 Apr 2017 13:58:34 +0200
Subject: Reduce number of items in feeds, optimize regeneration worker
 slightly, make regeneration worker unique, (only schedule/execute once at a
 time)

---
 Gemfile                                 | 2 ++
 Gemfile.lock                            | 9 +++++++++
 app/lib/feed_manager.rb                 | 6 +++---
 app/services/precompute_feed_service.rb | 8 +++++---
 app/workers/regeneration_worker.rb      | 2 +-
 5 files changed, 20 insertions(+), 7 deletions(-)

(limited to 'app')

diff --git a/Gemfile b/Gemfile
index cb9824131..41c636904 100644
--- a/Gemfile
+++ b/Gemfile
@@ -46,6 +46,8 @@ gem 'will_paginate'
 gem 'rack-attack'
 gem 'rack-cors', require: 'rack/cors'
 gem 'sidekiq'
+gem 'sidekiq-unique-jobs'
+gem 'sidekiq-merger'
 gem 'rails-settings-cached'
 gem 'simple-navigation'
 gem 'statsd-instrument'
diff --git a/Gemfile.lock b/Gemfile.lock
index 6e3115249..27de1bee0 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -387,6 +387,13 @@ GEM
       connection_pool (~> 2.2, >= 2.2.0)
       rack-protection (>= 1.5.0)
       redis (~> 3.2, >= 3.2.1)
+    sidekiq-merger (0.0.11)
+      activesupport (>= 3.2, < 6)
+      concurrent-ruby (~> 1.0)
+      sidekiq (>= 3.4, < 5)
+    sidekiq-unique-jobs (4.0.18)
+      sidekiq (>= 2.6)
+      thor
     simple-navigation (4.0.3)
       activesupport (>= 2.3.2)
     simple_form (3.2.1)
@@ -510,6 +517,8 @@ DEPENDENCIES
   sass-rails (~> 5.0)
   sdoc (~> 0.4.0)
   sidekiq
+  sidekiq-merger
+  sidekiq-unique-jobs
   simple-navigation
   simple_form
   simplecov
diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb
index 919bc3df9..a2efcce10 100644
--- a/app/lib/feed_manager.rb
+++ b/app/lib/feed_manager.rb
@@ -5,7 +5,7 @@ require 'singleton'
 class FeedManager
   include Singleton
 
-  MAX_ITEMS = 800
+  MAX_ITEMS = 400
 
   def key(type, id)
     "feed:#{type}:#{id}"
@@ -50,9 +50,9 @@ class FeedManager
 
   def merge_into_timeline(from_account, into_account)
     timeline_key = key(:home, into_account.id)
-    query        = from_account.statuses.limit(MAX_ITEMS)
+    query        = from_account.statuses.limit(FeedManager::MAX_ITEMS / 4)
 
-    if redis.zcard(timeline_key) >= FeedManager::MAX_ITEMS
+    if redis.zcard(timeline_key) >= FeedManager::MAX_ITEMS / 4
       oldest_home_score = redis.zrange(timeline_key, 0, 0, with_scores: true)&.first&.last&.to_i || 0
       query = query.where('id > ?', oldest_home_score)
     end
diff --git a/app/services/precompute_feed_service.rb b/app/services/precompute_feed_service.rb
index e1ec56e8d..a57c401d0 100644
--- a/app/services/precompute_feed_service.rb
+++ b/app/services/precompute_feed_service.rb
@@ -5,9 +5,11 @@ class PrecomputeFeedService < BaseService
   # @param [Symbol] type :home or :mentions
   # @param [Account] account
   def call(_, account)
-    Status.as_home_timeline(account).limit(FeedManager::MAX_ITEMS).each do |status|
-      next if status.direct_visibility? || FeedManager.instance.filter?(:home, status, account)
-      redis.zadd(FeedManager.instance.key(:home, account.id), status.id, status.reblog? ? status.reblog_of_id : status.id)
+    redis.pipelined do
+      Status.as_home_timeline(account).limit(FeedManager::MAX_ITEMS / 4).each do |status|
+        next if status.direct_visibility? || FeedManager.instance.filter?(:home, status, account)
+        redis.zadd(FeedManager.instance.key(:home, account.id), status.id, status.reblog? ? status.reblog_of_id : status.id)
+      end
     end
   end
 
diff --git a/app/workers/regeneration_worker.rb b/app/workers/regeneration_worker.rb
index 82665b581..da8b845f6 100644
--- a/app/workers/regeneration_worker.rb
+++ b/app/workers/regeneration_worker.rb
@@ -3,7 +3,7 @@
 class RegenerationWorker
   include Sidekiq::Worker
 
-  sidekiq_options queue: 'pull', backtrace: true
+  sidekiq_options queue: 'pull', backtrace: true, unique: :until_executed
 
   def perform(account_id, _ = :home)
     PrecomputeFeedService.new.call(:home, Account.find(account_id))
-- 
cgit