From f722bd2387df9163760014e9555928ec487ae95f Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Tue, 4 Apr 2017 00:53:20 +0200
Subject: Separate background jobs into different queues. ATTENTION: new queue
 "pull" must be added to the Sidekiq invokation in your systemd file

The pull queue will handle link crawling, thread resolving, and OStatus
processing. Such tasks are more likely to hang for a longer time (due to
network requests) so it is more sensible to not make the "in-house" tasks
wait for them.
---
 app/workers/regeneration_worker.rb | 2 ++
 1 file changed, 2 insertions(+)

(limited to 'app/workers/regeneration_worker.rb')

diff --git a/app/workers/regeneration_worker.rb b/app/workers/regeneration_worker.rb
index 3aece0ba2..289b63d84 100644
--- a/app/workers/regeneration_worker.rb
+++ b/app/workers/regeneration_worker.rb
@@ -3,6 +3,8 @@
 class RegenerationWorker
   include Sidekiq::Worker
 
+  sidekiq_options queue: 'pull', backtrace: true
+
   def perform(account_id, timeline_type)
     PrecomputeFeedService.new.call(timeline_type, Account.find(account_id))
   end
-- 
cgit 


From b510a56c0c3d8c1a48bb295a85b688af94466723 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Tue, 4 Apr 2017 02:00:10 +0200
Subject: Only call regeneration worker after first login after a 14 day break

---
 app/controllers/application_controller.rb          |  9 ++++++++-
 app/controllers/oauth/authorizations_controller.rb |  7 +++++++
 app/models/feed.rb                                 | 12 ++----------
 app/workers/regeneration_worker.rb                 |  4 ++--
 4 files changed, 19 insertions(+), 13 deletions(-)

(limited to 'app/workers/regeneration_worker.rb')

diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb
index ef9364897..c06142fd4 100644
--- a/app/controllers/application_controller.rb
+++ b/app/controllers/application_controller.rb
@@ -39,7 +39,14 @@ class ApplicationController < ActionController::Base
   end
 
   def set_user_activity
-    current_user.touch(:current_sign_in_at) if !current_user.nil? && (current_user.current_sign_in_at.nil? || current_user.current_sign_in_at < 24.hours.ago)
+    return unless !current_user.nil? && (current_user.current_sign_in_at.nil? || current_user.current_sign_in_at < 24.hours.ago)
+
+    # Mark user as signed-in today
+    current_user.update_tracked_fields(request)
+
+    # If the sign in is after a two week break, we need to regenerate their feed
+    RegenerationWorker.perform_async(current_user.account_id) if current_user.last_sign_in_at < 14.days.ago
+    return
   end
 
   def check_suspension
diff --git a/app/controllers/oauth/authorizations_controller.rb b/app/controllers/oauth/authorizations_controller.rb
index feaad04f6..7c25266d8 100644
--- a/app/controllers/oauth/authorizations_controller.rb
+++ b/app/controllers/oauth/authorizations_controller.rb
@@ -3,6 +3,7 @@
 class Oauth::AuthorizationsController < Doorkeeper::AuthorizationsController
   skip_before_action :authenticate_resource_owner!
 
+  before_action :set_locale
   before_action :store_current_location
   before_action :authenticate_resource_owner!
 
@@ -11,4 +12,10 @@ class Oauth::AuthorizationsController < Doorkeeper::AuthorizationsController
   def store_current_location
     store_location_for(:user, request.url)
   end
+
+  def set_locale
+    I18n.locale = current_user.try(:locale) || I18n.default_locale
+  rescue I18n::InvalidLocale
+    I18n.locale = I18n.default_locale
+  end
 end
diff --git a/app/models/feed.rb b/app/models/feed.rb
index 5e1905e15..3cbc160a0 100644
--- a/app/models/feed.rb
+++ b/app/models/feed.rb
@@ -10,17 +10,9 @@ class Feed
     max_id     = '+inf' if max_id.blank?
     since_id   = '-inf' if since_id.blank?
     unhydrated = redis.zrevrangebyscore(key, "(#{max_id}", "(#{since_id}", limit: [0, limit], with_scores: true).map(&:last).map(&:to_i)
+    status_map = Status.where(id: unhydrated).cache_ids.map { |s| [s.id, s] }.to_h
 
-    # If we're after most recent items and none are there, we need to precompute the feed
-    if unhydrated.empty? && max_id == '+inf' && since_id == '-inf'
-      RegenerationWorker.perform_async(@account.id, @type)
-      @statuses = Status.send("as_#{@type}_timeline", @account).cache_ids.paginate_by_max_id(limit, nil, nil)
-    else
-      status_map = Status.where(id: unhydrated).cache_ids.map { |s| [s.id, s] }.to_h
-      @statuses  = unhydrated.map { |id| status_map[id] }.compact
-    end
-
-    @statuses
+    unhydrated.map { |id| status_map[id] }.compact
   end
 
   private
diff --git a/app/workers/regeneration_worker.rb b/app/workers/regeneration_worker.rb
index 289b63d84..82665b581 100644
--- a/app/workers/regeneration_worker.rb
+++ b/app/workers/regeneration_worker.rb
@@ -5,7 +5,7 @@ class RegenerationWorker
 
   sidekiq_options queue: 'pull', backtrace: true
 
-  def perform(account_id, timeline_type)
-    PrecomputeFeedService.new.call(timeline_type, Account.find(account_id))
+  def perform(account_id, _ = :home)
+    PrecomputeFeedService.new.call(:home, Account.find(account_id))
   end
 end
-- 
cgit 


From 82aaedec467815c2947a11651d5216bb88ce4038 Mon Sep 17 00:00:00 2001
From: Eugen Rochko <eugen@zeonfederated.com>
Date: Tue, 4 Apr 2017 13:58:34 +0200
Subject: Reduce number of items in feeds, optimize regeneration worker
 slightly, make regeneration worker unique, (only schedule/execute once at a
 time)

---
 Gemfile                                 | 2 ++
 Gemfile.lock                            | 9 +++++++++
 app/lib/feed_manager.rb                 | 6 +++---
 app/services/precompute_feed_service.rb | 8 +++++---
 app/workers/regeneration_worker.rb      | 2 +-
 5 files changed, 20 insertions(+), 7 deletions(-)

(limited to 'app/workers/regeneration_worker.rb')

diff --git a/Gemfile b/Gemfile
index cb9824131..41c636904 100644
--- a/Gemfile
+++ b/Gemfile
@@ -46,6 +46,8 @@ gem 'will_paginate'
 gem 'rack-attack'
 gem 'rack-cors', require: 'rack/cors'
 gem 'sidekiq'
+gem 'sidekiq-unique-jobs'
+gem 'sidekiq-merger'
 gem 'rails-settings-cached'
 gem 'simple-navigation'
 gem 'statsd-instrument'
diff --git a/Gemfile.lock b/Gemfile.lock
index 6e3115249..27de1bee0 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -387,6 +387,13 @@ GEM
       connection_pool (~> 2.2, >= 2.2.0)
       rack-protection (>= 1.5.0)
       redis (~> 3.2, >= 3.2.1)
+    sidekiq-merger (0.0.11)
+      activesupport (>= 3.2, < 6)
+      concurrent-ruby (~> 1.0)
+      sidekiq (>= 3.4, < 5)
+    sidekiq-unique-jobs (4.0.18)
+      sidekiq (>= 2.6)
+      thor
     simple-navigation (4.0.3)
       activesupport (>= 2.3.2)
     simple_form (3.2.1)
@@ -510,6 +517,8 @@ DEPENDENCIES
   sass-rails (~> 5.0)
   sdoc (~> 0.4.0)
   sidekiq
+  sidekiq-merger
+  sidekiq-unique-jobs
   simple-navigation
   simple_form
   simplecov
diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb
index 919bc3df9..a2efcce10 100644
--- a/app/lib/feed_manager.rb
+++ b/app/lib/feed_manager.rb
@@ -5,7 +5,7 @@ require 'singleton'
 class FeedManager
   include Singleton
 
-  MAX_ITEMS = 800
+  MAX_ITEMS = 400
 
   def key(type, id)
     "feed:#{type}:#{id}"
@@ -50,9 +50,9 @@ class FeedManager
 
   def merge_into_timeline(from_account, into_account)
     timeline_key = key(:home, into_account.id)
-    query        = from_account.statuses.limit(MAX_ITEMS)
+    query        = from_account.statuses.limit(FeedManager::MAX_ITEMS / 4)
 
-    if redis.zcard(timeline_key) >= FeedManager::MAX_ITEMS
+    if redis.zcard(timeline_key) >= FeedManager::MAX_ITEMS / 4
       oldest_home_score = redis.zrange(timeline_key, 0, 0, with_scores: true)&.first&.last&.to_i || 0
       query = query.where('id > ?', oldest_home_score)
     end
diff --git a/app/services/precompute_feed_service.rb b/app/services/precompute_feed_service.rb
index e1ec56e8d..a57c401d0 100644
--- a/app/services/precompute_feed_service.rb
+++ b/app/services/precompute_feed_service.rb
@@ -5,9 +5,11 @@ class PrecomputeFeedService < BaseService
   # @param [Symbol] type :home or :mentions
   # @param [Account] account
   def call(_, account)
-    Status.as_home_timeline(account).limit(FeedManager::MAX_ITEMS).each do |status|
-      next if status.direct_visibility? || FeedManager.instance.filter?(:home, status, account)
-      redis.zadd(FeedManager.instance.key(:home, account.id), status.id, status.reblog? ? status.reblog_of_id : status.id)
+    redis.pipelined do
+      Status.as_home_timeline(account).limit(FeedManager::MAX_ITEMS / 4).each do |status|
+        next if status.direct_visibility? || FeedManager.instance.filter?(:home, status, account)
+        redis.zadd(FeedManager.instance.key(:home, account.id), status.id, status.reblog? ? status.reblog_of_id : status.id)
+      end
     end
   end
 
diff --git a/app/workers/regeneration_worker.rb b/app/workers/regeneration_worker.rb
index 82665b581..da8b845f6 100644
--- a/app/workers/regeneration_worker.rb
+++ b/app/workers/regeneration_worker.rb
@@ -3,7 +3,7 @@
 class RegenerationWorker
   include Sidekiq::Worker
 
-  sidekiq_options queue: 'pull', backtrace: true
+  sidekiq_options queue: 'pull', backtrace: true, unique: :until_executed
 
   def perform(account_id, _ = :home)
     PrecomputeFeedService.new.call(:home, Account.find(account_id))
-- 
cgit