Adding Sidekiq for background processing (firstly just of mailers)

6 files changed, 46 insertions, 2 deletions

diff --git a/Gemfile b/Gemfile
index 48bdbaee3..77b9c743b 100644
--- a/Gemfile
+++ b/Gemfile
@@ -37,6 +37,8 @@ gem 'simple_form'
 gem 'will_paginate', '~> 3.0.6'
 gem 'rack-attack'
 gem 'turbolinks', '~> 5.0.0.beta'
+gem 'sidekiq'
+gem 'sinatra', :require => nil
 
 group :development, :test do
   gem 'rspec-rails'
diff --git a/Gemfile.lock b/Gemfile.lock
index d17b63a1b..b7000b0b6 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -60,6 +60,7 @@ GEM
       execjs
     coffee-script-source (1.10.0)
     concurrent-ruby (1.0.1)
+    connection_pool (2.2.0)
     crack (0.4.3)
       safe_yaml (~> 1.0.0)
     debug_inspector (0.0.2)
@@ -188,6 +189,8 @@ GEM
       rack
     rack-mini-profiler (0.9.9.2)
       rack (>= 1.2.0)
+    rack-protection (1.5.3)
+      rack
     rack-test (0.6.3)
       rack (>= 1.0)
     rails (4.2.5.2)
@@ -271,6 +274,10 @@ GEM
       json (~> 1.7, >= 1.7.7)
       rdoc (~> 4.0)
     sexp_processor (4.7.0)
+    sidekiq (4.1.1)
+      concurrent-ruby (~> 1.0)
+      connection_pool (~> 2.2, >= 2.2.0)
+      redis (~> 3.2, >= 3.2.1)
     simple_form (3.2.1)
       actionpack (> 4, < 5.1)
       activemodel (> 4, < 5.1)
@@ -279,6 +286,10 @@ GEM
       json (~> 1.8)
       simplecov-html (~> 0.10.0)
     simplecov-html (0.10.0)
+    sinatra (1.4.7)
+      rack (~> 1.5)
+      rack-protection (~> 1.4)
+      tilt (>= 1.3, < 3)
     slop (3.6.0)
     sprockets (3.5.2)
       concurrent-ruby (~> 1.0)
@@ -362,8 +373,10 @@ DEPENDENCIES
   rubocop
   sass-rails (~> 5.0)
   sdoc (~> 0.4.0)
+  sidekiq
   simple_form
   simplecov
+  sinatra
   therubyracer
   turbolinks (~> 5.0.0.beta)
   uglifier (>= 1.3.0)
diff --git a/config/application.rb b/config/application.rb
index 9d8d6022d..1fbd8f7f6 100644
--- a/config/application.rb
+++ b/config/application.rb
@@ -28,6 +28,8 @@ module Mastodon
     config.paths.add File.join('app', 'api'), glob: File.join('**', '*.rb')
     config.autoload_paths += Dir[Rails.root.join('app', 'api', '*')]
 
+    config.active_job.queue_adapter = :sidekiq
+
     config.to_prepare do
       Doorkeeper::ApplicationsController.layout           'dashboard'
       Doorkeeper::AuthorizedApplicationsController.layout 'dashboard'
diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb
new file mode 100644
index 000000000..83958d700
--- /dev/null
+++ b/config/initializers/sidekiq.rb
@@ -0,0 +1,11 @@
+redis_conn = proc {
+  $redis.dup
+}
+
+Sidekiq.configure_server do |config|
+  config.redis = ConnectionPool.new(size: 5, &redis_conn)
+end
+
+Sidekiq.configure_client do |config|
+  config.redis = ConnectionPool.new(size: 25, &redis_conn)
+end
diff --git a/config/routes.rb b/config/routes.rb
index f595438b7..0dde9f111 100644
--- a/config/routes.rb
+++ b/config/routes.rb
@@ -1,4 +1,10 @@
+require 'sidekiq/web'
+
 Rails.application.routes.draw do
+  authenticate :user do
+    mount Sidekiq::Web => '/sidekiq'
+  end
+
   use_doorkeeper do
     controllers applications: 'oauth/applications'
   end
diff --git a/docker-compose.yml b/docker-compose.yml
index ed29808b0..b3b233713 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -4,8 +4,11 @@ services:
     image: postgres
   redis:
     image: redis
-  web:
+  app:
     build: .
+    env_file: .env.production
+  web:
+    extends: app
     command: bundle exec rails s -p 3000 -b '0.0.0.0'
     ports:
       - "3000:3000"
@@ -15,4 +18,11 @@ services:
     volumes:
       - ./public/assets:/mastodon/public/assets
       - ./public/system:/mastodon/public/system
-    env_file: .env.production
+  sidekiq:
+    extends: app
+    command: bundle exec sidekiq -q default -q mailers
+    depends_on:
+      - db
+      - redis
+    volumes:
+      - ./public/system:/mastodon/public/system