Merge branch 'glitch'`

14 files changed, 116 insertions, 58 deletions

diff --git a/config/initializers/0_duplicate_migrations.rb b/config/initializers/0_duplicate_migrations.rb
new file mode 100644
index 000000000..4ab806587
--- /dev/null
+++ b/config/initializers/0_duplicate_migrations.rb
@@ -0,0 +1,40 @@
+# Some migrations have been present in glitch-soc for a long time and have then
+# been merged in upstream Mastodon, under a different version number.
+#
+# This puts us in an uneasy situation in which if we remove upstream's
+# migration file, people migrating from upstream will end up having a conflict
+# with their already-ran migration.
+#
+# On the other hand, if we keep upstream's migration and remove our own,
+# any current glitch-soc user will have a conflict during migration.
+#
+# For lack of a better solution, as those migrations are indeed identical,
+# we decided monkey-patching Rails' Migrator to completely ignore the duplicate,
+# keeping only the one that has run, or an arbitrary one.
+
+ALLOWED_DUPLICATES = [20180410220657, 20180831171112].freeze
+
+module ActiveRecord
+  class Migrator
+    def self.new(direction, migrations, target_version = nil)
+      migrated = Set.new(Base.connection.migration_context.get_all_versions)
+
+      migrations.group_by(&:name).each do |name, duplicates|
+        if duplicates.length > 1 && duplicates.all? { |m| ALLOWED_DUPLICATES.include?(m.version) }
+          # We have a set of allowed duplicates. Keep the migrated one, if any.
+          non_migrated = duplicates.reject { |m| migrated.include?(m.version.to_i) }
+
+          if duplicates.length == non_migrated.length || non_migrated.length == 0
+            # There weren't any migrated one, so we have to pick one “canonical” migration
+            migrations = migrations - duplicates[1..-1]
+          else
+            # Just reject every duplicate which hasn't been migrated yet
+            migrations = migrations - non_migrated
+          end
+        end
+      end
+
+      super(direction, migrations, target_version)
+    end
+  end
+end
diff --git a/config/initializers/2_whitelist_mode.rb b/config/initializers/2_whitelist_mode.rb
new file mode 100644
index 000000000..a17ad07a2
--- /dev/null
+++ b/config/initializers/2_whitelist_mode.rb
@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+Rails.application.configure do
+  config.x.whitelist_mode = ENV['WHITELIST_MODE'] == 'true'
+end
diff --git a/config/initializers/active_model_serializers.rb b/config/initializers/active_model_serializers.rb
index 329a5fb2c..0e69e1d96 100644
--- a/config/initializers/active_model_serializers.rb
+++ b/config/initializers/active_model_serializers.rb
@@ -3,22 +3,3 @@ ActiveModelSerializers.config.tap do |config|
 end
 
 ActiveSupport::Notifications.unsubscribe(ActiveModelSerializers::Logging::RENDER_EVENT)
-
-class ActiveModel::Serializer::Reflection
-  # We monkey-patch this method so that when we include associations in a serializer,
-  # the nested serializers can send information about used contexts upwards back to
-  # the root. We do this via instance_options because the nesting can be dynamic.
-  def build_association(parent_serializer, parent_serializer_options, include_slice = {})
-    serializer = options[:serializer]
-
-    parent_serializer_options.merge!(named_contexts: serializer._named_contexts, context_extensions: serializer._context_extensions) if serializer.respond_to?(:_named_contexts)
-
-    association_options = {
-      parent_serializer: parent_serializer,
-      parent_serializer_options: parent_serializer_options,
-      include_slice: include_slice,
-    }
-
-    ActiveModel::Serializer::Association.new(self, association_options)
-  end
-end
diff --git a/config/initializers/chewy.rb b/config/initializers/chewy.rb
index d5347f2bf..9ff0dccc1 100644
--- a/config/initializers/chewy.rb
+++ b/config/initializers/chewy.rb
@@ -12,8 +12,9 @@ Chewy.settings = {
   sidekiq: { queue: 'pull' },
 }
 
-Chewy.root_strategy    = enabled ? :sidekiq : :bypass
-Chewy.request_strategy = enabled ? :sidekiq : :bypass
+Chewy.root_strategy              = :custom_sidekiq
+Chewy.request_strategy           = :custom_sidekiq
+Chewy.use_after_commit_callbacks = false
 
 module Chewy
   class << self
diff --git a/config/initializers/content_security_policy.rb b/config/initializers/content_security_policy.rb
index 2fe1a33fa..810aa2880 100644
--- a/config/initializers/content_security_policy.rb
+++ b/config/initializers/content_security_policy.rb
@@ -35,8 +35,8 @@ if Rails.env.production?
     p.style_src       :self, :unsafe_inline, assets_host
     p.media_src       :self, :data, *data_hosts
     p.frame_src       :self, :https
-    p.worker_src      :self, assets_host
-    p.connect_src     :self, :blob, Rails.configuration.x.streaming_api_base_url, *data_hosts
+    p.worker_src      :self, :blob, assets_host
+    p.connect_src     :self, :blob, :data, Rails.configuration.x.streaming_api_base_url, *data_hosts
     p.manifest_src    :self, assets_host
   end
 end
diff --git a/config/initializers/devise.rb b/config/initializers/devise.rb
index cd9bacf68..59e69ad37 100644
--- a/config/initializers/devise.rb
+++ b/config/initializers/devise.rb
@@ -53,6 +53,8 @@ module Devise
   @@ldap_base = nil
   mattr_accessor :ldap_uid
   @@ldap_uid = nil
+  mattr_accessor :ldap_mail
+  @@ldap_mail = nil
   mattr_accessor :ldap_bind_dn
   @@ldap_bind_dn = nil
   mattr_accessor :ldap_password
@@ -61,6 +63,12 @@ module Devise
   @@ldap_tls_no_verify = false
   mattr_accessor :ldap_search_filter
   @@ldap_search_filter = nil
+  mattr_accessor :ldap_uid_conversion_enabled
+  @@ldap_uid_conversion_enabled = false
+  mattr_accessor :ldap_uid_conversion_search
+  @@ldap_uid_conversion_search = nil
+  mattr_accessor :ldap_uid_conversion_replace
+  @@ldap_uid_conversion_replace = nil
 
   class Strategies::PamAuthenticatable
     def valid?
@@ -71,8 +79,8 @@ end
 
 Devise.setup do |config|
   config.warden do |manager|
-    manager.default_strategies(scope: :user).unshift :ldap_authenticatable if Devise.ldap_authentication
-    manager.default_strategies(scope: :user).unshift :pam_authenticatable  if Devise.pam_authentication
+    manager.default_strategies(scope: :user).unshift :two_factor_ldap_authenticatable if Devise.ldap_authentication
+    manager.default_strategies(scope: :user).unshift :two_factor_pam_authenticatable  if Devise.pam_authentication
     manager.default_strategies(scope: :user).unshift :two_factor_authenticatable
     manager.default_strategies(scope: :user).unshift :two_factor_backupable
   end
@@ -363,7 +371,11 @@ Devise.setup do |config|
     config.ldap_bind_dn        = ENV.fetch('LDAP_BIND_DN')
     config.ldap_password       = ENV.fetch('LDAP_PASSWORD')
     config.ldap_uid            = ENV.fetch('LDAP_UID', 'cn')
+    config.ldap_mail           = ENV.fetch('LDAP_MAIL', 'mail')
     config.ldap_tls_no_verify  = ENV['LDAP_TLS_NO_VERIFY'] == 'true'
-    config.ldap_search_filter  = ENV.fetch('LDAP_SEARCH_FILTER', '%{uid}=%{email}')
+    config.ldap_search_filter  = ENV.fetch('LDAP_SEARCH_FILTER', '(|(%{uid}=%{email})(%{mail}=%{email}))')
+    config.ldap_uid_conversion_enabled  = ENV['LDAP_UID_CONVERSION_ENABLED'] == 'true'
+    config.ldap_uid_conversion_search   = ENV.fetch('LDAP_UID_CONVERSION_SEARCH', '.,- ')
+    config.ldap_uid_conversion_replace  = ENV.fetch('LDAP_UID_CONVERSION_REPLACE', '_')
   end
 end
diff --git a/config/initializers/doorkeeper.rb b/config/initializers/doorkeeper.rb
index 2a963b32b..7784bec62 100644
--- a/config/initializers/doorkeeper.rb
+++ b/config/initializers/doorkeeper.rb
@@ -8,8 +8,20 @@ Doorkeeper.configure do
   end
 
   resource_owner_from_credentials do |_routes|
-    user = User.find_by(email: request.params[:username])
-    user if !user&.otp_required_for_login? && user&.valid_password?(request.params[:password])
+    if Devise.ldap_authentication
+      user = User.authenticate_with_ldap({ :email => request.params[:username], :password => request.params[:password] })
+    end
+
+    if Devise.pam_authentication
+      user ||= User.authenticate_with_ldap({ :email => request.params[:username], :password => request.params[:password] })
+    end
+
+    if user.nil?
+      user = User.find_by(email: request.params[:username])
+      user = nil unless user.valid_password?(request.params[:password])
+    end
+
+    user if !user&.otp_required_for_login?
   end
 
   # If you want to restrict access to the web interface for adding oauth authorized applications, you need to declare the block below.
@@ -82,7 +94,13 @@ Doorkeeper.configure do
                   :'read:search',
                   :'read:statuses',
                   :follow,
-                  :push
+                  :push,
+                  :'admin:read',
+                  :'admin:read:accounts',
+                  :'admin:read:reports',
+                  :'admin:write',
+                  :'admin:write:accounts',
+                  :'admin:write:reports'
 
   # Change the way client credentials are retrieved from the request object.
   # By default it retrieves first from the `HTTP_AUTHORIZATION` header, then
diff --git a/config/initializers/health_check.rb b/config/initializers/health_check.rb
new file mode 100644
index 000000000..6f1e78fed
--- /dev/null
+++ b/config/initializers/health_check.rb
@@ -0,0 +1,8 @@
+HealthCheck.setup do |config|
+  config.uri = 'health'
+
+  config.standard_checks = %w(database migrations cache)
+  config.full_checks = %w(database migrations cache)
+  
+  config.include_error_in_response_body = false
+end
diff --git a/config/initializers/inflections.rb b/config/initializers/inflections.rb
index bf0cb52a3..c65153b0a 100644
--- a/config/initializers/inflections.rb
+++ b/config/initializers/inflections.rb
@@ -18,4 +18,5 @@ ActiveSupport::Inflector.inflections(:en) do |inflect|
   inflect.acronym 'PubSubHubbub'
   inflect.acronym 'ActivityStreams'
   inflect.acronym 'JsonLd'
+  inflect.acronym 'NodeInfo'
 end
diff --git a/config/initializers/instrumentation.rb b/config/initializers/instrumentation.rb
deleted file mode 100644
index 8483f2be2..000000000
--- a/config/initializers/instrumentation.rb
+++ /dev/null
@@ -1,18 +0,0 @@
-# frozen_string_literal: true
-
-instrumentation_hostname = ENV.fetch('INSTRUMENTATION_HOSTNAME') { 'localhost' }
-
-ActiveSupport::Notifications.subscribe(/process_action.action_controller/) do |*args|
-  event      = ActiveSupport::Notifications::Event.new(*args)
-  controller = event.payload[:controller]
-  action     = event.payload[:action]
-  format     = event.payload[:format] || 'all'
-  format     = 'all' if format == '*/*'
-  status     = event.payload[:status]
-  key        = "#{controller}.#{action}.#{format}.#{instrumentation_hostname}"
-
-  ActiveSupport::Notifications.instrument :performance, action: :measure, measurement: "#{key}.total_duration", value: event.duration
-  ActiveSupport::Notifications.instrument :performance, action: :measure, measurement: "#{key}.db_time", value: event.payload[:db_runtime]
-  ActiveSupport::Notifications.instrument :performance, action: :measure, measurement: "#{key}.view_time", value: event.payload[:view_runtime]
-  ActiveSupport::Notifications.instrument :performance, measurement: "#{key}.status.#{status}"
-end
diff --git a/config/initializers/json_ld.rb b/config/initializers/json_ld.rb
index d5575d135..3ed3c4b31 100644
--- a/config/initializers/json_ld.rb
+++ b/config/initializers/json_ld.rb
@@ -1,3 +1,4 @@
 # frozen_string_literal: true
 
 require_relative '../../lib/json_ld/security'
+require_relative '../../lib/json_ld/identity'
diff --git a/config/initializers/paperclip.rb b/config/initializers/paperclip.rb
index ce4185e02..96607b7ce 100644
--- a/config/initializers/paperclip.rb
+++ b/config/initializers/paperclip.rb
@@ -1,10 +1,11 @@
 # frozen_string_literal: true
 
-Paperclip.options[:read_timeout] = 60
-
 Paperclip.interpolates :filename do |attachment, style|
-  return attachment.original_filename if style == :original
-  [basename(attachment, style), extension(attachment, style)].delete_if(&:blank?).join('.')
+  if style == :original
+    attachment.original_filename
+  else
+    [basename(attachment, style), extension(attachment, style)].delete_if(&:blank?).join('.')
+  end
 end
 
 Paperclip::Attachment.default_options.merge!(
@@ -24,21 +25,27 @@ if ENV['S3_ENABLED'] == 'true'
     storage: :s3,
     s3_protocol: s3_protocol,
     s3_host_name: s3_hostname,
+
     s3_headers: {
+      'X-Amz-Multipart-Threshold' => ENV.fetch('S3_MULTIPART_THRESHOLD') { 15.megabytes }.to_i,
       'Cache-Control' => 'public, max-age=315576000, immutable',
     },
+
     s3_permissions: ENV.fetch('S3_PERMISSION') { 'public-read' },
     s3_region: s3_region,
+
     s3_credentials: {
       bucket: ENV['S3_BUCKET'],
       access_key_id: ENV['AWS_ACCESS_KEY_ID'],
       secret_access_key: ENV['AWS_SECRET_ACCESS_KEY'],
     },
+
     s3_options: {
       signature_version: ENV.fetch('S3_SIGNATURE_VERSION') { 'v4' },
-      http_open_timeout: 5,
+      http_open_timeout: ENV.fetch('S3_OPEN_TIMEOUT'){ '5' }.to_i,
       http_read_timeout: 5,
       http_idle_timeout: 5,
+      retry_limit: 0,
     }
   )
 
@@ -47,6 +54,7 @@ if ENV['S3_ENABLED'] == 'true'
       endpoint: ENV['S3_ENDPOINT'],
       force_path_style: true
     )
+
     Paperclip::Attachment.default_options[:url] = ':s3_path_url'
   end
 
@@ -72,6 +80,7 @@ elsif ENV['SWIFT_ENABLED'] == 'true'
       openstack_region: ENV['SWIFT_REGION'],
       openstack_cache_ttl: ENV.fetch('SWIFT_CACHE_TTL') { 60 },
     },
+
     fog_directory: ENV['SWIFT_CONTAINER'],
     fog_host: ENV['SWIFT_OBJECT_URL'],
     fog_public: true
@@ -80,7 +89,7 @@ else
   Paperclip::Attachment.default_options.merge!(
     storage: :filesystem,
     use_timestamp: true,
-    path: (ENV['PAPERCLIP_ROOT_PATH'] || ':rails_root/public/system') + '/:class/:attachment/:id_partition/:style/:filename',
-    url: (ENV['PAPERCLIP_ROOT_URL'] || '/system') + '/:class/:attachment/:id_partition/:style/:filename',
+    path: ENV.fetch('PAPERCLIP_ROOT_PATH', ':rails_root/public/system') + '/:class/:attachment/:id_partition/:style/:filename',
+    url: ENV.fetch('PAPERCLIP_ROOT_URL', '/system') + '/:class/:attachment/:id_partition/:style/:filename',
   )
 end
diff --git a/config/initializers/rack_attack.rb b/config/initializers/rack_attack.rb
index 24ba16ae3..273cac9ca 100644
--- a/config/initializers/rack_attack.rb
+++ b/config/initializers/rack_attack.rb
@@ -65,7 +65,7 @@ class Rack::Attack
     req.authenticated_user_id if req.post? && req.path.start_with?('/api/v1/media')
   end
 
-  throttle('throttle_media_proxy', limit: 30, period: 30.minutes) do |req|
+  throttle('throttle_media_proxy', limit: 30, period: 10.minutes) do |req|
     req.remote_ip if req.path.start_with?('/media_proxy')
   end
 
diff --git a/config/initializers/statsd.rb b/config/initializers/statsd.rb
index ce83fd9de..93ea1d1e4 100644
--- a/config/initializers/statsd.rb
+++ b/config/initializers/statsd.rb
@@ -3,10 +3,10 @@
 if ENV['STATSD_ADDR'].present?
   host, port = ENV['STATSD_ADDR'].split(':')
 
-  statsd = ::Statsd.new(host, port)
-  statsd.namespace = ENV.fetch('STATSD_NAMESPACE') { ['Mastodon', Rails.env].join('.') }
+  $statsd = ::Statsd.new(host, port)
+  $statsd.namespace = ENV.fetch('STATSD_NAMESPACE') { ['Mastodon', Rails.env].join('.') }
 
-  ::NSA.inform_statsd(statsd) do |informant|
+  ::NSA.inform_statsd($statsd) do |informant|
     informant.collect(:action_controller, :web)
     informant.collect(:active_record, :db)
     informant.collect(:active_support_cache, :cache)