From 922837dc96154b0455a4cf660c3f8369c65aacb4 Mon Sep 17 00:00:00 2001 From: Jean byroot Boussier Date: Sat, 4 Mar 2023 16:38:28 +0100 Subject: Upgrade to latest redis-rb 4.x and fix deprecations (#23616) Co-authored-by: Jean Boussier --- config/environments/test.rb | 2 ++ 1 file changed, 2 insertions(+) (limited to 'config/environments/test.rb') diff --git a/config/environments/test.rb b/config/environments/test.rb index ef3cb2e48..9cbf31e8d 100644 --- a/config/environments/test.rb +++ b/config/environments/test.rb @@ -73,3 +73,5 @@ end # Catch serialization warnings early Sidekiq.strict_args! + +Redis.raise_deprecations = true -- cgit From f0e727f958cd9428b2c56a3c6a65bbbf176bfa0d Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Thu, 16 Mar 2023 02:55:54 +0100 Subject: Add cache headers to static files served through Rails (#24120) --- config/application.rb | 5 +++++ config/environments/development.rb | 5 ----- config/environments/production.rb | 12 +--------- config/environments/test.rb | 5 ----- lib/public_file_server_middleware.rb | 43 ++++++++++++++++++++++++++++++++++++ 5 files changed, 49 insertions(+), 21 deletions(-) create mode 100644 lib/public_file_server_middleware.rb (limited to 'config/environments/test.rb') diff --git a/config/application.rb b/config/application.rb index c51eacd68..43631c551 100644 --- a/config/application.rb +++ b/config/application.rb @@ -35,6 +35,7 @@ require_relative '../lib/terrapin/multi_pipe_extensions' require_relative '../lib/mastodon/snowflake' require_relative '../lib/mastodon/version' require_relative '../lib/mastodon/rack_middleware' +require_relative '../lib/public_file_server_middleware' require_relative '../lib/devise/two_factor_ldap_authenticatable' require_relative '../lib/devise/two_factor_pam_authenticatable' require_relative '../lib/chewy/strategy/mastodon' @@ -181,6 +182,10 @@ module Mastodon config.active_job.queue_adapter = :sidekiq config.action_mailer.deliver_later_queue_name = 'mailers' + # We use our own middleware for this + config.public_file_server.enabled = false + + config.middleware.use PublicFileServerMiddleware if Rails.env.development? || ENV['RAILS_SERVE_STATIC_FILES'] == 'true' config.middleware.use Rack::Attack config.middleware.use Mastodon::RackMiddleware diff --git a/config/environments/development.rb b/config/environments/development.rb index f5f6cbed8..32c5cce8b 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -16,12 +16,7 @@ Rails.application.configure do # Run rails dev:cache to toggle caching. if Rails.root.join('tmp/caching-dev.txt').exist? config.action_controller.perform_caching = true - config.cache_store = :redis_cache_store, REDIS_CACHE_PARAMS - - config.public_file_server.headers = { - 'Cache-Control' => "public, max-age=#{2.days.to_i}", - } else config.action_controller.perform_caching = false config.cache_store = :null_store diff --git a/config/environments/production.rb b/config/environments/production.rb index b8536c53a..00d783477 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -19,24 +19,14 @@ Rails.application.configure do # or in config/master.key. This key is used to decrypt credentials (and other encrypted files). # config.require_master_key = true - # Disable serving static files from the `/public` folder by default since - # Apache or NGINX already handles this. - config.public_file_server.enabled = ENV['RAILS_SERVE_STATIC_FILES'].present? - ActiveSupport::Logger.new(STDOUT).tap do |logger| logger.formatter = config.log_formatter config.logger = ActiveSupport::TaggedLogging.new(logger) end - # Compress JavaScripts and CSS. - # config.assets.js_compressor = Uglifier.new(mangle: false) - # config.assets.css_compressor = :sass - # Do not fallback to assets pipeline if a precompiled asset is missed. config.assets.compile = false - # `config.assets.precompile` and `config.assets.version` have moved to config/initializers/assets.rb - # Specifies the header that your server uses for sending files. config.action_dispatch.x_sendfile_header = ENV['SENDFILE_HEADER'] if ENV['SENDFILE_HEADER'].present? @@ -66,7 +56,7 @@ Rails.application.configure do # Enable locale fallbacks for I18n (makes lookups for any locale fall back to # English when a translation cannot be found). - config.i18n.fallbacks = [:en] + config.i18n.fallbacks = true # Send deprecation notices to registered listeners. config.active_support.deprecation = :notify diff --git a/config/environments/test.rb b/config/environments/test.rb index 9cbf31e8d..1328e155a 100644 --- a/config/environments/test.rb +++ b/config/environments/test.rb @@ -12,11 +12,6 @@ Rails.application.configure do # preloads Rails for running tests, you may have to set it to true. config.eager_load = false - # Configure public file server for tests with Cache-Control for performance. - config.public_file_server.enabled = true - config.public_file_server.headers = { - 'Cache-Control' => "public, max-age=#{1.hour.to_i}" - } config.assets.digest = false # Show full error reports and disable caching. diff --git a/lib/public_file_server_middleware.rb b/lib/public_file_server_middleware.rb new file mode 100644 index 000000000..3799230a2 --- /dev/null +++ b/lib/public_file_server_middleware.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +require 'action_dispatch/middleware/static' + +class PublicFileServerMiddleware + SERVICE_WORKER_TTL = 7.days.to_i + CACHE_TTL = 28.days.to_i + + def initialize(app) + @app = app + @file_handler = ActionDispatch::FileHandler.new(Rails.application.paths['public'].first) + end + + def call(env) + file = @file_handler.attempt(env) + + # If the request is not a static file, move on! + return @app.call(env) if file.nil? + + status, headers, response = file + + # Set cache headers on static files. Some paths require different cache headers + headers['Cache-Control'] = begin + request_path = env['REQUEST_PATH'] + + if request_path.start_with?('/sw.js') + "public, max-age=#{SERVICE_WORKER_TTL}, must-revalidate" + elsif request_path.start_with?(paperclip_root_url) + "public, max-age=#{CACHE_TTL}, immutable" + else + "public, max-age=#{CACHE_TTL}, must-revalidate" + end + end + + [status, headers, response] + end + + private + + def paperclip_root_url + ENV.fetch('PAPERCLIP_ROOT_URL', '/system') + end +end -- cgit From 160f38f03d821b026ea62b09e1c817eb311507e1 Mon Sep 17 00:00:00 2001 From: Jean byroot Boussier Date: Fri, 17 Mar 2023 13:37:30 +0000 Subject: Workaround the ActiveRecord / Marshal serialization bug on Ruby 3.2 (#24142) Co-authored-by: Jean Boussier --- app/controllers/concerns/cache_concern.rb | 163 +++++++++++++++++++++++++++++- config/environments/test.rb | 6 ++ 2 files changed, 166 insertions(+), 3 deletions(-) (limited to 'config/environments/test.rb') diff --git a/app/controllers/concerns/cache_concern.rb b/app/controllers/concerns/cache_concern.rb index 05e431b19..e606218ac 100644 --- a/app/controllers/concerns/cache_concern.rb +++ b/app/controllers/concerns/cache_concern.rb @@ -3,6 +3,158 @@ module CacheConcern extend ActiveSupport::Concern + module ActiveRecordCoder + EMPTY_HASH = {}.freeze + + class << self + def dump(record) + instances = InstanceTracker.new + serialized_associations = serialize_associations(record, instances) + serialized_records = instances.map { |r| serialize_record(r) } + [serialized_associations, *serialized_records] + end + + def load(payload) + instances = InstanceTracker.new + serialized_associations, *serialized_records = payload + serialized_records.each { |attrs| instances.push(deserialize_record(*attrs)) } + deserialize_associations(serialized_associations, instances) + end + + private + + # Records without associations, or which have already been visited before, + # are serialized by their id alone. + # + # Records with associations are serialized as a two-element array including + # their id and the record's association cache. + # + def serialize_associations(record, instances) + return unless record + + if (id = instances.lookup(record)) + payload = id + else + payload = instances.push(record) + + cached_associations = record.class.reflect_on_all_associations.select do |reflection| + record.association_cached?(reflection.name) + end + + unless cached_associations.empty? + serialized_associations = cached_associations.map do |reflection| + association = record.association(reflection.name) + + serialized_target = if reflection.collection? + association.target.map { |target_record| serialize_associations(target_record, instances) } + else + serialize_associations(association.target, instances) + end + + [reflection.name, serialized_target] + end + + payload = [payload, serialized_associations] + end + end + + payload + end + + def deserialize_associations(payload, instances) + return unless payload + + id, associations = payload + record = instances.fetch(id) + + associations&.each do |name, serialized_target| + begin + association = record.association(name) + rescue ActiveRecord::AssociationNotFoundError + raise AssociationMissingError, "undefined association: #{name}" + end + + target = if association.reflection.collection? + serialized_target.map! { |serialized_record| deserialize_associations(serialized_record, instances) } + else + deserialize_associations(serialized_target, instances) + end + + association.target = target + end + + record + end + + def serialize_record(record) + arguments = [record.class.name, attributes_for_database(record)] + arguments << true if record.new_record? + arguments + end + + if Rails.gem_version >= Gem::Version.new('7.0') + def attributes_for_database(record) + attributes = record.attributes_for_database + attributes.transform_values! { |attr| attr.is_a?(::ActiveModel::Type::Binary::Data) ? attr.to_s : attr } + attributes + end + else + def attributes_for_database(record) + attributes = record.instance_variable_get(:@attributes).send(:attributes).transform_values(&:value_for_database) + attributes.transform_values! { |attr| attr.is_a?(::ActiveModel::Type::Binary::Data) ? attr.to_s : attr } + attributes + end + end + + def deserialize_record(class_name, attributes_from_database, new_record = false) # rubocop:disable Style/OptionalBooleanParameter + begin + klass = Object.const_get(class_name) + rescue NameError + raise ClassMissingError, "undefined class: #{class_name}" + end + + # Ideally we'd like to call `klass.instantiate`, however it doesn't allow to pass + # wether the record was persisted or not. + attributes = klass.attributes_builder.build_from_database(attributes_from_database, EMPTY_HASH) + klass.allocate.init_with_attributes(attributes, new_record) + end + end + + class Error < StandardError + end + + class ClassMissingError < Error + end + + class AssociationMissingError < Error + end + + class InstanceTracker + def initialize + @instances = [] + @ids = {}.compare_by_identity + end + + def map(&block) + @instances.map(&block) + end + + def fetch(...) + @instances.fetch(...) + end + + def push(instance) + id = @ids[instance] = @instances.size + @instances << instance + id + end + + def lookup(instance) + @ids[instance] + end + end + end + def render_with_cache(**options) raise ArgumentError, 'only JSON render calls are supported' unless options.key?(:json) || block_given? @@ -34,8 +186,13 @@ module CacheConcern raw = raw.cache_ids.to_a if raw.is_a?(ActiveRecord::Relation) return [] if raw.empty? - cached_keys_with_value = Rails.cache.read_multi(*raw).transform_keys(&:id) - uncached_ids = raw.map(&:id) - cached_keys_with_value.keys + cached_keys_with_value = begin + Rails.cache.read_multi(*raw, namespace: 'v2').transform_keys(&:id).transform_values { |r| ActiveRecordCoder.load(r) } + rescue ActiveRecordCoder::Error + {} # The serialization format may have changed, let's pretend it's a cache miss. + end + + uncached_ids = raw.map(&:id) - cached_keys_with_value.keys klass.reload_stale_associations!(cached_keys_with_value.values) if klass.respond_to?(:reload_stale_associations!) @@ -43,7 +200,7 @@ module CacheConcern uncached = klass.where(id: uncached_ids).with_includes.index_by(&:id) uncached.each_value do |item| - Rails.cache.write(item, item) + Rails.cache.write(item, ActiveRecordCoder.dump(item), namespace: 'v2') end end diff --git a/config/environments/test.rb b/config/environments/test.rb index 1328e155a..493b041eb 100644 --- a/config/environments/test.rb +++ b/config/environments/test.rb @@ -51,6 +51,12 @@ Rails.application.configure do config.i18n.default_locale = :en config.i18n.fallbacks = true + + config.to_prepare do + # Force Status to always be SHAPE_TOO_COMPLEX + # Ref: https://github.com/mastodon/mastodon/issues/23644 + 10.times { |i| Status.allocate.instance_variable_set(:"@ivar_#{i}", nil) } + end end Paperclip::Attachment.default_options[:path] = "#{Rails.root}/spec/test_files/:class/:id_partition/:style.:extension" -- cgit