From 43eff898a0b0f31aaf042d9d387aaece2627a01d Mon Sep 17 00:00:00 2001 From: Claire Date: Wed, 17 Mar 2021 10:09:55 +0100 Subject: Prepare Mastodon for Rails 6 (#15911) * Fix misuse of foreign_type * Fix use of removed "add_template_helper" * Use response.media_type instead of response.content_type in tests * Fix CSV export controller test on Rails 6 Rails 6 sets a "filename*" field in the Content-Disposition header to explicitly encode the filename as UTF-8. This changes checks the first part of the Content-Disposition header so it matches in both Rails 5 and Rails 6. * Fix emoji formatting with Rails 6 * Make emoji output more idiomatic and robust * Switch from redis-rails gem to built-in Rails redis cache storage --- lib/mastodon/redis_config.rb | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/mastodon/redis_config.rb b/lib/mastodon/redis_config.rb index c3c8ff800..3f2a8f7c2 100644 --- a/lib/mastodon/redis_config.rb +++ b/lib/mastodon/redis_config.rb @@ -27,6 +27,8 @@ namespace = ENV.fetch('REDIS_NAMESPACE', nil) cache_namespace = namespace ? namespace + '_cache' : 'cache' REDIS_CACHE_PARAMS = { + driver: :hiredis, + url: ENV['REDIS_URL'], expires_in: 10.minutes, namespace: cache_namespace, }.freeze -- cgit From a4dcaef53b97c58fd153de6f151b6fada40f3442 Mon Sep 17 00:00:00 2001 From: Claire Date: Fri, 19 Mar 2021 02:42:43 +0100 Subject: Prepare Mastodon for zeitwerk autoloader (#15917) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Prepare Mastodon for zeitwerk autoloader (Rails 6) Add inflections and rename/move a few classes. In particular, app/lib/exceptions.rb and app/lib/sanitize_config.rb were manually loaded while still in autoload paths. * Add inflection for Url → URL --- app/lib/exceptions.rb | 23 ---- app/lib/formatter.rb | 1 - app/lib/sanitize_config.rb | 119 --------------------- app/validators/url_validator.rb | 2 +- config/application.rb | 3 +- config/initializers/inflections.rb | 4 + db/migrate/20160223165723_add_url_to_statuses.rb | 2 +- db/migrate/20160223165855_add_url_to_accounts.rb | 2 +- ...0322193748_add_avatar_remote_url_to_accounts.rb | 2 +- ...0318214217_add_header_remote_url_to_accounts.rb | 2 +- ...0171130000000_add_embed_url_to_preview_cards.rb | 2 +- ...3859_add_featured_collection_url_to_accounts.rb | 2 +- .../20200529214050_add_devices_url_to_accounts.rb | 2 +- lib/exceptions.rb | 23 ++++ lib/sanitize_ext/sanitize_config.rb | 119 +++++++++++++++++++++ spec/lib/sanitize_config_spec.rb | 1 - spec/validators/url_validator_spec.rb | 2 +- 17 files changed, 157 insertions(+), 154 deletions(-) delete mode 100644 app/lib/exceptions.rb delete mode 100644 app/lib/sanitize_config.rb create mode 100644 lib/exceptions.rb create mode 100644 lib/sanitize_ext/sanitize_config.rb (limited to 'lib') diff --git a/app/lib/exceptions.rb b/app/lib/exceptions.rb deleted file mode 100644 index 7c8e77871..000000000 --- a/app/lib/exceptions.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -module Mastodon - class Error < StandardError; end - class NotPermittedError < Error; end - class ValidationError < Error; end - class HostValidationError < ValidationError; end - class LengthValidationError < ValidationError; end - class DimensionsValidationError < ValidationError; end - class StreamValidationError < ValidationError; end - class RaceConditionError < Error; end - class RateLimitExceededError < Error; end - - class UnexpectedResponseError < Error - def initialize(response = nil) - if response.respond_to? :uri - super("#{response.uri} returned code #{response.code}") - else - super - end - end - end -end diff --git a/app/lib/formatter.rb b/app/lib/formatter.rb index 6fb5d5419..2611bcbae 100644 --- a/app/lib/formatter.rb +++ b/app/lib/formatter.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true require 'singleton' -require_relative './sanitize_config' class Formatter include Singleton diff --git a/app/lib/sanitize_config.rb b/app/lib/sanitize_config.rb deleted file mode 100644 index a2e1d9d01..000000000 --- a/app/lib/sanitize_config.rb +++ /dev/null @@ -1,119 +0,0 @@ -# frozen_string_literal: true - -class Sanitize - module Config - HTTP_PROTOCOLS = %w( - http - https - ).freeze - - LINK_PROTOCOLS = %w( - http - https - dat - dweb - ipfs - ipns - ssb - gopher - xmpp - magnet - gemini - ).freeze - - CLASS_WHITELIST_TRANSFORMER = lambda do |env| - node = env[:node] - class_list = node['class']&.split(/[\t\n\f\r ]/) - - return unless class_list - - class_list.keep_if do |e| - next true if /^(h|p|u|dt|e)-/.match?(e) # microformats classes - next true if /^(mention|hashtag)$/.match?(e) # semantic classes - next true if /^(ellipsis|invisible)$/.match?(e) # link formatting classes - end - - node['class'] = class_list.join(' ') - end - - UNSUPPORTED_HREF_TRANSFORMER = lambda do |env| - return unless env[:node_name] == 'a' - - current_node = env[:node] - - scheme = begin - if current_node['href'] =~ Sanitize::REGEX_PROTOCOL - Regexp.last_match(1).downcase - else - :relative - end - end - - current_node.replace(current_node.text) unless LINK_PROTOCOLS.include?(scheme) - end - - UNSUPPORTED_ELEMENTS_TRANSFORMER = lambda do |env| - return unless %w(h1 h2 h3 h4 h5 h6 blockquote pre ul ol li).include?(env[:node_name]) - - current_node = env[:node] - - case env[:node_name] - when 'li' - current_node.traverse do |node| - next unless %w(p ul ol li).include?(node.name) - - node.add_next_sibling('
') if node.next_sibling - node.replace(node.children) unless node.text? - end - else - current_node.name = 'p' - end - end - - MASTODON_STRICT ||= freeze_config( - elements: %w(p br span a), - - attributes: { - 'a' => %w(href rel class), - 'span' => %w(class), - }, - - add_attributes: { - 'a' => { - 'rel' => 'nofollow noopener noreferrer', - 'target' => '_blank', - }, - }, - - protocols: {}, - - transformers: [ - CLASS_WHITELIST_TRANSFORMER, - UNSUPPORTED_ELEMENTS_TRANSFORMER, - UNSUPPORTED_HREF_TRANSFORMER, - ] - ) - - MASTODON_OEMBED ||= freeze_config merge( - RELAXED, - elements: RELAXED[:elements] + %w(audio embed iframe source video), - - attributes: merge( - RELAXED[:attributes], - 'audio' => %w(controls), - 'embed' => %w(height src type width), - 'iframe' => %w(allowfullscreen frameborder height scrolling src width), - 'source' => %w(src type), - 'video' => %w(controls height loop width), - 'div' => [:data] - ), - - protocols: merge( - RELAXED[:protocols], - 'embed' => { 'src' => HTTP_PROTOCOLS }, - 'iframe' => { 'src' => HTTP_PROTOCOLS }, - 'source' => { 'src' => HTTP_PROTOCOLS } - ) - ) - end -end diff --git a/app/validators/url_validator.rb b/app/validators/url_validator.rb index d95a03fbf..f50abbe24 100644 --- a/app/validators/url_validator.rb +++ b/app/validators/url_validator.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class UrlValidator < ActiveModel::EachValidator +class URLValidator < ActiveModel::EachValidator def validate_each(record, attribute, value) record.errors.add(attribute, I18n.t('applications.invalid_url')) unless compliant?(value) end diff --git a/config/application.rb b/config/application.rb index 116eaf29d..0960247b3 100644 --- a/config/application.rb +++ b/config/application.rb @@ -6,8 +6,9 @@ require 'rails/all' # you've limited to :test, :development, or :production. Bundler.require(*Rails.groups) -require_relative '../app/lib/exceptions' +require_relative '../lib/exceptions' require_relative '../lib/enumerable' +require_relative '../lib/sanitize_ext/sanitize_config' require_relative '../lib/redis/namespace_extensions' require_relative '../lib/paperclip/url_generator_extensions' require_relative '../lib/paperclip/attachment_extensions' diff --git a/config/initializers/inflections.rb b/config/initializers/inflections.rb index ebb7541eb..9bc9a54b2 100644 --- a/config/initializers/inflections.rb +++ b/config/initializers/inflections.rb @@ -20,6 +20,10 @@ ActiveSupport::Inflector.inflections(:en) do |inflect| inflect.acronym 'JsonLd' inflect.acronym 'NodeInfo' inflect.acronym 'Ed25519' + inflect.acronym 'TOC' + inflect.acronym 'RSS' + inflect.acronym 'REST' + inflect.acronym 'URL' inflect.singular 'data', 'data' end diff --git a/db/migrate/20160223165723_add_url_to_statuses.rb b/db/migrate/20160223165723_add_url_to_statuses.rb index 80f4b3289..fee7f9c59 100644 --- a/db/migrate/20160223165723_add_url_to_statuses.rb +++ b/db/migrate/20160223165723_add_url_to_statuses.rb @@ -1,4 +1,4 @@ -class AddUrlToStatuses < ActiveRecord::Migration[4.2] +class AddURLToStatuses < ActiveRecord::Migration[4.2] def change add_column :statuses, :url, :string, null: true, default: nil end diff --git a/db/migrate/20160223165855_add_url_to_accounts.rb b/db/migrate/20160223165855_add_url_to_accounts.rb index c81b1c64f..a4db8814a 100644 --- a/db/migrate/20160223165855_add_url_to_accounts.rb +++ b/db/migrate/20160223165855_add_url_to_accounts.rb @@ -1,4 +1,4 @@ -class AddUrlToAccounts < ActiveRecord::Migration[4.2] +class AddURLToAccounts < ActiveRecord::Migration[4.2] def change add_column :accounts, :url, :string, null: true, default: nil end diff --git a/db/migrate/20160322193748_add_avatar_remote_url_to_accounts.rb b/db/migrate/20160322193748_add_avatar_remote_url_to_accounts.rb index f9c213d9b..0792863a3 100644 --- a/db/migrate/20160322193748_add_avatar_remote_url_to_accounts.rb +++ b/db/migrate/20160322193748_add_avatar_remote_url_to_accounts.rb @@ -1,4 +1,4 @@ -class AddAvatarRemoteUrlToAccounts < ActiveRecord::Migration[4.2] +class AddAvatarRemoteURLToAccounts < ActiveRecord::Migration[4.2] def change add_column :accounts, :avatar_remote_url, :string, null: true, default: nil end diff --git a/db/migrate/20170318214217_add_header_remote_url_to_accounts.rb b/db/migrate/20170318214217_add_header_remote_url_to_accounts.rb index 0ba38d3e0..20c965988 100644 --- a/db/migrate/20170318214217_add_header_remote_url_to_accounts.rb +++ b/db/migrate/20170318214217_add_header_remote_url_to_accounts.rb @@ -1,4 +1,4 @@ -class AddHeaderRemoteUrlToAccounts < ActiveRecord::Migration[5.0] +class AddHeaderRemoteURLToAccounts < ActiveRecord::Migration[5.0] def change add_column :accounts, :header_remote_url, :string, null: false, default: '' end diff --git a/db/migrate/20171130000000_add_embed_url_to_preview_cards.rb b/db/migrate/20171130000000_add_embed_url_to_preview_cards.rb index d19c0091b..8fcabef9f 100644 --- a/db/migrate/20171130000000_add_embed_url_to_preview_cards.rb +++ b/db/migrate/20171130000000_add_embed_url_to_preview_cards.rb @@ -1,6 +1,6 @@ require Rails.root.join('lib', 'mastodon', 'migration_helpers') -class AddEmbedUrlToPreviewCards < ActiveRecord::Migration[5.1] +class AddEmbedURLToPreviewCards < ActiveRecord::Migration[5.1] include Mastodon::MigrationHelpers disable_ddl_transaction! diff --git a/db/migrate/20180304013859_add_featured_collection_url_to_accounts.rb b/db/migrate/20180304013859_add_featured_collection_url_to_accounts.rb index e0b8ed5cc..1964b5121 100644 --- a/db/migrate/20180304013859_add_featured_collection_url_to_accounts.rb +++ b/db/migrate/20180304013859_add_featured_collection_url_to_accounts.rb @@ -1,4 +1,4 @@ -class AddFeaturedCollectionUrlToAccounts < ActiveRecord::Migration[5.1] +class AddFeaturedCollectionURLToAccounts < ActiveRecord::Migration[5.1] def change add_column :accounts, :featured_collection_url, :string end diff --git a/db/migrate/20200529214050_add_devices_url_to_accounts.rb b/db/migrate/20200529214050_add_devices_url_to_accounts.rb index 564877e5d..1323f8df7 100644 --- a/db/migrate/20200529214050_add_devices_url_to_accounts.rb +++ b/db/migrate/20200529214050_add_devices_url_to_accounts.rb @@ -1,4 +1,4 @@ -class AddDevicesUrlToAccounts < ActiveRecord::Migration[5.2] +class AddDevicesURLToAccounts < ActiveRecord::Migration[5.2] def change add_column :accounts, :devices_url, :string end diff --git a/lib/exceptions.rb b/lib/exceptions.rb new file mode 100644 index 000000000..7c8e77871 --- /dev/null +++ b/lib/exceptions.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Mastodon + class Error < StandardError; end + class NotPermittedError < Error; end + class ValidationError < Error; end + class HostValidationError < ValidationError; end + class LengthValidationError < ValidationError; end + class DimensionsValidationError < ValidationError; end + class StreamValidationError < ValidationError; end + class RaceConditionError < Error; end + class RateLimitExceededError < Error; end + + class UnexpectedResponseError < Error + def initialize(response = nil) + if response.respond_to? :uri + super("#{response.uri} returned code #{response.code}") + else + super + end + end + end +end diff --git a/lib/sanitize_ext/sanitize_config.rb b/lib/sanitize_ext/sanitize_config.rb new file mode 100644 index 000000000..a2e1d9d01 --- /dev/null +++ b/lib/sanitize_ext/sanitize_config.rb @@ -0,0 +1,119 @@ +# frozen_string_literal: true + +class Sanitize + module Config + HTTP_PROTOCOLS = %w( + http + https + ).freeze + + LINK_PROTOCOLS = %w( + http + https + dat + dweb + ipfs + ipns + ssb + gopher + xmpp + magnet + gemini + ).freeze + + CLASS_WHITELIST_TRANSFORMER = lambda do |env| + node = env[:node] + class_list = node['class']&.split(/[\t\n\f\r ]/) + + return unless class_list + + class_list.keep_if do |e| + next true if /^(h|p|u|dt|e)-/.match?(e) # microformats classes + next true if /^(mention|hashtag)$/.match?(e) # semantic classes + next true if /^(ellipsis|invisible)$/.match?(e) # link formatting classes + end + + node['class'] = class_list.join(' ') + end + + UNSUPPORTED_HREF_TRANSFORMER = lambda do |env| + return unless env[:node_name] == 'a' + + current_node = env[:node] + + scheme = begin + if current_node['href'] =~ Sanitize::REGEX_PROTOCOL + Regexp.last_match(1).downcase + else + :relative + end + end + + current_node.replace(current_node.text) unless LINK_PROTOCOLS.include?(scheme) + end + + UNSUPPORTED_ELEMENTS_TRANSFORMER = lambda do |env| + return unless %w(h1 h2 h3 h4 h5 h6 blockquote pre ul ol li).include?(env[:node_name]) + + current_node = env[:node] + + case env[:node_name] + when 'li' + current_node.traverse do |node| + next unless %w(p ul ol li).include?(node.name) + + node.add_next_sibling('
') if node.next_sibling + node.replace(node.children) unless node.text? + end + else + current_node.name = 'p' + end + end + + MASTODON_STRICT ||= freeze_config( + elements: %w(p br span a), + + attributes: { + 'a' => %w(href rel class), + 'span' => %w(class), + }, + + add_attributes: { + 'a' => { + 'rel' => 'nofollow noopener noreferrer', + 'target' => '_blank', + }, + }, + + protocols: {}, + + transformers: [ + CLASS_WHITELIST_TRANSFORMER, + UNSUPPORTED_ELEMENTS_TRANSFORMER, + UNSUPPORTED_HREF_TRANSFORMER, + ] + ) + + MASTODON_OEMBED ||= freeze_config merge( + RELAXED, + elements: RELAXED[:elements] + %w(audio embed iframe source video), + + attributes: merge( + RELAXED[:attributes], + 'audio' => %w(controls), + 'embed' => %w(height src type width), + 'iframe' => %w(allowfullscreen frameborder height scrolling src width), + 'source' => %w(src type), + 'video' => %w(controls height loop width), + 'div' => [:data] + ), + + protocols: merge( + RELAXED[:protocols], + 'embed' => { 'src' => HTTP_PROTOCOLS }, + 'iframe' => { 'src' => HTTP_PROTOCOLS }, + 'source' => { 'src' => HTTP_PROTOCOLS } + ) + ) + end +end diff --git a/spec/lib/sanitize_config_spec.rb b/spec/lib/sanitize_config_spec.rb index d66302e64..747d81158 100644 --- a/spec/lib/sanitize_config_spec.rb +++ b/spec/lib/sanitize_config_spec.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true require 'rails_helper' -require Rails.root.join('app', 'lib', 'sanitize_config.rb') describe Sanitize::Config do describe '::MASTODON_STRICT' do diff --git a/spec/validators/url_validator_spec.rb b/spec/validators/url_validator_spec.rb index e8d0e6494..a44878a44 100644 --- a/spec/validators/url_validator_spec.rb +++ b/spec/validators/url_validator_spec.rb @@ -2,7 +2,7 @@ require 'rails_helper' -RSpec.describe UrlValidator, type: :validator do +RSpec.describe URLValidator, type: :validator do describe '#validate_each' do before do allow(validator).to receive(:compliant?).with(value) { compliant } -- cgit From 9aaaa96d2ff5e27f065375a2544e86afa31a4e13 Mon Sep 17 00:00:00 2001 From: Claire Date: Fri, 19 Mar 2021 02:43:13 +0100 Subject: Use more robust hook for loading timestamp_id function into database (#15919) --- config/application.rb | 1 + .../20170920024819_status_ids_to_timestamp_ids.rb | 4 +- lib/active_record/database_tasks_extensions.rb | 20 ++++++++ lib/tasks/db.rake | 56 ---------------------- 4 files changed, 23 insertions(+), 58 deletions(-) create mode 100644 lib/active_record/database_tasks_extensions.rb (limited to 'lib') diff --git a/config/application.rb b/config/application.rb index 0960247b3..3267fa71b 100644 --- a/config/application.rb +++ b/config/application.rb @@ -28,6 +28,7 @@ require_relative '../lib/webpacker/manifest_extensions' require_relative '../lib/webpacker/helper_extensions' require_relative '../lib/action_dispatch/cookie_jar_extensions' require_relative '../lib/rails/engine_extensions' +require_relative '../lib/active_record/database_tasks_extensions' Dotenv::Railtie.load diff --git a/db/migrate/20170920024819_status_ids_to_timestamp_ids.rb b/db/migrate/20170920024819_status_ids_to_timestamp_ids.rb index c10aa2c4f..8679f8ece 100644 --- a/db/migrate/20170920024819_status_ids_to_timestamp_ids.rb +++ b/db/migrate/20170920024819_status_ids_to_timestamp_ids.rb @@ -1,7 +1,7 @@ class StatusIdsToTimestampIds < ActiveRecord::Migration[5.1] def up # Prepare the function we will use to generate IDs. - Rake::Task['db:define_timestamp_id'].execute + Mastodon::Snowflake.define_timestamp_id # Set up the statuses.id column to use our timestamp-based IDs. ActiveRecord::Base.connection.execute(<<~SQL) @@ -11,7 +11,7 @@ class StatusIdsToTimestampIds < ActiveRecord::Migration[5.1] SQL # Make sure we have a sequence to use. - Rake::Task['db:ensure_id_sequences_exist'].execute + Mastodon::Snowflake.ensure_id_sequences_exist end def down diff --git a/lib/active_record/database_tasks_extensions.rb b/lib/active_record/database_tasks_extensions.rb new file mode 100644 index 000000000..e274f476d --- /dev/null +++ b/lib/active_record/database_tasks_extensions.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +require_relative '../mastodon/snowflake' + +module ActiveRecord + module Tasks + module DatabaseTasks + original_load_schema = instance_method(:load_schema) + + define_method(:load_schema) do |db_config, *args| + ActiveRecord::Base.establish_connection(db_config) + Mastodon::Snowflake.define_timestamp_id + + original_load_schema.bind(self).call(db_config, *args) + + Mastodon::Snowflake.ensure_id_sequences_exist + end + end + end +end diff --git a/lib/tasks/db.rake b/lib/tasks/db.rake index f6c9c7eec..552a02b3f 100644 --- a/lib/tasks/db.rake +++ b/lib/tasks/db.rake @@ -1,36 +1,5 @@ # frozen_string_literal: true -require_relative '../mastodon/snowflake' - -def each_schema_load_environment - # If we're in development, also run this for the test environment. - # This is a somewhat hacky way to do this, so here's why: - # 1. We have to define this before we load the schema, or we won't - # have a timestamp_id function when we get to it in the schema. - # 2. db:setup calls db:schema:load_if_ruby, which calls - # db:schema:load, which we define above as having a prerequisite - # of this task. - # 3. db:schema:load ends up running - # ActiveRecord::Tasks::DatabaseTasks.load_schema_current, which - # calls a private method `each_current_configuration`, which - # explicitly also does the loading for the `test` environment - # if the current environment is `development`, so we end up - # needing to do the same, and we can't even use the same method - # to do it. - - if Rails.env.development? - test_conf = ActiveRecord::Base.configurations['test'] - - if test_conf['database']&.present? - ActiveRecord::Base.establish_connection(:test) - yield - ActiveRecord::Base.establish_connection(Rails.env.to_sym) - end - end - - yield -end - namespace :db do namespace :migrate do desc 'Setup the db or migrate depending on state of db' @@ -61,29 +30,4 @@ namespace :db do end Rake::Task['db:migrate'].enhance(['db:post_migration_hook']) - - # Before we load the schema, define the timestamp_id function. - # Idiomatically, we might do this in a migration, but then it - # wouldn't end up in schema.rb, so we'd need to figure out a way to - # get it in before doing db:setup as well. This is simpler, and - # ensures it's always in place. - Rake::Task['db:schema:load'].enhance ['db:define_timestamp_id'] - - # After we load the schema, make sure we have sequences for each - # table using timestamp IDs. - Rake::Task['db:schema:load'].enhance do - Rake::Task['db:ensure_id_sequences_exist'].invoke - end - - task :define_timestamp_id do - each_schema_load_environment do - Mastodon::Snowflake.define_timestamp_id - end - end - - task :ensure_id_sequences_exist do - each_schema_load_environment do - Mastodon::Snowflake.ensure_id_sequences_exist - end - end end -- cgit From b3582298341e32528929c6f3292e36a6fa261ba5 Mon Sep 17 00:00:00 2001 From: Claire Date: Fri, 19 Mar 2021 02:45:34 +0100 Subject: Further preparation for Rails 6 (#15916) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Use ActiveRecord::Result#to_ary instead of deprecated to_hash They do the same thing, and to_hash has been removed from Rails 6.1 * Explicitly name polymorphic indexes to workaround a bug in Rails 6.1 cf. https://github.com/rails/rails/issues/41693 * Fix incorrect usage of “foreign_key” in migration script * Use `ActiveModel::Errors#delete` instead of deprecated clear method * Fix link headers tests on Rails 6.1 Rails 6.1 adds values to the Link header by default, thus it is not a LinkHeader object anymore. Fix the test to parse the Link header instead of assuming it is a LinkHeader. --- app/controllers/admin/domain_blocks_controller.rb | 2 +- db/migrate/20161006213403_rails_settings_migration.rb | 8 ++++---- db/migrate/20171119172437_create_admin_action_logs.rb | 2 +- db/migrate/20180528141303_fix_accounts_unique_index.rb | 2 +- db/migrate/20181024224956_migrate_account_conversations.rb | 4 ++-- db/migrate/20181207011115_downcase_custom_emoji_domains.rb | 2 +- db/migrate/20190726175042_add_case_insensitive_index_to_tags.rb | 2 +- lib/mastodon/migration_helpers.rb | 6 +++--- spec/requests/link_headers_spec.rb | 2 +- 9 files changed, 15 insertions(+), 15 deletions(-) (limited to 'lib') diff --git a/app/controllers/admin/domain_blocks_controller.rb b/app/controllers/admin/domain_blocks_controller.rb index ba927b04a..b140c454c 100644 --- a/app/controllers/admin/domain_blocks_controller.rb +++ b/app/controllers/admin/domain_blocks_controller.rb @@ -22,7 +22,7 @@ module Admin if existing_domain_block.present? && !@domain_block.stricter_than?(existing_domain_block) @domain_block.save flash.now[:alert] = I18n.t('admin.domain_blocks.existing_domain_block_html', name: existing_domain_block.domain, unblock_url: admin_domain_block_path(existing_domain_block)).html_safe # rubocop:disable Rails/OutputSafety - @domain_block.errors[:domain].clear + @domain_block.errors.delete(:domain) render :new else if existing_domain_block.present? diff --git a/db/migrate/20161006213403_rails_settings_migration.rb b/db/migrate/20161006213403_rails_settings_migration.rb index 42875d7cb..9d565cb5c 100644 --- a/db/migrate/20161006213403_rails_settings_migration.rb +++ b/db/migrate/20161006213403_rails_settings_migration.rb @@ -7,12 +7,12 @@ end class RailsSettingsMigration < MIGRATION_BASE_CLASS def self.up create_table :settings do |t| - t.string :var, :null => false + t.string :var, null: false t.text :value - t.references :target, :null => false, :polymorphic => true - t.timestamps :null => true + t.references :target, null: false, polymorphic: true, index: { name: 'index_settings_on_target_type_and_target_id' } + t.timestamps null: true end - add_index :settings, [ :target_type, :target_id, :var ], :unique => true + add_index :settings, [ :target_type, :target_id, :var ], unique: true end def self.down diff --git a/db/migrate/20171119172437_create_admin_action_logs.rb b/db/migrate/20171119172437_create_admin_action_logs.rb index 0c2b6c623..b690735d2 100644 --- a/db/migrate/20171119172437_create_admin_action_logs.rb +++ b/db/migrate/20171119172437_create_admin_action_logs.rb @@ -3,7 +3,7 @@ class CreateAdminActionLogs < ActiveRecord::Migration[5.1] create_table :admin_action_logs do |t| t.belongs_to :account, foreign_key: { on_delete: :cascade } t.string :action, null: false, default: '' - t.references :target, polymorphic: true + t.references :target, polymorphic: true, index: { name: 'index_admin_action_logs_on_target_type_and_target_id' } t.text :recorded_changes, null: false, default: '' t.timestamps diff --git a/db/migrate/20180528141303_fix_accounts_unique_index.rb b/db/migrate/20180528141303_fix_accounts_unique_index.rb index 5d7b3c463..02813f363 100644 --- a/db/migrate/20180528141303_fix_accounts_unique_index.rb +++ b/db/migrate/20180528141303_fix_accounts_unique_index.rb @@ -37,7 +37,7 @@ class FixAccountsUniqueIndex < ActiveRecord::Migration[5.2] end end - duplicates = Account.connection.select_all('SELECT string_agg(id::text, \',\') AS ids FROM accounts GROUP BY lower(username), lower(domain) HAVING count(*) > 1').to_hash + duplicates = Account.connection.select_all('SELECT string_agg(id::text, \',\') AS ids FROM accounts GROUP BY lower(username), lower(domain) HAVING count(*) > 1').to_ary duplicates.each do |row| deduplicate_account!(row['ids'].split(',')) diff --git a/db/migrate/20181024224956_migrate_account_conversations.rb b/db/migrate/20181024224956_migrate_account_conversations.rb index 12e0a70fa..9e6497d81 100644 --- a/db/migrate/20181024224956_migrate_account_conversations.rb +++ b/db/migrate/20181024224956_migrate_account_conversations.rb @@ -17,8 +17,8 @@ class MigrateAccountConversations < ActiveRecord::Migration[5.2] belongs_to :account, optional: true belongs_to :activity, polymorphic: true, optional: true - belongs_to :status, foreign_type: 'Status', foreign_key: 'activity_id', optional: true - belongs_to :mention, foreign_type: 'Mention', foreign_key: 'activity_id', optional: true + belongs_to :status, foreign_key: 'activity_id', optional: true + belongs_to :mention, foreign_key: 'activity_id', optional: true def target_status mention&.status diff --git a/db/migrate/20181207011115_downcase_custom_emoji_domains.rb b/db/migrate/20181207011115_downcase_custom_emoji_domains.rb index 65f1fc8d9..e27e0249d 100644 --- a/db/migrate/20181207011115_downcase_custom_emoji_domains.rb +++ b/db/migrate/20181207011115_downcase_custom_emoji_domains.rb @@ -2,7 +2,7 @@ class DowncaseCustomEmojiDomains < ActiveRecord::Migration[5.2] disable_ddl_transaction! def up - duplicates = CustomEmoji.connection.select_all('SELECT string_agg(id::text, \',\') AS ids FROM custom_emojis GROUP BY shortcode, lower(domain) HAVING count(*) > 1').to_hash + duplicates = CustomEmoji.connection.select_all('SELECT string_agg(id::text, \',\') AS ids FROM custom_emojis GROUP BY shortcode, lower(domain) HAVING count(*) > 1').to_ary duplicates.each do |row| CustomEmoji.where(id: row['ids'].split(',')[0...-1]).destroy_all diff --git a/db/migrate/20190726175042_add_case_insensitive_index_to_tags.rb b/db/migrate/20190726175042_add_case_insensitive_index_to_tags.rb index 057fc86ba..eb03d7ca7 100644 --- a/db/migrate/20190726175042_add_case_insensitive_index_to_tags.rb +++ b/db/migrate/20190726175042_add_case_insensitive_index_to_tags.rb @@ -2,7 +2,7 @@ class AddCaseInsensitiveIndexToTags < ActiveRecord::Migration[5.2] disable_ddl_transaction! def up - Tag.connection.select_all('SELECT string_agg(id::text, \',\') AS ids FROM tags GROUP BY lower(name) HAVING count(*) > 1').to_hash.each do |row| + Tag.connection.select_all('SELECT string_agg(id::text, \',\') AS ids FROM tags GROUP BY lower(name) HAVING count(*) > 1').to_ary.each do |row| canonical_tag_id = row['ids'].split(',').first redundant_tag_ids = row['ids'].split(',')[1..-1] diff --git a/lib/mastodon/migration_helpers.rb b/lib/mastodon/migration_helpers.rb index bf2314ecb..fcaa9259e 100644 --- a/lib/mastodon/migration_helpers.rb +++ b/lib/mastodon/migration_helpers.rb @@ -319,7 +319,7 @@ module Mastodon count_arel = table.project(Arel.star.count.as('count')) count_arel = yield table, count_arel if block_given? - total = exec_query(count_arel.to_sql).to_hash.first['count'].to_i + total = exec_query(count_arel.to_sql).to_ary.first['count'].to_i return if total == 0 end @@ -335,7 +335,7 @@ module Mastodon start_arel = table.project(table[:id]).order(table[:id].asc).take(1) start_arel = yield table, start_arel if block_given? - first_row = exec_query(start_arel.to_sql).to_hash.first + first_row = exec_query(start_arel.to_sql).to_ary.first # In case there are no rows but we didn't catch it in the estimated size: return unless first_row start_id = first_row['id'].to_i @@ -356,7 +356,7 @@ module Mastodon .skip(batch_size) stop_arel = yield table, stop_arel if block_given? - stop_row = exec_query(stop_arel.to_sql).to_hash.first + stop_row = exec_query(stop_arel.to_sql).to_ary.first update_arel = Arel::UpdateManager.new .table(table) diff --git a/spec/requests/link_headers_spec.rb b/spec/requests/link_headers_spec.rb index 712ee262b..c32e0f79a 100644 --- a/spec/requests/link_headers_spec.rb +++ b/spec/requests/link_headers_spec.rb @@ -25,7 +25,7 @@ describe 'Link headers' do end def link_header_with_type(type) - response.headers['Link'].links.find do |link| + LinkHeader.parse(response.headers['Link'].to_s).links.find do |link| link.attr_pairs.any? { |pair| pair == ['type', type] } end end -- cgit From 82caed594c3ad2cafe0c83b814879f30942fe57b Mon Sep 17 00:00:00 2001 From: Claire Date: Fri, 19 Mar 2021 11:07:56 +0100 Subject: Change deduplication order of tootctl maintenance fix-duplicates (#15923) Hopefully fixes #15922 Also update support up to latest database schema version --- lib/mastodon/maintenance_cli.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/mastodon/maintenance_cli.rb b/lib/mastodon/maintenance_cli.rb index 029d42a05..9f1eaf263 100644 --- a/lib/mastodon/maintenance_cli.rb +++ b/lib/mastodon/maintenance_cli.rb @@ -14,7 +14,7 @@ module Mastodon end MIN_SUPPORTED_VERSION = 2019_10_01_213028 - MAX_SUPPORTED_VERSION = 2020_12_18_054746 + MAX_SUPPORTED_VERSION = 2021_03_08_133107 # Stubs to enjoy ActiveRecord queries while not depending on a particular # version of the code/database @@ -142,7 +142,6 @@ module Mastodon @prompt.warn 'Please make sure to stop Mastodon and have a backup.' exit(1) unless @prompt.yes?('Continue?') - deduplicate_accounts! deduplicate_users! deduplicate_account_domain_blocks! deduplicate_account_identity_proofs! @@ -157,6 +156,7 @@ module Mastodon deduplicate_media_attachments! deduplicate_preview_cards! deduplicate_statuses! + deduplicate_accounts! deduplicate_tags! deduplicate_webauthn_credentials! -- cgit From c31c95ffe4fbf80981a0ee03484d72ee6d75d2ee Mon Sep 17 00:00:00 2001 From: Claire Date: Fri, 19 Mar 2021 13:14:40 +0100 Subject: Remove MySQL-specific code from Mastodon::MigrationHelpers (#15924) Mastodon::MigrationHelpers has been forked from Gitlab a long time ago, but Mastodon has never supported using a MySQL database. Removing MySQL support from Mastodon::MigrationHelpers makes it a little easier to maintain. In particular, it removes code that would need updating with Rails 6. --- lib/mastodon/migration_helpers.rb | 84 +++++++-------------------------------- 1 file changed, 14 insertions(+), 70 deletions(-) (limited to 'lib') diff --git a/lib/mastodon/migration_helpers.rb b/lib/mastodon/migration_helpers.rb index fcaa9259e..147642a1c 100644 --- a/lib/mastodon/migration_helpers.rb +++ b/lib/mastodon/migration_helpers.rb @@ -41,42 +41,18 @@ module Mastodon module MigrationHelpers - # Stub for Database.postgresql? from GitLab - def self.postgresql? - ActiveRecord::Base.configurations[Rails.env]['adapter'].casecmp('postgresql').zero? - end - - # Stub for Database.mysql? from GitLab - def self.mysql? - ActiveRecord::Base.configurations[Rails.env]['adapter'].casecmp('mysql2').zero? - end - # Model that can be used for querying permissions of a SQL user. class Grant < ActiveRecord::Base - self.table_name = - if Mastodon::MigrationHelpers.postgresql? - 'information_schema.role_table_grants' - else - 'mysql.user' - end + self.table_name = 'information_schema.role_table_grants' def self.scope_to_current_user - if Mastodon::MigrationHelpers.postgresql? - where('grantee = user') - else - where("CONCAT(User, '@', Host) = current_user()") - end + where('grantee = user') end # Returns true if the current user can create and execute triggers on the # given table. def self.create_and_execute_trigger?(table) - priv = - if Mastodon::MigrationHelpers.postgresql? - where(privilege_type: 'TRIGGER', table_name: table) - else - where(Trigger_priv: 'Y') - end + priv = where(privilege_type: 'TRIGGER', table_name: table) priv.scope_to_current_user.any? end @@ -141,10 +117,8 @@ module Mastodon 'in the body of your migration class' end - if MigrationHelpers.postgresql? - options = options.merge({ algorithm: :concurrently }) - disable_statement_timeout - end + options = options.merge({ algorithm: :concurrently }) + disable_statement_timeout add_index(table_name, column_name, options) end @@ -199,8 +173,6 @@ module Mastodon # Only available on Postgresql >= 9.2 def supports_drop_index_concurrently? - return false unless MigrationHelpers.postgresql? - version = select_one("SELECT current_setting('server_version_num') AS v")['v'].to_i version >= 90200 @@ -226,13 +198,7 @@ module Mastodon # While MySQL does allow disabling of foreign keys it has no equivalent # of PostgreSQL's "VALIDATE CONSTRAINT". As a result we'll just fall # back to the normal foreign key procedure. - if MigrationHelpers.mysql? - return add_foreign_key(source, target, - column: column, - on_delete: on_delete) - else - on_delete = 'SET NULL' if on_delete == :nullify - end + on_delete = 'SET NULL' if on_delete == :nullify disable_statement_timeout @@ -270,7 +236,7 @@ module Mastodon # the database. Disable the session's statement timeout to ensure # migrations don't get killed prematurely. (PostgreSQL only) def disable_statement_timeout - execute('SET statement_timeout TO 0') if MigrationHelpers.postgresql? + execute('SET statement_timeout TO 0') end # Updates the value of a column in batches. @@ -487,11 +453,7 @@ module Mastodon # If we were in the middle of update_column_in_batches, we should remove # the old column and start over, as we have no idea where we were. if column_for(table, new) - if MigrationHelpers.postgresql? - remove_rename_triggers_for_postgresql(table, trigger_name) - else - remove_rename_triggers_for_mysql(trigger_name) - end + remove_rename_triggers_for_postgresql(table, trigger_name) remove_column(table, new) end @@ -521,13 +483,8 @@ module Mastodon quoted_old = quote_column_name(old) quoted_new = quote_column_name(new) - if MigrationHelpers.postgresql? - install_rename_triggers_for_postgresql(trigger_name, quoted_table, - quoted_old, quoted_new) - else - install_rename_triggers_for_mysql(trigger_name, quoted_table, - quoted_old, quoted_new) - end + install_rename_triggers_for_postgresql(trigger_name, quoted_table, + quoted_old, quoted_new) update_column_in_batches(table, new, Arel::Table.new(table)[old]) @@ -685,11 +642,7 @@ module Mastodon check_trigger_permissions!(table) - if MigrationHelpers.postgresql? - remove_rename_triggers_for_postgresql(table, trigger_name) - else - remove_rename_triggers_for_mysql(trigger_name) - end + remove_rename_triggers_for_postgresql(table, trigger_name) remove_column(table, old) end @@ -844,18 +797,9 @@ module Mastodon quoted_pattern = Arel::Nodes::Quoted.new(pattern.to_s) quoted_replacement = Arel::Nodes::Quoted.new(replacement.to_s) - if MigrationHelpers.mysql? - locate = Arel::Nodes::NamedFunction - .new('locate', [quoted_pattern, column]) - insert_in_place = Arel::Nodes::NamedFunction - .new('insert', [column, locate, pattern.size, quoted_replacement]) - - Arel::Nodes::SqlLiteral.new(insert_in_place.to_sql) - else - replace = Arel::Nodes::NamedFunction - .new("regexp_replace", [column, quoted_pattern, quoted_replacement]) - Arel::Nodes::SqlLiteral.new(replace.to_sql) - end + replace = Arel::Nodes::NamedFunction + .new("regexp_replace", [column, quoted_pattern, quoted_replacement]) + Arel::Nodes::SqlLiteral.new(replace.to_sql) end def remove_foreign_key_without_error(*args) -- cgit From 741d0952b174740e70a09fe6db6862624dfe1e44 Mon Sep 17 00:00:00 2001 From: Claire Date: Fri, 19 Mar 2021 13:14:57 +0100 Subject: Improve account counters handling (#15913) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Improve account counters handling * Use ActiveRecord::Base::sanitize_sql to pass values instead of interpolating them Keep using string interpolation for `key` as it is safe and using “ActiveRecord::Base::sanitize_sql_hash_for_assignment” would require stitching bits of SQL in a way that is not more easily checked for safety. * Add migration hook to catch PostgreSQL versions earlier than 9.5 --- app/models/account_stat.rb | 42 ------------------- app/models/concerns/account_counters.rb | 60 ++++++++++++++++++++++++++- lib/tasks/db.rake | 8 +++- spec/models/account_stat_spec.rb | 57 ------------------------- spec/models/concerns/account_counters_spec.rb | 60 +++++++++++++++++++++++++++ 5 files changed, 125 insertions(+), 102 deletions(-) delete mode 100644 spec/models/account_stat_spec.rb create mode 100644 spec/models/concerns/account_counters_spec.rb (limited to 'lib') diff --git a/app/models/account_stat.rb b/app/models/account_stat.rb index e70b54d79..a826a9af3 100644 --- a/app/models/account_stat.rb +++ b/app/models/account_stat.rb @@ -18,46 +18,4 @@ class AccountStat < ApplicationRecord belongs_to :account, inverse_of: :account_stat update_index('accounts#account', :account) - - def increment_count!(key) - update(attributes_for_increment(key)) - rescue ActiveRecord::StaleObjectError, ActiveRecord::RecordNotUnique - begin - reload_with_id - rescue ActiveRecord::RecordNotFound - return - end - - retry - end - - def decrement_count!(key) - update(attributes_for_decrement(key)) - rescue ActiveRecord::StaleObjectError, ActiveRecord::RecordNotUnique - begin - reload_with_id - rescue ActiveRecord::RecordNotFound - return - end - - retry - end - - private - - def attributes_for_increment(key) - attrs = { key => public_send(key) + 1 } - attrs[:last_status_at] = Time.now.utc if key == :statuses_count - attrs - end - - def attributes_for_decrement(key) - attrs = { key => [public_send(key) - 1, 0].max } - attrs - end - - def reload_with_id - self.id = self.class.find_by!(account: account).id if new_record? - reload - end end diff --git a/app/models/concerns/account_counters.rb b/app/models/concerns/account_counters.rb index 6e25e1905..fd3f161ad 100644 --- a/app/models/concerns/account_counters.rb +++ b/app/models/concerns/account_counters.rb @@ -3,6 +3,8 @@ module AccountCounters extend ActiveSupport::Concern + ALLOWED_COUNTER_KEYS = %i(statuses_count following_count followers_count).freeze + included do has_one :account_stat, inverse_of: :account after_save :save_account_stat @@ -14,11 +16,65 @@ module AccountCounters :following_count=, :followers_count, :followers_count=, - :increment_count!, - :decrement_count!, :last_status_at, to: :account_stat + # @param [Symbol] key + def increment_count!(key) + update_count!(key, 1) + end + + # @param [Symbol] key + def decrement_count!(key) + update_count!(key, -1) + end + + # @param [Symbol] key + # @param [Integer] value + def update_count!(key, value) + raise ArgumentError, "Invalid key #{key}" unless ALLOWED_COUNTER_KEYS.include?(key) + raise ArgumentError, 'Do not call update_count! on dirty objects' if association(:account_stat).loaded? && account_stat&.changed? && account_stat.changed_attribute_names_to_save == %w(id) + + value = value.to_i + default_value = value.positive? ? value : 0 + + # We do an upsert using manually written SQL, as Rails' upsert method does + # not seem to support writing expressions in the UPDATE clause, but only + # re-insert the provided values instead. + # Even ARel seem to be missing proper handling of upserts. + sql = if value.positive? && key == :statuses_count + <<-SQL.squish + INSERT INTO account_stats(account_id, #{key}, created_at, updated_at, last_status_at) + VALUES (:account_id, :default_value, now(), now(), now()) + ON CONFLICT (account_id) DO UPDATE + SET #{key} = account_stats.#{key} + :value, + last_status_at = now(), + lock_version = account_stats.lock_version + 1, + updated_at = now() + RETURNING id; + SQL + else + <<-SQL.squish + INSERT INTO account_stats(account_id, #{key}, created_at, updated_at) + VALUES (:account_id, :default_value, now(), now()) + ON CONFLICT (account_id) DO UPDATE + SET #{key} = account_stats.#{key} + :value, + lock_version = account_stats.lock_version + 1, + updated_at = now() + RETURNING id; + SQL + end + + sql = AccountStat.sanitize_sql([sql, account_id: id, default_value: default_value, value: value]) + account_stat_id = AccountStat.connection.exec_query(sql)[0]['id'] + + # Reload account_stat if it was loaded, taking into account newly-created unsaved records + if association(:account_stat).loaded? + account_stat.id = account_stat_id if account_stat.new_record? + account_stat.reload + end + end + def account_stat super || build_account_stat end diff --git a/lib/tasks/db.rake b/lib/tasks/db.rake index 552a02b3f..7e6c1c8fc 100644 --- a/lib/tasks/db.rake +++ b/lib/tasks/db.rake @@ -19,7 +19,7 @@ namespace :db do task :post_migration_hook do at_exit do - unless %w(C POSIX).include?(ActiveRecord::Base.connection.execute('SELECT datcollate FROM pg_database WHERE datname = current_database();').first['datcollate']) + unless %w(C POSIX).include?(ActiveRecord::Base.connection.select_one('SELECT datcollate FROM pg_database WHERE datname = current_database();')['datcollate']) warn <<~WARNING Your database collation is susceptible to index corruption. (This warning does not indicate that index corruption has occured and can be ignored) @@ -29,5 +29,11 @@ namespace :db do end end + task :pre_migration_check do + version = ActiveRecord::Base.connection.select_one("SELECT current_setting('server_version_num') AS v")['v'].to_i + abort 'ERROR: This version of Mastodon requires PostgreSQL 9.5 or newer. Please update PostgreSQL before updating Mastodon.' if version < 90_500 + end + + Rake::Task['db:migrate'].enhance(['db:pre_migration_check']) Rake::Task['db:migrate'].enhance(['db:post_migration_hook']) end diff --git a/spec/models/account_stat_spec.rb b/spec/models/account_stat_spec.rb deleted file mode 100644 index 8adc0d1d6..000000000 --- a/spec/models/account_stat_spec.rb +++ /dev/null @@ -1,57 +0,0 @@ -require 'rails_helper' - -RSpec.describe AccountStat, type: :model do - describe '#increment_count!' do - it 'increments the count' do - account_stat = AccountStat.create(account: Fabricate(:account)) - expect(account_stat.followers_count).to eq 0 - account_stat.increment_count!(:followers_count) - expect(account_stat.followers_count).to eq 1 - end - - it 'increments the count in multi-threaded an environment' do - account_stat = AccountStat.create(account: Fabricate(:account), statuses_count: 0) - increment_by = 15 - wait_for_start = true - - threads = Array.new(increment_by) do - Thread.new do - true while wait_for_start - AccountStat.find(account_stat.id).increment_count!(:statuses_count) - end - end - - wait_for_start = false - threads.each(&:join) - - expect(account_stat.reload.statuses_count).to eq increment_by - end - end - - describe '#decrement_count!' do - it 'decrements the count' do - account_stat = AccountStat.create(account: Fabricate(:account), followers_count: 15) - expect(account_stat.followers_count).to eq 15 - account_stat.decrement_count!(:followers_count) - expect(account_stat.followers_count).to eq 14 - end - - it 'decrements the count in multi-threaded an environment' do - account_stat = AccountStat.create(account: Fabricate(:account), statuses_count: 15) - decrement_by = 10 - wait_for_start = true - - threads = Array.new(decrement_by) do - Thread.new do - true while wait_for_start - AccountStat.find(account_stat.id).decrement_count!(:statuses_count) - end - end - - wait_for_start = false - threads.each(&:join) - - expect(account_stat.reload.statuses_count).to eq 5 - end - end -end diff --git a/spec/models/concerns/account_counters_spec.rb b/spec/models/concerns/account_counters_spec.rb new file mode 100644 index 000000000..4350496e7 --- /dev/null +++ b/spec/models/concerns/account_counters_spec.rb @@ -0,0 +1,60 @@ +require 'rails_helper' + +describe AccountCounters do + let!(:account) { Fabricate(:account) } + + describe '#increment_count!' do + it 'increments the count' do + expect(account.followers_count).to eq 0 + account.increment_count!(:followers_count) + expect(account.followers_count).to eq 1 + end + + it 'increments the count in multi-threaded an environment' do + increment_by = 15 + wait_for_start = true + + threads = Array.new(increment_by) do + Thread.new do + true while wait_for_start + account.increment_count!(:statuses_count) + end + end + + wait_for_start = false + threads.each(&:join) + + expect(account.statuses_count).to eq increment_by + end + end + + describe '#decrement_count!' do + it 'decrements the count' do + account.followers_count = 15 + account.save! + expect(account.followers_count).to eq 15 + account.decrement_count!(:followers_count) + expect(account.followers_count).to eq 14 + end + + it 'decrements the count in multi-threaded an environment' do + decrement_by = 10 + wait_for_start = true + + account.statuses_count = 15 + account.save! + + threads = Array.new(decrement_by) do + Thread.new do + true while wait_for_start + account.decrement_count!(:statuses_count) + end + end + + wait_for_start = false + threads.each(&:join) + + expect(account.statuses_count).to eq 5 + end + end +end -- cgit From 82556834cf816030eac0d3f962028cb8dafac2aa Mon Sep 17 00:00:00 2001 From: Claire Date: Wed, 24 Mar 2021 10:37:24 +0100 Subject: Change mastodon:setup to not call assets:precompile in docker (#13942) It appears assets are built during image build, and they shouldn't need to be rebuilt, since we now have reproducible builds. --- lib/tasks/mastodon.rake | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/lib/tasks/mastodon.rake b/lib/tasks/mastodon.rake index 2ad1e778b..72bacb5eb 100644 --- a/lib/tasks/mastodon.rake +++ b/lib/tasks/mastodon.rake @@ -371,18 +371,20 @@ namespace :mastodon do end end - prompt.say "\n" - prompt.say 'The final step is compiling CSS/JS assets.' - prompt.say 'This may take a while and consume a lot of RAM.' + unless using_docker + prompt.say "\n" + prompt.say 'The final step is compiling CSS/JS assets.' + prompt.say 'This may take a while and consume a lot of RAM.' - if prompt.yes?('Compile the assets now?') - prompt.say 'Running `RAILS_ENV=production rails assets:precompile` ...' - prompt.say "\n\n" + if prompt.yes?('Compile the assets now?') + prompt.say 'Running `RAILS_ENV=production rails assets:precompile` ...' + prompt.say "\n\n" - if !system(env.transform_values(&:to_s).merge({ 'RAILS_ENV' => 'production' }), 'rails assets:precompile') - prompt.error 'That failed! Maybe you need swap space?' - else - prompt.say 'Done!' + if !system(env.transform_values(&:to_s).merge({ 'RAILS_ENV' => 'production' }), 'rails assets:precompile') + prompt.error 'That failed! Maybe you need swap space?' + else + prompt.say 'Done!' + end end end -- cgit From cbd0ee1d07c0d48e4ed14bd446cd23d334e76da8 Mon Sep 17 00:00:00 2001 From: Claire Date: Wed, 24 Mar 2021 10:44:31 +0100 Subject: Update Mastodon to Rails 6.1 (#15910) * Update devise-two-factor to unreleased fork for Rails 6 support Update tests to match new `rotp` version. * Update nsa gem to unreleased fork for Rails 6 support * Update rails to 6.1.3 and rails-i18n to 6.0 * Update to unreleased fork of pluck_each for Ruby 6 support * Run "rails app:update" * Add missing ActiveStorage config file * Use config.ssl_options instead of removed ApplicationController#force_ssl Disabled force_ssl-related tests as they do not seem to be easily testable anymore. * Fix nonce directives by removing Rails 5 specific monkey-patching * Fix fixture_file_upload deprecation warning * Fix yield-based test failing with Rails 6 * Use Rails 6's index_with when possible * Use ActiveRecord::Cache::Store#delete_multi from Rails 6 This will yield better performances when deleting an account * Disable Rails 6.1's automatic preload link headers Since Rails 6.1, ActionView adds preload links for javascript files in the Links header per default. In our case, that will bloat headers too much and potentially cause issues with reverse proxies. Furhermore, we don't need those links, as we already output them as HTML link tags. * Switch to Rails 6.0 default config * Switch to Rails 6.1 default config * Do not include autoload paths in the load path --- Gemfile | 10 +- Gemfile.lock | 181 +++++++++++++-------- app/controllers/application_controller.rb | 6 - app/lib/delivery_failure_tracker.rb | 2 +- app/lib/feed_manager.rb | 12 +- app/lib/settings/scoped_settings.rb | 2 +- app/models/concerns/account_interactions.rb | 2 +- app/models/report.rb | 2 +- app/services/delete_account_service.rb | 3 +- app/services/import_service.rb | 4 +- bin/setup | 16 +- bin/yarn | 12 +- config/application.rb | 3 +- config/environments/production.rb | 7 + .../application_controller_renderer.rb | 10 +- config/initializers/backtrace_silencers.rb | 7 +- config/initializers/content_security_policy.rb | 12 +- config/initializers/permissions_policy.rb | 11 ++ config/initializers/preload_link_headers.rb | 8 + config/storage.yml | 0 lib/tasks/emojis.rake | 2 +- .../api/v1/accounts/credentials_controller_spec.rb | 4 +- spec/controllers/api/v1/media_controller_spec.rb | 10 +- spec/controllers/application_controller_spec.rb | 14 -- .../settings/imports_controller_spec.rb | 4 +- .../settings/profiles_controller_spec.rb | 4 +- .../confirmations_controller_spec.rb | 2 +- spec/models/setting_spec.rb | 11 +- spec/models/user_spec.rb | 2 +- 29 files changed, 204 insertions(+), 159 deletions(-) create mode 100644 config/initializers/permissions_policy.rb create mode 100644 config/initializers/preload_link_headers.rb create mode 100644 config/storage.yml (limited to 'lib') diff --git a/Gemfile b/Gemfile index 98af92def..0b2fdf156 100644 --- a/Gemfile +++ b/Gemfile @@ -6,7 +6,7 @@ ruby '>= 2.5.0', '< 3.0.0' gem 'pkg-config', '~> 1.4' gem 'puma', '~> 5.2' -gem 'rails', '~> 5.2.4.5' +gem 'rails', '~> 6.1.3' gem 'sprockets', '~> 3.7.2' gem 'thor', '~> 1.1' gem 'rack', '~> 2.2.3' @@ -34,7 +34,7 @@ gem 'iso-639' gem 'chewy', '~> 5.2' gem 'cld3', '~> 3.4.1' gem 'devise', '~> 4.7' -gem 'devise-two-factor', '~> 3.1' +gem 'devise-two-factor', git: 'https://github.com/ClearlyClaire/devise-two-factor', ref: '594bb8a32e6f94df7e5ba7c9399eaf9ff25bac0d' group :pam_authentication, optional: true do gem 'devise_pam_authenticatable2', '~> 9.2' @@ -65,7 +65,7 @@ gem 'link_header', '~> 0.0' gem 'mime-types', '~> 3.3.1', require: 'mime/types/columnar' gem 'nilsimsa', git: 'https://github.com/witgo/nilsimsa', ref: 'fd184883048b922b176939f851338d0a4971a532' gem 'nokogiri', '~> 1.11' -gem 'nsa', '~> 0.2' +gem 'nsa', git: 'https://github.com/Gargron/nsa', ref: 'd1079e0cdafdfed7f9f35478d13b9bdaa65965c0' gem 'oj', '~> 3.11' gem 'ox', '~> 2.14' gem 'parslet' @@ -75,7 +75,7 @@ gem 'pundit', '~> 2.1' gem 'premailer-rails' gem 'rack-attack', '~> 6.5' gem 'rack-cors', '~> 1.1', require: 'rack/cors' -gem 'rails-i18n', '~> 5.1' +gem 'rails-i18n', '~> 6.0' gem 'rails-settings-cached', '~> 0.6' gem 'redis', '~> 4.2', require: ['redis', 'redis/connection/hiredis'] gem 'mario-redis-lock', '~> 1.2', require: 'redis_lock' @@ -159,4 +159,4 @@ gem 'concurrent-ruby', require: false gem 'connection_pool', require: false gem 'xorcist', '~> 1.1' -gem 'pluck_each', '~> 0.1.3' +gem 'pluck_each', git: 'https://github.com/nsommer/pluck_each', ref: '73be0947c52fc54bf6d7085378db008358aac5eb' diff --git a/Gemfile.lock b/Gemfile.lock index 1a67f893d..1f7183b9d 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,3 +1,26 @@ +GIT + remote: https://github.com/ClearlyClaire/devise-two-factor + revision: 594bb8a32e6f94df7e5ba7c9399eaf9ff25bac0d + ref: 594bb8a32e6f94df7e5ba7c9399eaf9ff25bac0d + specs: + devise-two-factor (3.1.0) + activesupport (< 7.0) + attr_encrypted (>= 1.3, < 4, != 2) + devise + railties (< 7.0) + rotp (~> 6) + +GIT + remote: https://github.com/Gargron/nsa + revision: d1079e0cdafdfed7f9f35478d13b9bdaa65965c0 + ref: d1079e0cdafdfed7f9f35478d13b9bdaa65965c0 + specs: + nsa (0.2.8) + activesupport (>= 4.2, < 7) + concurrent-ruby (~> 1.0, >= 1.0.2) + sidekiq (>= 3.5) + statsd-ruby (~> 1.4, >= 1.4.0) + GIT remote: https://github.com/ianheggie/health_check revision: 0b799ead604f900ed50685e9b2d469cd2befba5b @@ -6,6 +29,15 @@ GIT health_check (4.0.0.pre) rails (>= 4.0) +GIT + remote: https://github.com/nsommer/pluck_each + revision: 73be0947c52fc54bf6d7085378db008358aac5eb + ref: 73be0947c52fc54bf6d7085378db008358aac5eb + specs: + pluck_each (0.1.3) + activerecord (>= 6.1.0) + activesupport (>= 6.1.0) + GIT remote: https://github.com/witgo/nilsimsa revision: fd184883048b922b176939f851338d0a4971a532 @@ -16,53 +48,71 @@ GIT GEM remote: https://rubygems.org/ specs: - actioncable (5.2.4.5) - actionpack (= 5.2.4.5) + actioncable (6.1.3) + actionpack (= 6.1.3) + activesupport (= 6.1.3) nio4r (~> 2.0) websocket-driver (>= 0.6.1) - actionmailer (5.2.4.5) - actionpack (= 5.2.4.5) - actionview (= 5.2.4.5) - activejob (= 5.2.4.5) + actionmailbox (6.1.3) + actionpack (= 6.1.3) + activejob (= 6.1.3) + activerecord (= 6.1.3) + activestorage (= 6.1.3) + activesupport (= 6.1.3) + mail (>= 2.7.1) + actionmailer (6.1.3) + actionpack (= 6.1.3) + actionview (= 6.1.3) + activejob (= 6.1.3) + activesupport (= 6.1.3) mail (~> 2.5, >= 2.5.4) rails-dom-testing (~> 2.0) - actionpack (5.2.4.5) - actionview (= 5.2.4.5) - activesupport (= 5.2.4.5) - rack (~> 2.0, >= 2.0.8) + actionpack (6.1.3) + actionview (= 6.1.3) + activesupport (= 6.1.3) + rack (~> 2.0, >= 2.0.9) rack-test (>= 0.6.3) rails-dom-testing (~> 2.0) - rails-html-sanitizer (~> 1.0, >= 1.0.2) - actionview (5.2.4.5) - activesupport (= 5.2.4.5) + rails-html-sanitizer (~> 1.0, >= 1.2.0) + actiontext (6.1.3) + actionpack (= 6.1.3) + activerecord (= 6.1.3) + activestorage (= 6.1.3) + activesupport (= 6.1.3) + nokogiri (>= 1.8.5) + actionview (6.1.3) + activesupport (= 6.1.3) builder (~> 3.1) erubi (~> 1.4) rails-dom-testing (~> 2.0) - rails-html-sanitizer (~> 1.0, >= 1.0.3) + rails-html-sanitizer (~> 1.1, >= 1.2.0) active_model_serializers (0.10.12) actionpack (>= 4.1, < 6.2) activemodel (>= 4.1, < 6.2) case_transform (>= 0.2) jsonapi-renderer (>= 0.1.1.beta1, < 0.3) active_record_query_trace (1.8) - activejob (5.2.4.5) - activesupport (= 5.2.4.5) + activejob (6.1.3) + activesupport (= 6.1.3) globalid (>= 0.3.6) - activemodel (5.2.4.5) - activesupport (= 5.2.4.5) - activerecord (5.2.4.5) - activemodel (= 5.2.4.5) - activesupport (= 5.2.4.5) - arel (>= 9.0) - activestorage (5.2.4.5) - actionpack (= 5.2.4.5) - activerecord (= 5.2.4.5) + activemodel (6.1.3) + activesupport (= 6.1.3) + activerecord (6.1.3) + activemodel (= 6.1.3) + activesupport (= 6.1.3) + activestorage (6.1.3) + actionpack (= 6.1.3) + activejob (= 6.1.3) + activerecord (= 6.1.3) + activesupport (= 6.1.3) marcel (~> 0.3.1) - activesupport (5.2.4.5) + mimemagic (~> 0.3.2) + activesupport (6.1.3) concurrent-ruby (~> 1.0, >= 1.0.2) - i18n (>= 0.7, < 2) - minitest (~> 5.1) - tzinfo (~> 1.1) + i18n (>= 1.6, < 2) + minitest (>= 5.1) + tzinfo (~> 2.0) + zeitwerk (~> 2.3) addressable (2.7.0) public_suffix (>= 2.0.2, < 5.0) airbrussh (1.4.0) @@ -71,7 +121,6 @@ GEM annotate (3.1.1) activerecord (>= 3.2, < 7.0) rake (>= 10.4, < 14.0) - arel (9.0.0) ast (2.4.2) attr_encrypted (3.1.0) encryptor (~> 3.0.0) @@ -175,12 +224,6 @@ GEM railties (>= 4.1.0) responders warden (~> 1.2.3) - devise-two-factor (3.1.0) - activesupport (< 6.1) - attr_encrypted (>= 1.3, < 4, != 2) - devise (~> 4.0) - railties (< 6.1) - rotp (~> 2.0) devise_pam_authenticatable2 (9.2.0) devise (>= 4.0.0) rpam2 (~> 4.0) @@ -370,11 +413,6 @@ GEM racc (~> 1.4) nokogumbo (2.0.4) nokogiri (~> 1.8, >= 1.8.4) - nsa (0.2.7) - activesupport (>= 4.2, < 6) - concurrent-ruby (~> 1.0, >= 1.0.2) - sidekiq (>= 3.5) - statsd-ruby (~> 1.4, >= 1.4.0) oj (3.11.3) omniauth (1.9.1) hashie (>= 3.4.6) @@ -414,9 +452,6 @@ GEM pghero (2.8.0) activerecord (>= 5) pkg-config (1.4.5) - pluck_each (0.1.3) - activerecord (> 3.2.0) - activesupport (> 3.0.0) posix-spawn (0.3.15) premailer (1.14.2) addressable @@ -450,18 +485,20 @@ GEM rack rack-test (1.1.0) rack (>= 1.0, < 3) - rails (5.2.4.5) - actioncable (= 5.2.4.5) - actionmailer (= 5.2.4.5) - actionpack (= 5.2.4.5) - actionview (= 5.2.4.5) - activejob (= 5.2.4.5) - activemodel (= 5.2.4.5) - activerecord (= 5.2.4.5) - activestorage (= 5.2.4.5) - activesupport (= 5.2.4.5) - bundler (>= 1.3.0) - railties (= 5.2.4.5) + rails (6.1.3) + actioncable (= 6.1.3) + actionmailbox (= 6.1.3) + actionmailer (= 6.1.3) + actionpack (= 6.1.3) + actiontext (= 6.1.3) + actionview (= 6.1.3) + activejob (= 6.1.3) + activemodel (= 6.1.3) + activerecord (= 6.1.3) + activestorage (= 6.1.3) + activesupport (= 6.1.3) + bundler (>= 1.15.0) + railties (= 6.1.3) sprockets-rails (>= 2.0.0) rails-controller-testing (1.0.5) actionpack (>= 5.0.1.rc1) @@ -472,17 +509,17 @@ GEM nokogiri (>= 1.6) rails-html-sanitizer (1.3.0) loofah (~> 2.3) - rails-i18n (5.1.3) + rails-i18n (6.0.0) i18n (>= 0.7, < 2) - railties (>= 5.0, < 6) + railties (>= 6.0.0, < 7) rails-settings-cached (0.6.6) rails (>= 4.2.0) - railties (5.2.4.5) - actionpack (= 5.2.4.5) - activesupport (= 5.2.4.5) + railties (6.1.3) + actionpack (= 6.1.3) + activesupport (= 6.1.3) method_source rake (>= 0.8.7) - thor (>= 0.19.0, < 2.0) + thor (~> 1.0) rainbow (3.0.0) rake (13.0.3) rdf (3.1.13) @@ -500,7 +537,7 @@ GEM actionpack (>= 5.0) railties (>= 5.0) rexml (3.2.4) - rotp (2.1.2) + rotp (6.2.0) rpam2 (4.0.2) rqrcode (1.2.0) chunky_png (~> 1.0) @@ -600,7 +637,7 @@ GEM net-scp (>= 1.1.2) net-ssh (>= 2.8.0) stackprof (0.2.16) - statsd-ruby (1.4.0) + statsd-ruby (1.5.0) stoplight (2.2.1) streamio-ffmpeg (3.0.2) multi_json (~> 1.8) @@ -612,7 +649,6 @@ GEM terrapin (0.6.0) climate_control (>= 0.0.3, < 1.0) thor (1.1.0) - thread_safe (0.3.6) thwait (0.2.0) e2mmap tilt (2.0.10) @@ -632,8 +668,8 @@ GEM twitter-text (3.1.0) idn-ruby unf (~> 0.1.0) - tzinfo (1.2.9) - thread_safe (~> 0.1) + tzinfo (2.0.4) + concurrent-ruby (~> 1.0) tzinfo-data (1.2021.1) tzinfo (>= 1.0.0) unf (0.1.4) @@ -672,6 +708,7 @@ GEM xorcist (1.1.2) xpath (3.2.0) nokogiri (~> 1.8) + zeitwerk (2.4.2) PLATFORMS ruby @@ -703,7 +740,7 @@ DEPENDENCIES concurrent-ruby connection_pool devise (~> 4.7) - devise-two-factor (~> 3.1) + devise-two-factor! devise_pam_authenticatable2 (~> 9.2) discard (~> 1.2) doorkeeper (~> 5.5) @@ -741,7 +778,7 @@ DEPENDENCIES net-ldap (~> 0.17) nilsimsa! nokogiri (~> 1.11) - nsa (~> 0.2) + nsa! oj (~> 3.11) omniauth (~> 1.9) omniauth-cas (~> 2.0) @@ -756,7 +793,7 @@ DEPENDENCIES pg (~> 1.2) pghero (~> 2.8) pkg-config (~> 1.4) - pluck_each (~> 0.1.3) + pluck_each! posix-spawn premailer-rails private_address_check (~> 0.5) @@ -767,9 +804,9 @@ DEPENDENCIES rack (~> 2.2.3) rack-attack (~> 6.5) rack-cors (~> 1.1) - rails (~> 5.2.4.5) + rails (~> 6.1.3) rails-controller-testing (~> 1.0) - rails-i18n (~> 5.1) + rails-i18n (~> 6.0) rails-settings-cached (~> 0.6) rdf-normalize (~> 0.4) redis (~> 4.2) diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 5b7eec94f..6361d4b27 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -5,8 +5,6 @@ class ApplicationController < ActionController::Base # For APIs, you may want to use :null_session instead. protect_from_forgery with: :exception - force_ssl if: :https_enabled? - include Localized include UserTrackingConcern include SessionTrackingConcern @@ -42,10 +40,6 @@ class ApplicationController < ActionController::Base private - def https_enabled? - Rails.env.production? && !request.path.start_with?('/health') && !request.headers["Host"].end_with?(".onion") - end - def authorized_fetch_mode? ENV['AUTHORIZED_FETCH'] == 'true' || Rails.configuration.x.whitelist_mode end diff --git a/app/lib/delivery_failure_tracker.rb b/app/lib/delivery_failure_tracker.rb index 25fa694d2..2cd6ef7ad 100644 --- a/app/lib/delivery_failure_tracker.rb +++ b/app/lib/delivery_failure_tracker.rb @@ -29,7 +29,7 @@ class DeliveryFailureTracker class << self def without_unavailable(urls) - unavailable_domains_map = Rails.cache.fetch('unavailable_domains') { UnavailableDomain.pluck(:domain).each_with_object({}) { |domain, hash| hash[domain] = true } } + unavailable_domains_map = Rails.cache.fetch('unavailable_domains') { UnavailableDomain.pluck(:domain).index_with(true) } urls.reject do |url| host = Addressable::URI.parse(url).normalized_host diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb index 165338437..43aeecb35 100644 --- a/app/lib/feed_manager.rb +++ b/app/lib/feed_manager.rb @@ -533,12 +533,12 @@ class FeedManager arr end - crutches[:following] = Follow.where(account_id: receiver_id, target_account_id: statuses.map(&:in_reply_to_account_id).compact).pluck(:target_account_id).each_with_object({}) { |id, mapping| mapping[id] = true } - crutches[:hiding_reblogs] = Follow.where(account_id: receiver_id, target_account_id: statuses.map { |s| s.account_id if s.reblog? }.compact, show_reblogs: false).pluck(:target_account_id).each_with_object({}) { |id, mapping| mapping[id] = true } - crutches[:blocking] = Block.where(account_id: receiver_id, target_account_id: check_for_blocks).pluck(:target_account_id).each_with_object({}) { |id, mapping| mapping[id] = true } - crutches[:muting] = Mute.where(account_id: receiver_id, target_account_id: check_for_blocks).pluck(:target_account_id).each_with_object({}) { |id, mapping| mapping[id] = true } - crutches[:domain_blocking] = AccountDomainBlock.where(account_id: receiver_id, domain: statuses.map { |s| s.reblog&.account&.domain }.compact).pluck(:domain).each_with_object({}) { |domain, mapping| mapping[domain] = true } - crutches[:blocked_by] = Block.where(target_account_id: receiver_id, account_id: statuses.map { |s| s.reblog&.account_id }.compact).pluck(:account_id).each_with_object({}) { |id, mapping| mapping[id] = true } + crutches[:following] = Follow.where(account_id: receiver_id, target_account_id: statuses.map(&:in_reply_to_account_id).compact).pluck(:target_account_id).index_with(true) + crutches[:hiding_reblogs] = Follow.where(account_id: receiver_id, target_account_id: statuses.map { |s| s.account_id if s.reblog? }.compact, show_reblogs: false).pluck(:target_account_id).index_with(true) + crutches[:blocking] = Block.where(account_id: receiver_id, target_account_id: check_for_blocks).pluck(:target_account_id).index_with(true) + crutches[:muting] = Mute.where(account_id: receiver_id, target_account_id: check_for_blocks).pluck(:target_account_id).index_with(true) + crutches[:domain_blocking] = AccountDomainBlock.where(account_id: receiver_id, domain: statuses.map { |s| s.reblog&.account&.domain }.compact).pluck(:domain).index_with(true) + crutches[:blocked_by] = Block.where(target_account_id: receiver_id, account_id: statuses.map { |s| s.reblog&.account_id }.compact).pluck(:account_id).index_with(true) crutches end diff --git a/app/lib/settings/scoped_settings.rb b/app/lib/settings/scoped_settings.rb index acabf0c8e..1e18d6d46 100644 --- a/app/lib/settings/scoped_settings.rb +++ b/app/lib/settings/scoped_settings.rb @@ -63,7 +63,7 @@ module Settings class << self def default_settings - defaulting = DEFAULTING_TO_UNSCOPED.each_with_object({}) { |k, h| h[k] = Setting[k] } + defaulting = DEFAULTING_TO_UNSCOPED.index_with { |k| Setting[k] } Setting.default_settings.merge!(defaulting) end end diff --git a/app/models/concerns/account_interactions.rb b/app/models/concerns/account_interactions.rb index 974f57820..51e8e04a8 100644 --- a/app/models/concerns/account_interactions.rb +++ b/app/models/concerns/account_interactions.rb @@ -67,7 +67,7 @@ module AccountInteractions private def follow_mapping(query, field) - query.pluck(field).each_with_object({}) { |id, mapping| mapping[id] = true } + query.pluck(field).index_with(true) end end diff --git a/app/models/report.rb b/app/models/report.rb index cd08120e4..ef41547d9 100644 --- a/app/models/report.rb +++ b/app/models/report.rb @@ -32,7 +32,7 @@ class Report < ApplicationRecord scope :unresolved, -> { where(action_taken: false) } scope :resolved, -> { where(action_taken: true) } - scope :with_accounts, -> { includes([:account, :target_account, :action_taken_by_account, :assigned_account].each_with_object({}) { |k, h| h[k] = { user: [:invite_request, :invite] } }) } + scope :with_accounts, -> { includes([:account, :target_account, :action_taken_by_account, :assigned_account].index_with({ user: [:invite_request, :invite] })) } validates :comment, length: { maximum: 1000 } diff --git a/app/services/delete_account_service.rb b/app/services/delete_account_service.rb index 802799ccd..182f0e127 100644 --- a/app/services/delete_account_service.rb +++ b/app/services/delete_account_service.rb @@ -188,8 +188,7 @@ class DeleteAccountService < BaseService ids = favourites.pluck(:status_id) StatusStat.where(status_id: ids).update_all('favourites_count = GREATEST(0, favourites_count - 1)') Chewy.strategy.current.update(StatusesIndex::Status, ids) if Chewy.enabled? - # Rails.cache.delete_multi would be better, but we don't have it yet - ids.each { |id| Rails.cache.delete("statuses/#{id}") } + Rails.cache.delete_multi(ids.map { |id| "statuses/#{id}" }) favourites.delete_all end end diff --git a/app/services/import_service.rb b/app/services/import_service.rb index b11532283..74ad5b79f 100644 --- a/app/services/import_service.rb +++ b/app/services/import_service.rb @@ -45,7 +45,7 @@ class ImportService < BaseService items = @data.take(ROWS_PROCESSING_LIMIT).map { |row| row['#domain'].strip } if @import.overwrite? - presence_hash = items.each_with_object({}) { |id, mapping| mapping[id] = true } + presence_hash = items.index_with(true) @account.domain_blocks.find_each do |domain_block| if presence_hash[domain_block.domain] @@ -96,7 +96,7 @@ class ImportService < BaseService items = @data.take(ROWS_PROCESSING_LIMIT).map { |row| row['#uri'].strip } if @import.overwrite? - presence_hash = items.each_with_object({}) { |id, mapping| mapping[id] = true } + presence_hash = items.index_with(true) @account.bookmarks.find_each do |bookmark| if presence_hash[bookmark.status.uri] diff --git a/bin/setup b/bin/setup index fc77b0809..90700ac4f 100755 --- a/bin/setup +++ b/bin/setup @@ -1,6 +1,5 @@ #!/usr/bin/env ruby -require 'fileutils' -include FileUtils +require "fileutils" # path to your application root. APP_ROOT = File.expand_path('..', __dir__) @@ -9,22 +8,25 @@ def system!(*args) system(*args) || abort("\n== Command #{args} failed ==") end -chdir APP_ROOT do - # This script is a starting point to setup your application. +FileUtils.chdir APP_ROOT do + # This script is a way to set up or update your development environment automatically. + # This script is idempotent, so that you can run it at any time and get an expectable outcome. # Add necessary setup steps to this file. puts '== Installing dependencies ==' system! 'gem install bundler --conservative' system('bundle check') || system!('bundle install') - system!('yarn install') + + # Install JavaScript dependencies + system! 'bin/yarn' # puts "\n== Copying sample files ==" # unless File.exist?('config/database.yml') - # cp 'config/database.yml.sample', 'config/database.yml' + # FileUtils.cp 'config/database.yml.sample', 'config/database.yml' # end puts "\n== Preparing database ==" - system! 'bin/rails db:setup' + system! 'bin/rails db:prepare' puts "\n== Removing old logs and tempfiles ==" system! 'bin/rails log:clear tmp:clear' diff --git a/bin/yarn b/bin/yarn index 460dd565b..9fab2c350 100755 --- a/bin/yarn +++ b/bin/yarn @@ -1,9 +1,15 @@ #!/usr/bin/env ruby APP_ROOT = File.expand_path('..', __dir__) Dir.chdir(APP_ROOT) do - begin - exec "yarnpkg", *ARGV - rescue Errno::ENOENT + yarn = ENV["PATH"].split(File::PATH_SEPARATOR). + select { |dir| File.expand_path(dir) != __dir__ }. + product(["yarn", "yarn.cmd", "yarn.ps1"]). + map { |dir, file| File.expand_path(file, dir) }. + find { |file| File.executable?(file) } + + if yarn + exec yarn, *ARGV + else $stderr.puts "Yarn executable was not detected in the system." $stderr.puts "Download Yarn at https://yarnpkg.com/en/docs/install" exit 1 diff --git a/config/application.rb b/config/application.rb index 3267fa71b..c911e76dc 100644 --- a/config/application.rb +++ b/config/application.rb @@ -39,7 +39,8 @@ require_relative '../lib/mastodon/redis_config' module Mastodon class Application < Rails::Application # Initialize configuration defaults for originally generated Rails version. - config.load_defaults 5.2 + config.load_defaults 6.1 + config.add_autoload_paths_to_load_path = false # Settings in config/environments/* take precedence over those specified here. # Application configuration should go into files in config/initializers diff --git a/config/environments/production.rb b/config/environments/production.rb index 81a67902e..6df0a3365 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -44,6 +44,13 @@ Rails.application.configure do # Allow to specify public IP of reverse proxy if it's needed config.action_dispatch.trusted_proxies = ENV['TRUSTED_PROXY_IP'].split.map { |item| IPAddr.new(item) } if ENV['TRUSTED_PROXY_IP'].present? + config.force_ssl = true + config.ssl_options = { + redirect: { + exclude: -> request { request.path.start_with?('/health') || request.headers["Host"].end_with?('.onion') } + } + } + # Use the lowest log level to ensure availability of diagnostic information # when problems arise. config.log_level = ENV.fetch('RAILS_LOG_LEVEL', 'info').to_sym diff --git a/config/initializers/application_controller_renderer.rb b/config/initializers/application_controller_renderer.rb index 51639b67a..89d2efab2 100644 --- a/config/initializers/application_controller_renderer.rb +++ b/config/initializers/application_controller_renderer.rb @@ -1,6 +1,8 @@ # Be sure to restart your server when you modify this file. -# ApplicationController.renderer.defaults.merge!( -# http_host: 'example.org', -# https: false -# ) +# ActiveSupport::Reloader.to_prepare do +# ApplicationController.renderer.defaults.merge!( +# http_host: 'example.org', +# https: false +# ) +# end diff --git a/config/initializers/backtrace_silencers.rb b/config/initializers/backtrace_silencers.rb index 59385cdf3..33699c309 100644 --- a/config/initializers/backtrace_silencers.rb +++ b/config/initializers/backtrace_silencers.rb @@ -1,7 +1,8 @@ # Be sure to restart your server when you modify this file. # You can add backtrace silencers for libraries that you're using but don't wish to see in your backtraces. -# Rails.backtrace_cleaner.add_silencer { |line| line =~ /my_noisy_library/ } +# Rails.backtrace_cleaner.add_silencer { |line| /my_noisy_library/.match?(line) } -# You can also remove all the silencers if you're trying to debug a problem that might stem from framework code. -# Rails.backtrace_cleaner.remove_silencers! +# You can also remove all the silencers if you're trying to debug a problem that might stem from framework code +# by setting BACKTRACE=1 before calling your invocation, like "BACKTRACE=1 ./bin/rails runner 'MyClass.perform'". +Rails.backtrace_cleaner.remove_silencers! if ENV["BACKTRACE"] diff --git a/config/initializers/content_security_policy.rb b/config/initializers/content_security_policy.rb index 98dc711e1..92645ff28 100644 --- a/config/initializers/content_security_policy.rb +++ b/config/initializers/content_security_policy.rb @@ -49,17 +49,7 @@ end Rails.application.config.content_security_policy_nonce_generator = -> request { SecureRandom.base64(16) } -# Monkey-patching Rails 5 -module ActionDispatch - class ContentSecurityPolicy - def nonce_directive?(directive) - directive == 'style-src' - end - end -end - -# Rails 6 would require the following instead: -# Rails.application.config.content_security_policy_nonce_directives = %w(style-src) +Rails.application.config.content_security_policy_nonce_directives = %w(style-src) PgHero::HomeController.content_security_policy do |p| p.script_src :self, :unsafe_inline, assets_host diff --git a/config/initializers/permissions_policy.rb b/config/initializers/permissions_policy.rb new file mode 100644 index 000000000..00f64d71b --- /dev/null +++ b/config/initializers/permissions_policy.rb @@ -0,0 +1,11 @@ +# Define an application-wide HTTP permissions policy. For further +# information see https://developers.google.com/web/updates/2018/06/feature-policy +# +# Rails.application.config.permissions_policy do |f| +# f.camera :none +# f.gyroscope :none +# f.microphone :none +# f.usb :none +# f.fullscreen :self +# f.payment :self, "https://secure.example.com" +# end diff --git a/config/initializers/preload_link_headers.rb b/config/initializers/preload_link_headers.rb new file mode 100644 index 000000000..9f21c45ec --- /dev/null +++ b/config/initializers/preload_link_headers.rb @@ -0,0 +1,8 @@ +# Since Rails 6.1, ActionView adds preload links for javascript files +# in the Links header per default. + +# In our case, that will bloat headers too much and potentially cause +# issues with reverse proxies. Furhermore, we don't need those links, +# as we already output them as HTML link tags. + +Rails.application.config.action_view.preload_links_header = false diff --git a/config/storage.yml b/config/storage.yml new file mode 100644 index 000000000..e69de29bb diff --git a/lib/tasks/emojis.rake b/lib/tasks/emojis.rake index 01ae95564..c8655cc47 100644 --- a/lib/tasks/emojis.rake +++ b/lib/tasks/emojis.rake @@ -69,7 +69,7 @@ namespace :emojis do end end - existence_maps = grouped_codes.map { |c| c.map { |cc| [cc, File.exist?(Rails.root.join('public', 'emoji', codepoints_to_filename(cc) + '.svg'))] }.to_h } + existence_maps = grouped_codes.map { |c| c.index_with { |cc| File.exist?(Rails.root.join('public', 'emoji', codepoints_to_filename(cc) + '.svg')) } } map = {} existence_maps.each do |group| diff --git a/spec/controllers/api/v1/accounts/credentials_controller_spec.rb b/spec/controllers/api/v1/accounts/credentials_controller_spec.rb index ebd462a03..9fb0d8770 100644 --- a/spec/controllers/api/v1/accounts/credentials_controller_spec.rb +++ b/spec/controllers/api/v1/accounts/credentials_controller_spec.rb @@ -30,8 +30,8 @@ describe Api::V1::Accounts::CredentialsController do patch :update, params: { display_name: "Alice Isn't Dead", note: "Hi!\n\nToot toot!", - avatar: fixture_file_upload('files/avatar.gif', 'image/gif'), - header: fixture_file_upload('files/attachment.jpg', 'image/jpeg'), + avatar: fixture_file_upload('avatar.gif', 'image/gif'), + header: fixture_file_upload('attachment.jpg', 'image/jpeg'), source: { privacy: 'unlisted', sensitive: true, diff --git a/spec/controllers/api/v1/media_controller_spec.rb b/spec/controllers/api/v1/media_controller_spec.rb index 4e3037208..3eb015a1c 100644 --- a/spec/controllers/api/v1/media_controller_spec.rb +++ b/spec/controllers/api/v1/media_controller_spec.rb @@ -15,7 +15,7 @@ RSpec.describe Api::V1::MediaController, type: :controller do context 'when imagemagick cant identify the file type' do before do expect_any_instance_of(Account).to receive_message_chain(:media_attachments, :create!).and_raise(Paperclip::Errors::NotIdentifiedByImageMagickError) - post :create, params: { file: fixture_file_upload('files/attachment.jpg', 'image/jpeg') } + post :create, params: { file: fixture_file_upload('attachment.jpg', 'image/jpeg') } end it 'returns http 422' do @@ -26,7 +26,7 @@ RSpec.describe Api::V1::MediaController, type: :controller do context 'when there is a generic error' do before do expect_any_instance_of(Account).to receive_message_chain(:media_attachments, :create!).and_raise(Paperclip::Error) - post :create, params: { file: fixture_file_upload('files/attachment.jpg', 'image/jpeg') } + post :create, params: { file: fixture_file_upload('attachment.jpg', 'image/jpeg') } end it 'returns http 422' do @@ -37,7 +37,7 @@ RSpec.describe Api::V1::MediaController, type: :controller do context 'image/jpeg' do before do - post :create, params: { file: fixture_file_upload('files/attachment.jpg', 'image/jpeg') } + post :create, params: { file: fixture_file_upload('attachment.jpg', 'image/jpeg') } end it 'returns http success' do @@ -59,7 +59,7 @@ RSpec.describe Api::V1::MediaController, type: :controller do context 'image/gif' do before do - post :create, params: { file: fixture_file_upload('files/attachment.gif', 'image/gif') } + post :create, params: { file: fixture_file_upload('attachment.gif', 'image/gif') } end it 'returns http success' do @@ -81,7 +81,7 @@ RSpec.describe Api::V1::MediaController, type: :controller do context 'video/webm' do before do - post :create, params: { file: fixture_file_upload('files/attachment.webm', 'video/webm') } + post :create, params: { file: fixture_file_upload('attachment.webm', 'video/webm') } end it do diff --git a/spec/controllers/application_controller_spec.rb b/spec/controllers/application_controller_spec.rb index e73a08a0e..458298a6b 100644 --- a/spec/controllers/application_controller_spec.rb +++ b/spec/controllers/application_controller_spec.rb @@ -42,20 +42,6 @@ describe ApplicationController, type: :controller do include_examples 'respond_with_error', 422 end - it "does not force ssl if Rails.env.production? is not 'true'" do - routes.draw { get 'success' => 'anonymous#success' } - allow(Rails.env).to receive(:production?).and_return(false) - get 'success' - expect(response).to have_http_status(200) - end - - it "forces ssl if Rails.env.production? is 'true'" do - routes.draw { get 'success' => 'anonymous#success' } - allow(Rails.env).to receive(:production?).and_return(true) - get 'success' - expect(response).to redirect_to('https://test.host/success') - end - describe 'helper_method :current_account' do it 'returns nil if not signed in' do expect(controller.view_context.current_account).to be_nil diff --git a/spec/controllers/settings/imports_controller_spec.rb b/spec/controllers/settings/imports_controller_spec.rb index 7a9b02195..b8caf5941 100644 --- a/spec/controllers/settings/imports_controller_spec.rb +++ b/spec/controllers/settings/imports_controller_spec.rb @@ -21,7 +21,7 @@ RSpec.describe Settings::ImportsController, type: :controller do post :create, params: { import: { type: 'following', - data: fixture_file_upload('files/imports.txt') + data: fixture_file_upload('imports.txt') } } @@ -34,7 +34,7 @@ RSpec.describe Settings::ImportsController, type: :controller do post :create, params: { import: { type: 'blocking', - data: fixture_file_upload('files/imports.txt') + data: fixture_file_upload('imports.txt') } } diff --git a/spec/controllers/settings/profiles_controller_spec.rb b/spec/controllers/settings/profiles_controller_spec.rb index 5b1fe3aca..1ac286254 100644 --- a/spec/controllers/settings/profiles_controller_spec.rb +++ b/spec/controllers/settings/profiles_controller_spec.rb @@ -33,7 +33,7 @@ RSpec.describe Settings::ProfilesController, type: :controller do account = Fabricate(:account, user: @user, display_name: 'AvatarTest') expect(account.avatar.instance.avatar_file_name).to be_nil - put :update, params: { account: { avatar: fixture_file_upload('files/avatar.gif', 'image/gif') } } + put :update, params: { account: { avatar: fixture_file_upload('avatar.gif', 'image/gif') } } expect(response).to redirect_to(settings_profile_path) expect(account.reload.avatar.instance.avatar_file_name).not_to be_nil expect(ActivityPub::UpdateDistributionWorker).to have_received(:perform_async).with(account.id) @@ -44,7 +44,7 @@ RSpec.describe Settings::ProfilesController, type: :controller do it 'gives the user an error message' do allow(ActivityPub::UpdateDistributionWorker).to receive(:perform_async) account = Fabricate(:account, user: @user, display_name: 'AvatarTest') - put :update, params: { account: { avatar: fixture_file_upload('files/4096x4097.png', 'image/png') } } + put :update, params: { account: { avatar: fixture_file_upload('4096x4097.png', 'image/png') } } expect(response.body).to include('images are not supported') end end diff --git a/spec/controllers/settings/two_factor_authentication/confirmations_controller_spec.rb b/spec/controllers/settings/two_factor_authentication/confirmations_controller_spec.rb index cdfeef8d6..7b86513be 100644 --- a/spec/controllers/settings/two_factor_authentication/confirmations_controller_spec.rb +++ b/spec/controllers/settings/two_factor_authentication/confirmations_controller_spec.rb @@ -11,7 +11,7 @@ describe Settings::TwoFactorAuthentication::ConfirmationsController do subject expect(assigns(:confirmation)).to be_instance_of Form::TwoFactorConfirmation - expect(assigns(:provision_url)).to eq 'otpauth://totp/local-part@domain?secret=thisisasecretforthespecofnewview&issuer=cb6e6126.ngrok.io' + expect(assigns(:provision_url)).to eq 'otpauth://totp/cb6e6126.ngrok.io:local-part%40domain?secret=thisisasecretforthespecofnewview&issuer=cb6e6126.ngrok.io' expect(assigns(:qrcode)).to be_instance_of RQRCode::QRCode expect(response).to have_http_status(200) expect(response).to render_template(:new) diff --git a/spec/models/setting_spec.rb b/spec/models/setting_spec.rb index 1cc528674..3ccc21d6c 100644 --- a/spec/models/setting_spec.rb +++ b/spec/models/setting_spec.rb @@ -99,11 +99,12 @@ RSpec.describe Setting, type: :model do end it 'does not query the database' do - expect do |callback| - ActiveSupport::Notifications.subscribed callback, 'sql.active_record' do - described_class[key] - end - end.not_to yield_control + callback = double + allow(callback).to receive(:call) + ActiveSupport::Notifications.subscribed callback, 'sql.active_record' do + described_class[key] + end + expect(callback).not_to have_received(:call) end it 'returns the cached value' do diff --git a/spec/models/user_spec.rb b/spec/models/user_spec.rb index 1dae43536..5db249be2 100644 --- a/spec/models/user_spec.rb +++ b/spec/models/user_spec.rb @@ -175,7 +175,7 @@ RSpec.describe User, type: :model do user = Fabricate(:user) ActiveJob::Base.queue_adapter = :test - expect { user.send_confirmation_instructions }.to have_enqueued_job(ActionMailer::DeliveryJob) + expect { user.send_confirmation_instructions }.to have_enqueued_job(ActionMailer::MailDeliveryJob) end end -- cgit From d33351af3c668f585b046e19233523083bfb7075 Mon Sep 17 00:00:00 2001 From: Stanislas Date: Wed, 24 Mar 2021 10:55:16 +0100 Subject: tootctl emoji import: case insensitive duplicate check (#15738) --- lib/mastodon/emoji_cli.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/mastodon/emoji_cli.rb b/lib/mastodon/emoji_cli.rb index 0a1f538e6..5bee70ea5 100644 --- a/lib/mastodon/emoji_cli.rb +++ b/lib/mastodon/emoji_cli.rb @@ -49,7 +49,7 @@ module Mastodon next if filename.start_with?('._') shortcode = [options[:prefix], filename, options[:suffix]].compact.join - custom_emoji = CustomEmoji.local.find_by(shortcode: shortcode) + custom_emoji = CustomEmoji.local.find_by("LOWER(shortcode) = ?", shortcode.downcase) if custom_emoji && !options[:overwrite] skipped += 1 -- cgit