diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/active_record/database_tasks_extensions.rb | 20 | ||||
| -rw-r--r-- | lib/exceptions.rb | 23 | ||||
| -rw-r--r-- | lib/mastodon/emoji_cli.rb | 2 | ||||
| -rw-r--r-- | lib/mastodon/maintenance_cli.rb | 4 | ||||
| -rw-r--r-- | lib/mastodon/migration_helpers.rb | 90 | ||||
| -rw-r--r-- | lib/mastodon/redis_config.rb | 2 | ||||
| -rw-r--r-- | lib/sanitize_ext/sanitize_config.rb | 140 | ||||
| -rw-r--r-- | lib/tasks/db.rake | 62 | ||||
| -rw-r--r-- | lib/tasks/emojis.rake | 2 | ||||
| -rw-r--r-- | lib/tasks/mastodon.rake | 22 |
10 files changed, 224 insertions, 143 deletions
diff --git a/lib/active_record/database_tasks_extensions.rb b/lib/active_record/database_tasks_extensions.rb new file mode 100644 index 000000000..e274f476d --- /dev/null +++ b/lib/active_record/database_tasks_extensions.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +require_relative '../mastodon/snowflake' + +module ActiveRecord + module Tasks + module DatabaseTasks + original_load_schema = instance_method(:load_schema) + + define_method(:load_schema) do |db_config, *args| + ActiveRecord::Base.establish_connection(db_config) + Mastodon::Snowflake.define_timestamp_id + + original_load_schema.bind(self).call(db_config, *args) + + Mastodon::Snowflake.ensure_id_sequences_exist + end + end + end +end diff --git a/lib/exceptions.rb b/lib/exceptions.rb new file mode 100644 index 000000000..7c8e77871 --- /dev/null +++ b/lib/exceptions.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Mastodon + class Error < StandardError; end + class NotPermittedError < Error; end + class ValidationError < Error; end + class HostValidationError < ValidationError; end + class LengthValidationError < ValidationError; end + class DimensionsValidationError < ValidationError; end + class StreamValidationError < ValidationError; end + class RaceConditionError < Error; end + class RateLimitExceededError < Error; end + + class UnexpectedResponseError < Error + def initialize(response = nil) + if response.respond_to? :uri + super("#{response.uri} returned code #{response.code}") + else + super + end + end + end +end diff --git a/lib/mastodon/emoji_cli.rb b/lib/mastodon/emoji_cli.rb index 0a1f538e6..5bee70ea5 100644 --- a/lib/mastodon/emoji_cli.rb +++ b/lib/mastodon/emoji_cli.rb @@ -49,7 +49,7 @@ module Mastodon next if filename.start_with?('._') shortcode = [options[:prefix], filename, options[:suffix]].compact.join - custom_emoji = CustomEmoji.local.find_by(shortcode: shortcode) + custom_emoji = CustomEmoji.local.find_by("LOWER(shortcode) = ?", shortcode.downcase) if custom_emoji && !options[:overwrite] skipped += 1 diff --git a/lib/mastodon/maintenance_cli.rb b/lib/mastodon/maintenance_cli.rb index 029d42a05..9f1eaf263 100644 --- a/lib/mastodon/maintenance_cli.rb +++ b/lib/mastodon/maintenance_cli.rb @@ -14,7 +14,7 @@ module Mastodon end MIN_SUPPORTED_VERSION = 2019_10_01_213028 - MAX_SUPPORTED_VERSION = 2020_12_18_054746 + MAX_SUPPORTED_VERSION = 2021_03_08_133107 # Stubs to enjoy ActiveRecord queries while not depending on a particular # version of the code/database @@ -142,7 +142,6 @@ module Mastodon @prompt.warn 'Please make sure to stop Mastodon and have a backup.' exit(1) unless @prompt.yes?('Continue?') - deduplicate_accounts! deduplicate_users! deduplicate_account_domain_blocks! deduplicate_account_identity_proofs! @@ -157,6 +156,7 @@ module Mastodon deduplicate_media_attachments! deduplicate_preview_cards! deduplicate_statuses! + deduplicate_accounts! deduplicate_tags! deduplicate_webauthn_credentials! diff --git a/lib/mastodon/migration_helpers.rb b/lib/mastodon/migration_helpers.rb index bf2314ecb..147642a1c 100644 --- a/lib/mastodon/migration_helpers.rb +++ b/lib/mastodon/migration_helpers.rb @@ -41,42 +41,18 @@ module Mastodon module MigrationHelpers - # Stub for Database.postgresql? from GitLab - def self.postgresql? - ActiveRecord::Base.configurations[Rails.env]['adapter'].casecmp('postgresql').zero? - end - - # Stub for Database.mysql? from GitLab - def self.mysql? - ActiveRecord::Base.configurations[Rails.env]['adapter'].casecmp('mysql2').zero? - end - # Model that can be used for querying permissions of a SQL user. class Grant < ActiveRecord::Base - self.table_name = - if Mastodon::MigrationHelpers.postgresql? - 'information_schema.role_table_grants' - else - 'mysql.user' - end + self.table_name = 'information_schema.role_table_grants' def self.scope_to_current_user - if Mastodon::MigrationHelpers.postgresql? - where('grantee = user') - else - where("CONCAT(User, '@', Host) = current_user()") - end + where('grantee = user') end # Returns true if the current user can create and execute triggers on the # given table. def self.create_and_execute_trigger?(table) - priv = - if Mastodon::MigrationHelpers.postgresql? - where(privilege_type: 'TRIGGER', table_name: table) - else - where(Trigger_priv: 'Y') - end + priv = where(privilege_type: 'TRIGGER', table_name: table) priv.scope_to_current_user.any? end @@ -141,10 +117,8 @@ module Mastodon 'in the body of your migration class' end - if MigrationHelpers.postgresql? - options = options.merge({ algorithm: :concurrently }) - disable_statement_timeout - end + options = options.merge({ algorithm: :concurrently }) + disable_statement_timeout add_index(table_name, column_name, options) end @@ -199,8 +173,6 @@ module Mastodon # Only available on Postgresql >= 9.2 def supports_drop_index_concurrently? - return false unless MigrationHelpers.postgresql? - version = select_one("SELECT current_setting('server_version_num') AS v")['v'].to_i version >= 90200 @@ -226,13 +198,7 @@ module Mastodon # While MySQL does allow disabling of foreign keys it has no equivalent # of PostgreSQL's "VALIDATE CONSTRAINT". As a result we'll just fall # back to the normal foreign key procedure. - if MigrationHelpers.mysql? - return add_foreign_key(source, target, - column: column, - on_delete: on_delete) - else - on_delete = 'SET NULL' if on_delete == :nullify - end + on_delete = 'SET NULL' if on_delete == :nullify disable_statement_timeout @@ -270,7 +236,7 @@ module Mastodon # the database. Disable the session's statement timeout to ensure # migrations don't get killed prematurely. (PostgreSQL only) def disable_statement_timeout - execute('SET statement_timeout TO 0') if MigrationHelpers.postgresql? + execute('SET statement_timeout TO 0') end # Updates the value of a column in batches. @@ -319,7 +285,7 @@ module Mastodon count_arel = table.project(Arel.star.count.as('count')) count_arel = yield table, count_arel if block_given? - total = exec_query(count_arel.to_sql).to_hash.first['count'].to_i + total = exec_query(count_arel.to_sql).to_ary.first['count'].to_i return if total == 0 end @@ -335,7 +301,7 @@ module Mastodon start_arel = table.project(table[:id]).order(table[:id].asc).take(1) start_arel = yield table, start_arel if block_given? - first_row = exec_query(start_arel.to_sql).to_hash.first + first_row = exec_query(start_arel.to_sql).to_ary.first # In case there are no rows but we didn't catch it in the estimated size: return unless first_row start_id = first_row['id'].to_i @@ -356,7 +322,7 @@ module Mastodon .skip(batch_size) stop_arel = yield table, stop_arel if block_given? - stop_row = exec_query(stop_arel.to_sql).to_hash.first + stop_row = exec_query(stop_arel.to_sql).to_ary.first update_arel = Arel::UpdateManager.new .table(table) @@ -487,11 +453,7 @@ module Mastodon # If we were in the middle of update_column_in_batches, we should remove # the old column and start over, as we have no idea where we were. if column_for(table, new) - if MigrationHelpers.postgresql? - remove_rename_triggers_for_postgresql(table, trigger_name) - else - remove_rename_triggers_for_mysql(trigger_name) - end + remove_rename_triggers_for_postgresql(table, trigger_name) remove_column(table, new) end @@ -521,13 +483,8 @@ module Mastodon quoted_old = quote_column_name(old) quoted_new = quote_column_name(new) - if MigrationHelpers.postgresql? - install_rename_triggers_for_postgresql(trigger_name, quoted_table, - quoted_old, quoted_new) - else - install_rename_triggers_for_mysql(trigger_name, quoted_table, - quoted_old, quoted_new) - end + install_rename_triggers_for_postgresql(trigger_name, quoted_table, + quoted_old, quoted_new) update_column_in_batches(table, new, Arel::Table.new(table)[old]) @@ -685,11 +642,7 @@ module Mastodon check_trigger_permissions!(table) - if MigrationHelpers.postgresql? - remove_rename_triggers_for_postgresql(table, trigger_name) - else - remove_rename_triggers_for_mysql(trigger_name) - end + remove_rename_triggers_for_postgresql(table, trigger_name) remove_column(table, old) end @@ -844,18 +797,9 @@ module Mastodon quoted_pattern = Arel::Nodes::Quoted.new(pattern.to_s) quoted_replacement = Arel::Nodes::Quoted.new(replacement.to_s) - if MigrationHelpers.mysql? - locate = Arel::Nodes::NamedFunction - .new('locate', [quoted_pattern, column]) - insert_in_place = Arel::Nodes::NamedFunction - .new('insert', [column, locate, pattern.size, quoted_replacement]) - - Arel::Nodes::SqlLiteral.new(insert_in_place.to_sql) - else - replace = Arel::Nodes::NamedFunction - .new("regexp_replace", [column, quoted_pattern, quoted_replacement]) - Arel::Nodes::SqlLiteral.new(replace.to_sql) - end + replace = Arel::Nodes::NamedFunction + .new("regexp_replace", [column, quoted_pattern, quoted_replacement]) + Arel::Nodes::SqlLiteral.new(replace.to_sql) end def remove_foreign_key_without_error(*args) diff --git a/lib/mastodon/redis_config.rb b/lib/mastodon/redis_config.rb index c3c8ff800..3f2a8f7c2 100644 --- a/lib/mastodon/redis_config.rb +++ b/lib/mastodon/redis_config.rb @@ -27,6 +27,8 @@ namespace = ENV.fetch('REDIS_NAMESPACE', nil) cache_namespace = namespace ? namespace + '_cache' : 'cache' REDIS_CACHE_PARAMS = { + driver: :hiredis, + url: ENV['REDIS_URL'], expires_in: 10.minutes, namespace: cache_namespace, }.freeze diff --git a/lib/sanitize_ext/sanitize_config.rb b/lib/sanitize_ext/sanitize_config.rb new file mode 100644 index 000000000..fed504cf2 --- /dev/null +++ b/lib/sanitize_ext/sanitize_config.rb @@ -0,0 +1,140 @@ +# frozen_string_literal: true + +class Sanitize + module Config + HTTP_PROTOCOLS = %w( + http + https + ).freeze + + LINK_PROTOCOLS = %w( + http + https + dat + dweb + ipfs + ipns + ssb + gemini + gopher + xmpp + magnet + gemini + ).freeze + + CLASS_WHITELIST_TRANSFORMER = lambda do |env| + node = env[:node] + class_list = node['class']&.split(/[\t\n\f\r ]/) + + return unless class_list + + class_list.keep_if do |e| + next true if /^(h|p|u|dt|e)-/.match?(e) # microformats classes + next true if /^(mention|hashtag)$/.match?(e) # semantic classes + next true if /^(ellipsis|invisible)$/.match?(e) # link formatting classes + end + + node['class'] = class_list.join(' ') + end + + IMG_TAG_TRANSFORMER = lambda do |env| + node = env[:node] + + return unless env[:node_name] == 'img' + + node.name = 'a' + + node['href'] = node['src'] + if node['alt'].present? + node.content = "[🖼 #{node['alt']}]" + else + url = node['href'] + prefix = url.match(/\Ahttps?:\/\/(www\.)?/).to_s + text = url[prefix.length, 30] + text = text + "…" if url[prefix.length..-1].length > 30 + node.content = "[🖼 #{text}]" + end + end + + LINK_REL_TRANSFORMER = lambda do |env| + return unless env[:node_name] == 'a' and env[:node]['href'] + + node = env[:node] + + rel = (node['rel'] || '').split(' ') & ['tag'] + unless env[:config][:outgoing] && TagManager.instance.local_url?(node['href']) + rel += ['nofollow', 'noopener', 'noreferrer'] + end + node['rel'] = rel.join(' ') + end + + UNSUPPORTED_HREF_TRANSFORMER = lambda do |env| + return unless env[:node_name] == 'a' + + current_node = env[:node] + + scheme = begin + if current_node['href'] =~ Sanitize::REGEX_PROTOCOL + Regexp.last_match(1).downcase + else + :relative + end + end + + current_node.replace(current_node.text) unless LINK_PROTOCOLS.include?(scheme) + end + + MASTODON_STRICT ||= freeze_config( + elements: %w(p br span a abbr del pre blockquote code b strong u sub sup i em h1 h2 h3 h4 h5 ul ol li details summary), + + attributes: { + 'a' => %w(href rel class title), + 'span' => %w(class), + 'abbr' => %w(title), + 'blockquote' => %w(cite), + 'ol' => %w(start reversed), + 'li' => %w(value), + }, + + add_attributes: { + 'a' => { + 'target' => '_blank', + }, + }, + + protocols: { + 'a' => { 'href' => LINK_PROTOCOLS }, + 'blockquote' => { 'cite' => LINK_PROTOCOLS }, + }, + + transformers: [ + CLASS_WHITELIST_TRANSFORMER, + IMG_TAG_TRANSFORMER, + UNSUPPORTED_HREF_TRANSFORMER, + LINK_REL_TRANSFORMER, + ] + ) + + MASTODON_OEMBED ||= freeze_config merge( + RELAXED, + elements: RELAXED[:elements] + %w(audio embed iframe source video), + + attributes: merge( + RELAXED[:attributes], + 'audio' => %w(controls), + 'embed' => %w(height src type width), + 'iframe' => %w(allowfullscreen frameborder height scrolling src width), + 'source' => %w(src type), + 'video' => %w(controls height loop width), + 'div' => [:data] + ), + + protocols: merge( + RELAXED[:protocols], + 'embed' => { 'src' => HTTP_PROTOCOLS }, + 'iframe' => { 'src' => HTTP_PROTOCOLS }, + 'source' => { 'src' => HTTP_PROTOCOLS } + ) + ) + end +end diff --git a/lib/tasks/db.rake b/lib/tasks/db.rake index f6c9c7eec..7e6c1c8fc 100644 --- a/lib/tasks/db.rake +++ b/lib/tasks/db.rake @@ -1,36 +1,5 @@ # frozen_string_literal: true -require_relative '../mastodon/snowflake' - -def each_schema_load_environment - # If we're in development, also run this for the test environment. - # This is a somewhat hacky way to do this, so here's why: - # 1. We have to define this before we load the schema, or we won't - # have a timestamp_id function when we get to it in the schema. - # 2. db:setup calls db:schema:load_if_ruby, which calls - # db:schema:load, which we define above as having a prerequisite - # of this task. - # 3. db:schema:load ends up running - # ActiveRecord::Tasks::DatabaseTasks.load_schema_current, which - # calls a private method `each_current_configuration`, which - # explicitly also does the loading for the `test` environment - # if the current environment is `development`, so we end up - # needing to do the same, and we can't even use the same method - # to do it. - - if Rails.env.development? - test_conf = ActiveRecord::Base.configurations['test'] - - if test_conf['database']&.present? - ActiveRecord::Base.establish_connection(:test) - yield - ActiveRecord::Base.establish_connection(Rails.env.to_sym) - end - end - - yield -end - namespace :db do namespace :migrate do desc 'Setup the db or migrate depending on state of db' @@ -50,7 +19,7 @@ namespace :db do task :post_migration_hook do at_exit do - unless %w(C POSIX).include?(ActiveRecord::Base.connection.execute('SELECT datcollate FROM pg_database WHERE datname = current_database();').first['datcollate']) + unless %w(C POSIX).include?(ActiveRecord::Base.connection.select_one('SELECT datcollate FROM pg_database WHERE datname = current_database();')['datcollate']) warn <<~WARNING Your database collation is susceptible to index corruption. (This warning does not indicate that index corruption has occured and can be ignored) @@ -60,30 +29,11 @@ namespace :db do end end - Rake::Task['db:migrate'].enhance(['db:post_migration_hook']) - - # Before we load the schema, define the timestamp_id function. - # Idiomatically, we might do this in a migration, but then it - # wouldn't end up in schema.rb, so we'd need to figure out a way to - # get it in before doing db:setup as well. This is simpler, and - # ensures it's always in place. - Rake::Task['db:schema:load'].enhance ['db:define_timestamp_id'] - - # After we load the schema, make sure we have sequences for each - # table using timestamp IDs. - Rake::Task['db:schema:load'].enhance do - Rake::Task['db:ensure_id_sequences_exist'].invoke - end - - task :define_timestamp_id do - each_schema_load_environment do - Mastodon::Snowflake.define_timestamp_id - end + task :pre_migration_check do + version = ActiveRecord::Base.connection.select_one("SELECT current_setting('server_version_num') AS v")['v'].to_i + abort 'ERROR: This version of Mastodon requires PostgreSQL 9.5 or newer. Please update PostgreSQL before updating Mastodon.' if version < 90_500 end - task :ensure_id_sequences_exist do - each_schema_load_environment do - Mastodon::Snowflake.ensure_id_sequences_exist - end - end + Rake::Task['db:migrate'].enhance(['db:pre_migration_check']) + Rake::Task['db:migrate'].enhance(['db:post_migration_hook']) end diff --git a/lib/tasks/emojis.rake b/lib/tasks/emojis.rake index 01ae95564..c8655cc47 100644 --- a/lib/tasks/emojis.rake +++ b/lib/tasks/emojis.rake @@ -69,7 +69,7 @@ namespace :emojis do end end - existence_maps = grouped_codes.map { |c| c.map { |cc| [cc, File.exist?(Rails.root.join('public', 'emoji', codepoints_to_filename(cc) + '.svg'))] }.to_h } + existence_maps = grouped_codes.map { |c| c.index_with { |cc| File.exist?(Rails.root.join('public', 'emoji', codepoints_to_filename(cc) + '.svg')) } } map = {} existence_maps.each do |group| diff --git a/lib/tasks/mastodon.rake b/lib/tasks/mastodon.rake index 2ad1e778b..72bacb5eb 100644 --- a/lib/tasks/mastodon.rake +++ b/lib/tasks/mastodon.rake @@ -371,18 +371,20 @@ namespace :mastodon do end end - prompt.say "\n" - prompt.say 'The final step is compiling CSS/JS assets.' - prompt.say 'This may take a while and consume a lot of RAM.' + unless using_docker + prompt.say "\n" + prompt.say 'The final step is compiling CSS/JS assets.' + prompt.say 'This may take a while and consume a lot of RAM.' - if prompt.yes?('Compile the assets now?') - prompt.say 'Running `RAILS_ENV=production rails assets:precompile` ...' - prompt.say "\n\n" + if prompt.yes?('Compile the assets now?') + prompt.say 'Running `RAILS_ENV=production rails assets:precompile` ...' + prompt.say "\n\n" - if !system(env.transform_values(&:to_s).merge({ 'RAILS_ENV' => 'production' }), 'rails assets:precompile') - prompt.error 'That failed! Maybe you need swap space?' - else - prompt.say 'Done!' + if !system(env.transform_values(&:to_s).merge({ 'RAILS_ENV' => 'production' }), 'rails assets:precompile') + prompt.error 'That failed! Maybe you need swap space?' + else + prompt.say 'Done!' + end end end |