diff options
| author | Starfall <us@starfall.systems> | 2022-01-31 12:50:14 -0600 |
|---|---|---|
| committer | Starfall <us@starfall.systems> | 2022-01-31 12:50:14 -0600 |
| commit | 17265f47f8f931e70699088dd8bd2a1c7b78112b (patch) | |
| tree | a1dde2630cd8e481cc4c5d047c4af241a251def0 /lib | |
| parent | 129962006c2ebcd195561ac556887dc87d32081c (diff) | |
| parent | d6f3261c6cb810ea4eb6f74b9ee62af0d94cbd52 (diff) | |
Merge branch 'glitchsoc'
Diffstat (limited to 'lib')
24 files changed, 659 insertions, 274 deletions
diff --git a/lib/cli.rb b/lib/cli.rb index 3f1658566..35c00e736 100644 --- a/lib/cli.rb +++ b/lib/cli.rb @@ -13,6 +13,7 @@ require_relative 'mastodon/preview_cards_cli' require_relative 'mastodon/cache_cli' require_relative 'mastodon/upgrade_cli' require_relative 'mastodon/email_domain_blocks_cli' +require_relative 'mastodon/canonical_email_blocks_cli' require_relative 'mastodon/ip_blocks_cli' require_relative 'mastodon/maintenance_cli' require_relative 'mastodon/version' @@ -62,6 +63,9 @@ module Mastodon desc 'ip_blocks SUBCOMMAND ...ARGS', 'Manage IP blocks' subcommand 'ip_blocks', Mastodon::IpBlocksCLI + desc 'canonical_email_blocks SUBCOMMAND ...ARGS', 'Manage canonical e-mail blocks' + subcommand 'canonical_email_blocks', Mastodon::CanonicalEmailBlocksCLI + desc 'maintenance SUBCOMMAND ...ARGS', 'Various maintenance utilities' subcommand 'maintenance', Mastodon::MaintenanceCLI @@ -94,17 +98,22 @@ module Mastodon exit(1) unless prompt.ask('Type in the domain of the server to confirm:', required: true) == Rails.configuration.x.local_domain - prompt.warn('This operation WILL NOT be reversible. It can also take a long time.') - prompt.warn('While the data won\'t be erased locally, the server will be in a BROKEN STATE afterwards.') - prompt.warn('A running Sidekiq process is required. Do not shut it down until queues clear.') + unless options[:dry_run] + prompt.warn('This operation WILL NOT be reversible. It can also take a long time.') + prompt.warn('While the data won\'t be erased locally, the server will be in a BROKEN STATE afterwards.') + prompt.warn('A running Sidekiq process is required. Do not shut it down until queues clear.') - exit(1) if prompt.no?('Are you sure you want to proceed?') + exit(1) if prompt.no?('Are you sure you want to proceed?') + end inboxes = Account.inboxes processed = 0 dry_run = options[:dry_run] ? ' (DRY RUN)' : '' + Setting.registrations_mode = 'none' unless options[:dry_run] + if inboxes.empty? + Account.local.without_suspended.in_batches.update_all(suspended_at: Time.now.utc, suspension_origin: :local) unless options[:dry_run] prompt.ok('It seems like your server has not federated with anything') prompt.ok('You can shut it down and delete it any time') return @@ -112,9 +121,7 @@ module Mastodon prompt.warn('Do NOT interrupt this process...') - Setting.registrations_mode = 'none' - - Account.local.without_suspended.find_each do |account| + delete_account = ->(account) do payload = ActiveModelSerializers::SerializableResource.new( account, serializer: ActivityPub::DeleteActorSerializer, @@ -128,12 +135,15 @@ module Mastodon [json, account.id, inbox_url] end - account.suspend! + account.suspend!(block_email: false) end processed += 1 end + Account.local.without_suspended.find_each { |account| delete_account.call(account) } + Account.local.suspended.joins(:deletion_request).find_each { |account| delete_account.call(account) } + prompt.ok("Queued #{inboxes.size * processed} items into Sidekiq for #{processed} accounts#{dry_run}") prompt.ok('Wait until Sidekiq processes all items, then you can shut everything down and delete the data') rescue TTY::Reader::InputInterrupt diff --git a/lib/mastodon/accounts_cli.rb b/lib/mastodon/accounts_cli.rb index 74162256f..2ef85d0a9 100644 --- a/lib/mastodon/accounts_cli.rb +++ b/lib/mastodon/accounts_cli.rb @@ -54,7 +54,8 @@ module Mastodon option :email, required: true option :confirmed, type: :boolean - option :role, default: 'user' + option :role, default: 'user', enum: %w(user moderator admin) + option :skip_sign_in_token, type: :boolean option :reattach, type: :boolean option :force, type: :boolean desc 'create USERNAME', 'Create a new user' @@ -68,6 +69,9 @@ module Mastodon With the --role option one of "user", "admin" or "moderator" can be supplied. Defaults to "user" + With the --skip-sign-in-token option, you can ensure that + the user is never asked for an e-mailed security code. + With the --reattach option, the new user will be reattached to a given existing username of an old account. If the old account is still in use by someone else, you can supply @@ -77,7 +81,7 @@ module Mastodon def create(username) account = Account.new(username: username) password = SecureRandom.hex - user = User.new(email: options[:email], password: password, agreement: true, approved: true, admin: options[:role] == 'admin', moderator: options[:role] == 'moderator', confirmed_at: options[:confirmed] ? Time.now.utc : nil, bypass_invite_request_check: true) + user = User.new(email: options[:email], password: password, agreement: true, approved: true, admin: options[:role] == 'admin', moderator: options[:role] == 'moderator', confirmed_at: options[:confirmed] ? Time.now.utc : nil, bypass_invite_request_check: true, skip_sign_in_token: options[:skip_sign_in_token]) if options[:reattach] account = Account.find_local(username) || Account.new(username: username) @@ -113,7 +117,7 @@ module Mastodon end end - option :role + option :role, enum: %w(user moderator admin) option :email option :confirm, type: :boolean option :enable, type: :boolean @@ -121,6 +125,7 @@ module Mastodon option :disable_2fa, type: :boolean option :approve, type: :boolean option :reset_password, type: :boolean + option :skip_sign_in_token, type: :boolean desc 'modify USERNAME', 'Modify a user' long_desc <<-LONG_DESC Modify a user account. @@ -142,6 +147,9 @@ module Mastodon With the --reset-password option, the user's password is replaced by a randomly-generated one, printed in the output. + + With the --skip-sign-in-token option, you can ensure that + the user is never asked for an e-mailed security code. LONG_DESC def modify(username) user = Account.find_local(username)&.user @@ -163,6 +171,7 @@ module Mastodon user.disabled = true if options[:disable] user.approved = true if options[:approve] user.otp_required_for_login = false if options[:disable_2fa] + user.skip_sign_in_token = options[:skip_sign_in_token] unless options[:skip_sign_in_token].nil? user.confirm if options[:confirm] if user.save @@ -278,7 +287,7 @@ module Mastodon option :concurrency, type: :numeric, default: 5, aliases: [:c] option :dry_run, type: :boolean - desc 'cull', 'Remove remote accounts that no longer exist' + desc 'cull [DOMAIN...]', 'Remove remote accounts that no longer exist' long_desc <<-LONG_DESC Query every single remote account in the database to determine if it still exists on the origin server, and if it doesn't, @@ -287,19 +296,22 @@ module Mastodon Accounts that have had confirmed activity within the last week are excluded from the checks. LONG_DESC - def cull + def cull(*domains) skip_threshold = 7.days.ago dry_run = options[:dry_run] ? ' (DRY RUN)' : '' skip_domains = Concurrent::Set.new - processed, culled = parallelize_with_progress(Account.remote.where(protocol: :activitypub).partitioned) do |account| + query = Account.remote.where(protocol: :activitypub) + query = query.where(domain: domains) unless domains.empty? + + processed, culled = parallelize_with_progress(query.partitioned) do |account| next if account.updated_at >= skip_threshold || (account.last_webfingered_at.present? && account.last_webfingered_at >= skip_threshold) || skip_domains.include?(account.domain) code = 0 begin code = Request.new(:head, account.uri).perform(&:code) - rescue HTTP::ConnectionError + rescue HTTP::TimeoutError, HTTP::ConnectionError, OpenSSL::SSL::SSLError skip_domains << account.domain end diff --git a/lib/mastodon/canonical_email_blocks_cli.rb b/lib/mastodon/canonical_email_blocks_cli.rb new file mode 100644 index 000000000..64b72e603 --- /dev/null +++ b/lib/mastodon/canonical_email_blocks_cli.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +require 'concurrent' +require_relative '../../config/boot' +require_relative '../../config/environment' +require_relative 'cli_helper' + +module Mastodon + class CanonicalEmailBlocksCLI < Thor + include CLIHelper + + def self.exit_on_failure? + true + end + + desc 'find EMAIL', 'Find a given e-mail address in the canonical e-mail blocks' + long_desc <<-LONG_DESC + When suspending a local user, a hash of a "canonical" version of their e-mail + address is stored to prevent them from signing up again. + + This command can be used to find whether a known email address is blocked, + and if so, which account it was attached to. + LONG_DESC + def find(email) + accts = CanonicalEmailBlock.find_blocks(email).map(&:reference_account).map(&:acct).to_a + if accts.empty? + say("#{email} is not blocked", :yellow) + else + accts.each do |acct| + say(acct, :white) + end + end + end + + desc 'remove EMAIL', 'Remove a canonical e-mail block' + long_desc <<-LONG_DESC + When suspending a local user, a hash of a "canonical" version of their e-mail + address is stored to prevent them from signing up again. + + This command allows removing a canonical email block. + LONG_DESC + def remove(email) + blocks = CanonicalEmailBlock.find_blocks(email) + if blocks.empty? + say("#{email} is not blocked", :yellow) + else + blocks.destroy_all + say("Removed canonical email block for #{email}", :green) + end + end + + private + + def color(processed, failed) + if !processed.zero? && failed.zero? + :green + elsif failed.zero? + :yellow + else + :red + end + end + end +end diff --git a/lib/mastodon/domains_cli.rb b/lib/mastodon/domains_cli.rb index 4ebd8a1e2..a7c78c4a7 100644 --- a/lib/mastodon/domains_cli.rb +++ b/lib/mastodon/domains_cli.rb @@ -17,6 +17,7 @@ module Mastodon option :verbose, type: :boolean, aliases: [:v] option :dry_run, type: :boolean option :limited_federation_mode, type: :boolean + option :by_uri, type: :boolean desc 'purge [DOMAIN...]', 'Remove accounts from a DOMAIN without a trace' long_desc <<-LONG_DESC Remove all accounts from a given DOMAIN without leaving behind any @@ -26,6 +27,12 @@ module Mastodon When the --limited-federation-mode option is given, instead of purging accounts from a single domain, all accounts from domains that have not been explicitly allowed are removed from the database. + + When the --by-uri option is given, DOMAIN is used to match the domain part of actor + URIs rather than the domain part of the webfinger handle. For instance, an account + that has the handle `foo@bar.com` but whose profile is at the URL + `https://mastodon-bar.com/users/foo`, would be purged by either + `tootctl domains purge bar.com` or `tootctl domains purge --by-uri mastodon-bar.com`. LONG_DESC def purge(*domains) dry_run = options[:dry_run] ? ' (DRY RUN)' : '' @@ -34,7 +41,11 @@ module Mastodon if options[:limited_federation_mode] Account.remote.where.not(domain: DomainAllow.pluck(:domain)) elsif !domains.empty? - Account.remote.where(domain: domains) + if options[:by_uri] + domains.map { |domain| Account.remote.where(Account.arel_table[:uri].matches("https://#{domain}/%", false, true)) }.reduce(:or) + else + Account.remote.where(domain: domains) + end else say('No domain(s) given', :red) exit(1) diff --git a/lib/mastodon/maintenance_cli.rb b/lib/mastodon/maintenance_cli.rb index 47e2d78bb..00861df77 100644 --- a/lib/mastodon/maintenance_cli.rb +++ b/lib/mastodon/maintenance_cli.rb @@ -14,7 +14,7 @@ module Mastodon end MIN_SUPPORTED_VERSION = 2019_10_01_213028 - MAX_SUPPORTED_VERSION = 2021_05_26_193025 + MAX_SUPPORTED_VERSION = 2022_01_18_183123 # Stubs to enjoy ActiveRecord queries while not depending on a particular # version of the code/database @@ -84,13 +84,14 @@ module Mastodon owned_classes = [ Status, StatusPin, MediaAttachment, Poll, Report, Tombstone, Favourite, - Follow, FollowRequest, Block, Mute, AccountIdentityProof, + Follow, FollowRequest, Block, Mute, AccountModerationNote, AccountPin, AccountStat, ListAccount, PollVote, Mention ] owned_classes << AccountDeletionRequest if ActiveRecord::Base.connection.table_exists?(:account_deletion_requests) owned_classes << AccountNote if ActiveRecord::Base.connection.table_exists?(:account_notes) owned_classes << FollowRecommendationSuppression if ActiveRecord::Base.connection.table_exists?(:follow_recommendation_suppressions) + owned_classes << AccountIdentityProof if ActiveRecord::Base.connection.table_exists?(:account_identity_proofs) owned_classes.each do |klass| klass.where(account_id: other_account.id).find_each do |record| @@ -139,17 +140,22 @@ module Mastodon @prompt = TTY::Prompt.new if ActiveRecord::Migrator.current_version < MIN_SUPPORTED_VERSION - @prompt.warn 'Your version of the database schema is too old and is not supported by this script.' - @prompt.warn 'Please update to at least Mastodon 3.0.0 before running this script.' + @prompt.error 'Your version of the database schema is too old and is not supported by this script.' + @prompt.error 'Please update to at least Mastodon 3.0.0 before running this script.' exit(1) elsif ActiveRecord::Migrator.current_version > MAX_SUPPORTED_VERSION @prompt.warn 'Your version of the database schema is more recent than this script, this may cause unexpected errors.' - exit(1) unless @prompt.yes?('Continue anyway?') + exit(1) unless @prompt.yes?('Continue anyway? (Yes/No)') + end + + if Sidekiq::ProcessSet.new.any? + @prompt.error 'It seems Sidekiq is running. All Mastodon processes need to be stopped when using this script.' + exit(1) end @prompt.warn 'This task will take a long time to run and is potentially destructive.' @prompt.warn 'Please make sure to stop Mastodon and have a backup.' - exit(1) unless @prompt.yes?('Continue?') + exit(1) unless @prompt.yes?('Continue? (Yes/No)') deduplicate_users! deduplicate_account_domain_blocks! @@ -236,12 +242,14 @@ module Mastodon end end - ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users WHERE remember_token IS NOT NULL GROUP BY remember_token HAVING count(*) > 1").each do |row| - users = User.where(id: row['ids'].split(',')).sort_by(&:updated_at).reverse.drop(1) - @prompt.warn "Unsetting remember token for those accounts: #{users.map(&:account).map(&:acct).join(', ')}" + if ActiveRecord::Migrator.current_version < 20220118183010 + ActiveRecord::Base.connection.select_all("SELECT string_agg(id::text, ',') AS ids FROM users WHERE remember_token IS NOT NULL GROUP BY remember_token HAVING count(*) > 1").each do |row| + users = User.where(id: row['ids'].split(',')).sort_by(&:updated_at).reverse.drop(1) + @prompt.warn "Unsetting remember token for those accounts: #{users.map(&:account).map(&:acct).join(', ')}" - users.each do |user| - user.update!(remember_token: nil) + users.each do |user| + user.update!(remember_token: nil) + end end end @@ -257,7 +265,7 @@ module Mastodon @prompt.say 'Restoring users indexes…' ActiveRecord::Base.connection.add_index :users, ['confirmation_token'], name: 'index_users_on_confirmation_token', unique: true ActiveRecord::Base.connection.add_index :users, ['email'], name: 'index_users_on_email', unique: true - ActiveRecord::Base.connection.add_index :users, ['remember_token'], name: 'index_users_on_remember_token', unique: true + ActiveRecord::Base.connection.add_index :users, ['remember_token'], name: 'index_users_on_remember_token', unique: true if ActiveRecord::Migrator.current_version < 20220118183010 ActiveRecord::Base.connection.add_index :users, ['reset_password_token'], name: 'index_users_on_reset_password_token', unique: true end @@ -274,6 +282,8 @@ module Mastodon end def deduplicate_account_identity_proofs! + return unless ActiveRecord::Base.connection.table_exists?(:account_identity_proofs) + remove_index_if_exists!(:account_identity_proofs, 'index_account_proofs_on_account_and_provider_and_username') @prompt.say 'Removing duplicate account identity proofs…' diff --git a/lib/mastodon/media_cli.rb b/lib/mastodon/media_cli.rb index 59c118500..36ca71844 100644 --- a/lib/mastodon/media_cli.rb +++ b/lib/mastodon/media_cli.rb @@ -230,6 +230,7 @@ module Mastodon processed, aggregate = parallelize_with_progress(scope) do |media_attachment| next if media_attachment.remote_url.blank? || (!options[:force] && media_attachment.file_file_name.present?) + next if DomainBlock.reject_media?(media_attachment.account.domain) unless options[:dry_run] media_attachment.reset_file! diff --git a/lib/mastodon/migration_helpers.rb b/lib/mastodon/migration_helpers.rb index 39a6e0680..5bc903349 100644 --- a/lib/mastodon/migration_helpers.rb +++ b/lib/mastodon/migration_helpers.rb @@ -295,7 +295,7 @@ module Mastodon table = Arel::Table.new(table_name) total = estimate_rows_in_table(table_name).to_i - if total == 0 + if total < 1 count_arel = table.project(Arel.star.count.as('count')) count_arel = yield table, count_arel if block_given? diff --git a/lib/mastodon/search_cli.rb b/lib/mastodon/search_cli.rb index 0126dfcff..6ad9d7b6a 100644 --- a/lib/mastodon/search_cli.rb +++ b/lib/mastodon/search_cli.rb @@ -17,10 +17,11 @@ module Mastodon ].freeze option :concurrency, type: :numeric, default: 2, aliases: [:c], desc: 'Workload will be split between this number of threads' + option :batch_size, type: :numeric, default: 1_000, aliases: [:b], desc: 'Number of records in each batch' option :only, type: :array, enum: %w(accounts tags statuses), desc: 'Only process these indices' - desc 'deploy', 'Create or upgrade ElasticSearch indices and populate them' + desc 'deploy', 'Create or upgrade Elasticsearch indices and populate them' long_desc <<~LONG_DESC - If ElasticSearch is empty, this command will create the necessary indices + If Elasticsearch is empty, this command will create the necessary indices and then import data from the database into those indices. This command will also upgrade indices if the underlying schema has been @@ -35,6 +36,11 @@ module Mastodon exit(1) end + if options[:batch_size] < 1 + say('Cannot run with this batch_size setting, must be at least 1', :red) + exit(1) + end + indices = begin if options[:only] options[:only].map { |str| "#{str.camelize}Index".constantize } @@ -64,11 +70,7 @@ module Mastodon progress.title = 'Estimating workload ' # Estimate the amount of data that has to be imported first - indices.each do |index| - index.types.each do |type| - progress.total = (progress.total || 0) + type.adapter.default_scope.count - end - end + progress.total = indices.sum { |index| index.adapter.default_scope.count } # Now import all the actual data. Mind that unlike chewy:sync, we don't # fetch and compare all record IDs from the database and the index to @@ -77,70 +79,71 @@ module Mastodon # is uneconomical. So we only ever add. indices.each do |index| progress.title = "Importing #{index} " - batch_size = 1_000 + batch_size = options[:batch_size] slice_size = (batch_size / options[:concurrency]).ceil - index.types.each do |type| - type.adapter.default_scope.reorder(nil).find_in_batches(batch_size: batch_size) do |batch| - futures = [] + index.adapter.default_scope.reorder(nil).find_in_batches(batch_size: batch_size) do |batch| + futures = [] - batch.each_slice(slice_size) do |records| - futures << Concurrent::Future.execute(executor: pool) do - begin - if !progress.total.nil? && progress.progress + records.size > progress.total - # The number of items has changed between start and now, - # since there is no good way to predict the final count from - # here, just change the progress bar to an indeterminate one + batch.each_slice(slice_size) do |records| + futures << Concurrent::Future.execute(executor: pool) do + begin + if !progress.total.nil? && progress.progress + records.size > progress.total + # The number of items has changed between start and now, + # since there is no good way to predict the final count from + # here, just change the progress bar to an indeterminate one - progress.total = nil - end + progress.total = nil + end - grouped_records = nil - bulk_body = nil - index_count = 0 - delete_count = 0 + grouped_records = nil + bulk_body = nil + index_count = 0 + delete_count = 0 - ActiveRecord::Base.connection_pool.with_connection do - grouped_records = type.adapter.send(:grouped_objects, records) - bulk_body = Chewy::Type::Import::BulkBuilder.new(type, **grouped_records).bulk_body + ActiveRecord::Base.connection_pool.with_connection do + grouped_records = records.to_a.group_by do |record| + index.adapter.send(:delete_from_index?, record) ? :delete : :to_index end - index_count = grouped_records[:index].size if grouped_records.key?(:index) - delete_count = grouped_records[:delete].size if grouped_records.key?(:delete) - - # The following is an optimization for statuses specifically, since - # we want to de-index statuses that cannot be searched by anybody, - # but can't use Chewy's delete_if logic because it doesn't use - # crutches and our searchable_by logic depends on them - if type == StatusesIndex::Status - bulk_body.map! do |entry| - if entry[:index] && entry.dig(:index, :data, 'searchable_by').blank? - index_count -= 1 - delete_count += 1 - - { delete: entry[:index].except(:data) } - else - entry - end + bulk_body = Chewy::Index::Import::BulkBuilder.new(index, **grouped_records).bulk_body + end + + index_count = grouped_records[:to_index].size if grouped_records.key?(:to_index) + delete_count = grouped_records[:delete].size if grouped_records.key?(:delete) + + # The following is an optimization for statuses specifically, since + # we want to de-index statuses that cannot be searched by anybody, + # but can't use Chewy's delete_if logic because it doesn't use + # crutches and our searchable_by logic depends on them + if index == StatusesIndex + bulk_body.map! do |entry| + if entry[:to_index] && entry.dig(:to_index, :data, 'searchable_by').blank? + index_count -= 1 + delete_count += 1 + + { delete: entry[:to_index].except(:data) } + else + entry end end + end - Chewy::Type::Import::BulkRequest.new(type).perform(bulk_body) + Chewy::Index::Import::BulkRequest.new(index).perform(bulk_body) - progress.progress += records.size + progress.progress += records.size - added.increment(index_count) - removed.increment(delete_count) + added.increment(index_count) + removed.increment(delete_count) - sleep 1 - rescue => e - progress.log pastel.red("Error importing #{index}: #{e}") - end + sleep 1 + rescue => e + progress.log pastel.red("Error importing #{index}: #{e}") end end - - futures.map(&:value) end + + futures.map(&:value) end end diff --git a/lib/mastodon/snowflake.rb b/lib/mastodon/snowflake.rb index 9e5bc7383..fe0dc1722 100644 --- a/lib/mastodon/snowflake.rb +++ b/lib/mastodon/snowflake.rb @@ -84,10 +84,7 @@ module Mastodon::Snowflake -- Take the first two bytes (four hex characters) substr( -- Of the MD5 hash of the data we documented - md5(table_name || - '#{SecureRandom.hex(16)}' || - time_part::text - ), + md5(table_name || '#{SecureRandom.hex(16)}' || time_part::text), 1, 4 ) -- And turn it into a bigint @@ -138,10 +135,11 @@ module Mastodon::Snowflake end end - def id_at(timestamp) - id = timestamp.to_i * 1000 + rand(1000) + def id_at(timestamp, with_random: true) + id = timestamp.to_i * 1000 + id += rand(1000) if with_random id = id << 16 - id += rand(2**16) + id += rand(2**16) if with_random id end diff --git a/lib/mastodon/statuses_cli.rb b/lib/mastodon/statuses_cli.rb index 8a18a3b2f..e273e2614 100644 --- a/lib/mastodon/statuses_cli.rb +++ b/lib/mastodon/statuses_cli.rb @@ -6,6 +6,7 @@ require_relative 'cli_helper' module Mastodon class StatusesCLI < Thor + include CLIHelper include ActionView::Helpers::NumberHelper def self.exit_on_failure? @@ -13,64 +14,213 @@ module Mastodon end option :days, type: :numeric, default: 90 - option :clean_followed, type: :boolean - option :skip_media_remove, type: :boolean + option :batch_size, type: :numeric, default: 1_000, aliases: [:b], desc: 'Number of records in each batch' + option :continue, type: :boolean, default: false, desc: 'If remove is not completed, execute from the previous continuation' + option :clean_followed, type: :boolean, default: false, desc: 'Include the status of remote accounts that are followed by local accounts as candidates for remove' + option :skip_status_remove, type: :boolean, default: false, desc: 'Skip status remove (run only cleanup tasks)' + option :skip_media_remove, type: :boolean, default: false, desc: 'Skip remove orphaned media attachments' + option :compress_database, type: :boolean, default: false, desc: 'Compress database and update the statistics. This option locks the table for a long time, so run it offline' desc 'remove', 'Remove unreferenced statuses' long_desc <<~LONG_DESC Remove statuses that are not referenced by local user activity, such as ones that came from relays, or belonging to users that were once followed by someone locally but no longer are. + It also removes orphaned records and performs additional cleanup tasks + such as updating statistics and recovering disk space. + This is a computationally heavy procedure that creates extra database indices before commencing, and removes them afterward. LONG_DESC def remove + if options[:batch_size] < 1 + say('Cannot run with this batch_size setting, must be at least 1', :red) + exit(1) + end + + remove_statuses + vacuum_and_analyze_statuses + remove_orphans_media_attachments + remove_orphans_conversations + vacuum_and_analyze_conversations + end + + private + + def remove_statuses + return if options[:skip_status_remove] + say('Creating temporary database indices...') - ActiveRecord::Base.connection.add_index(:accounts, :id, name: :index_accounts_local, where: 'domain is null', algorithm: :concurrently) unless ActiveRecord::Base.connection.index_name_exists?(:accounts, :index_accounts_local) - ActiveRecord::Base.connection.add_index(:status_pins, :status_id, name: :index_status_pins_status_id, algorithm: :concurrently) unless ActiveRecord::Base.connection.index_name_exists?(:status_pins, :index_status_pins_status_id) - ActiveRecord::Base.connection.add_index(:media_attachments, :remote_url, name: :index_media_attachments_remote_url, where: 'remote_url is not null', algorithm: :concurrently) unless ActiveRecord::Base.connection.index_name_exists?(:media_attachments, :index_media_attachments_remote_url) + ActiveRecord::Base.connection.add_index(:media_attachments, :remote_url, name: :index_media_attachments_remote_url, where: 'remote_url is not null', algorithm: :concurrently, if_not_exists: true) max_id = Mastodon::Snowflake.id_at(options[:days].days.ago) start_at = Time.now.to_f - say('Beginning removal... This might take a while...') - - scope = Status.remote.where('id < ?', max_id) - # Skip reblogs of local statuses - scope = scope.where('reblog_of_id NOT IN (SELECT statuses1.id FROM statuses AS statuses1 WHERE statuses1.id = statuses.reblog_of_id AND (statuses1.uri IS NULL OR statuses1.local))') - # Skip statuses that are pinned on profiles - scope = scope.where('id NOT IN (SELECT status_pins.status_id FROM status_pins WHERE statuses.id = status_id)') - # Skip statuses that mention local accounts - scope = scope.where('id NOT IN (SELECT mentions.status_id FROM mentions WHERE statuses.id = mentions.status_id AND mentions.account_id IN (SELECT accounts.id FROM accounts WHERE domain IS NULL))') - # Skip statuses which have replies - scope = scope.where('id NOT IN (SELECT statuses1.in_reply_to_id FROM statuses AS statuses1 WHERE statuses.id = statuses1.in_reply_to_id)') - # Skip statuses reblogged by local accounts or with recent boosts - scope = scope.where('id NOT IN (SELECT statuses1.reblog_of_id FROM statuses AS statuses1 WHERE statuses.id = statuses1.reblog_of_id AND (statuses1.uri IS NULL OR statuses1.local OR statuses1.id >= ?))', max_id) - # Skip statuses favourited by local users - scope = scope.where('id NOT IN (SELECT favourites.status_id FROM favourites WHERE statuses.id = favourites.status_id AND favourites.account_id IN (SELECT accounts.id FROM accounts WHERE domain IS NULL))') - # Skip statuses bookmarked by local users - scope = scope.where('id NOT IN (SELECT bookmarks.status_id FROM bookmarks WHERE statuses.id = bookmarks.status_id)') - - unless options[:clean_followed] + unless options[:continue] && ActiveRecord::Base.connection.table_exists?('statuses_to_be_deleted') + ActiveRecord::Base.connection.add_index(:accounts, :id, name: :index_accounts_local, where: 'domain is null', algorithm: :concurrently, if_not_exists: true) + ActiveRecord::Base.connection.add_index(:status_pins, :status_id, name: :index_status_pins_status_id, algorithm: :concurrently, if_not_exists: true) + + say('Extract the deletion target from statuses... This might take a while...') + + ActiveRecord::Base.connection.create_table('statuses_to_be_deleted', force: true) + # Skip accounts followed by local accounts - scope = scope.where('account_id NOT IN (SELECT follows.target_account_id FROM follows WHERE statuses.account_id = follows.target_account_id)') + clean_followed_sql = 'AND NOT EXISTS (SELECT 1 FROM follows WHERE statuses.account_id = follows.target_account_id)' unless options[:clean_followed] + + ActiveRecord::Base.connection.exec_insert(<<-SQL.squish, 'SQL', [[nil, max_id]]) + INSERT INTO statuses_to_be_deleted (id) + SELECT statuses.id FROM statuses WHERE deleted_at IS NULL AND NOT local AND uri IS NOT NULL AND (id < $1) + AND NOT EXISTS (SELECT 1 FROM statuses AS statuses1 WHERE statuses.id = statuses1.in_reply_to_id) + AND NOT EXISTS (SELECT 1 FROM statuses AS statuses1 WHERE statuses1.id = statuses.reblog_of_id AND (statuses1.uri IS NULL OR statuses1.local)) + AND NOT EXISTS (SELECT 1 FROM statuses AS statuses1 WHERE statuses.id = statuses1.reblog_of_id AND (statuses1.uri IS NULL OR statuses1.local OR statuses1.id >= $1)) + AND NOT EXISTS (SELECT 1 FROM status_pins WHERE statuses.id = status_id) + AND NOT EXISTS (SELECT 1 FROM mentions WHERE statuses.id = mentions.status_id AND mentions.account_id IN (SELECT accounts.id FROM accounts WHERE domain IS NULL)) + AND NOT EXISTS (SELECT 1 FROM favourites WHERE statuses.id = favourites.status_id AND favourites.account_id IN (SELECT accounts.id FROM accounts WHERE domain IS NULL)) + AND NOT EXISTS (SELECT 1 FROM bookmarks WHERE statuses.id = bookmarks.status_id AND bookmarks.account_id IN (SELECT accounts.id FROM accounts WHERE domain IS NULL)) + #{clean_followed_sql} + SQL + + say('Removing temporary database indices to restore write performance...') + + ActiveRecord::Base.connection.remove_index(:accounts, name: :index_accounts_local, if_exists: true) + ActiveRecord::Base.connection.remove_index(:status_pins, name: :index_status_pins_status_id, if_exists: true) + end + + say('Beginning statuses removal... This might take a while...') + + klass = Class.new(ApplicationRecord) do |c| + c.table_name = 'statuses_to_be_deleted' + end + + Object.const_set('StatusToBeDeleted', klass) + + scope = StatusToBeDeleted + processed = 0 + removed = 0 + progress = create_progress_bar(scope.count.fdiv(options[:batch_size]).ceil) + + scope.reorder(nil).in_batches(of: options[:batch_size]) do |relation| + ids = relation.pluck(:id) + processed += ids.count + removed += Status.unscoped.where(id: ids).delete_all + progress.increment + end + + progress.stop + + ActiveRecord::Base.connection.drop_table('statuses_to_be_deleted') + + say("Done after #{Time.now.to_f - start_at}s, removed #{removed} out of #{processed} statuses.", :green) + ensure + say('Removing temporary database indices to restore write performance...') + + ActiveRecord::Base.connection.remove_index(:accounts, name: :index_accounts_local, if_exists: true) + ActiveRecord::Base.connection.remove_index(:status_pins, name: :index_status_pins_status_id, if_exists: true) + ActiveRecord::Base.connection.remove_index(:media_attachments, name: :index_media_attachments_remote_url, if_exists: true) + end + + def remove_orphans_media_attachments + return if options[:skip_media_remove] + + start_at = Time.now.to_f + + say('Beginning removal of now-orphaned media attachments to free up disk space...') + + scope = MediaAttachment.reorder(nil).unattached.where('created_at < ?', options[:days].pred.days.ago) + processed = 0 + removed = 0 + progress = create_progress_bar(scope.count) + + scope.find_each do |media_attachment| + media_attachment.destroy! + + removed += 1 + rescue => e + progress.log pastel.red("Error processing #{media_attachment.id}: #{e}") + ensure + progress.increment + processed += 1 end - scope.in_batches.delete_all + progress.stop + + say("Done after #{Time.now.to_f - start_at}s, removed #{removed} out of #{processed} media_attachments.", :green) + end + + def remove_orphans_conversations + start_at = Time.now.to_f + + unless options[:continue] && ActiveRecord::Base.connection.table_exists?('conversations_to_be_deleted') + say('Creating temporary database indices...') + + ActiveRecord::Base.connection.add_index(:statuses, :conversation_id, name: :index_statuses_conversation_id, algorithm: :concurrently, if_not_exists: true) + + say('Extract the deletion target from coversations... This might take a while...') + + ActiveRecord::Base.connection.create_table('conversations_to_be_deleted', force: true) + + ActiveRecord::Base.connection.exec_insert(<<-SQL.squish, 'SQL') + INSERT INTO conversations_to_be_deleted (id) + SELECT id FROM conversations WHERE NOT EXISTS (SELECT 1 FROM statuses WHERE statuses.conversation_id = conversations.id) + SQL - unless options[:skip_media_remove] - say('Beginning removal of now-orphaned media attachments to free up disk space...') - Scheduler::MediaCleanupScheduler.new.perform + say('Removing temporary database indices to restore write performance...') + ActiveRecord::Base.connection.remove_index(:statuses, name: :index_statuses_conversation_id, if_exists: true) end - say("Done after #{Time.now.to_f - start_at}s", :green) + say('Beginning orphans removal... This might take a while...') + + klass = Class.new(ApplicationRecord) do |c| + c.table_name = 'conversations_to_be_deleted' + end + + Object.const_set('ConversationsToBeDeleted', klass) + + scope = ConversationsToBeDeleted + processed = 0 + removed = 0 + progress = create_progress_bar(scope.count.fdiv(options[:batch_size]).ceil) + + scope.in_batches(of: options[:batch_size]) do |relation| + ids = relation.pluck(:id) + processed += ids.count + removed += Conversation.unscoped.where(id: ids).delete_all + progress.increment + end + + progress.stop + + ActiveRecord::Base.connection.drop_table('conversations_to_be_deleted') + + say("Done after #{Time.now.to_f - start_at}s, removed #{removed} out of #{processed} conversations.", :green) ensure say('Removing temporary database indices to restore write performance...') + ActiveRecord::Base.connection.remove_index(:statuses, name: :index_statuses_conversation_id, if_exists: true) + end - ActiveRecord::Base.connection.remove_index(:accounts, name: :index_accounts_local) if ActiveRecord::Base.connection.index_name_exists?(:accounts, :index_accounts_local) - ActiveRecord::Base.connection.remove_index(:status_pins, name: :index_status_pins_status_id) if ActiveRecord::Base.connection.index_name_exists?(:status_pins, :index_status_pins_status_id) - ActiveRecord::Base.connection.remove_index(:media_attachments, name: :index_media_attachments_remote_url) if ActiveRecord::Base.connection.index_name_exists?(:media_attachments, :index_media_attachments_remote_url) + def vacuum_and_analyze_statuses + if options[:compress_database] + say('Run VACUUM FULL ANALYZE to statuses...') + ActiveRecord::Base.connection.execute('VACUUM FULL ANALYZE statuses') + say('Run REINDEX to statuses...') + ActiveRecord::Base.connection.execute('REINDEX TABLE statuses') + else + say('Run ANALYZE to statuses...') + ActiveRecord::Base.connection.execute('ANALYZE statuses') + end + end + + def vacuum_and_analyze_conversations + if options[:compress_database] + say('Run VACUUM FULL ANALYZE to conversations...') + ActiveRecord::Base.connection.execute('VACUUM FULL ANALYZE conversations') + say('Run REINDEX to conversations...') + ActiveRecord::Base.connection.execute('REINDEX TABLE conversations') + else + say('Run ANALYZE to conversations...') + ActiveRecord::Base.connection.execute('ANALYZE conversations') + end end end end diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb index 58786f35b..eb72dd1ed 100644 --- a/lib/mastodon/version.rb +++ b/lib/mastodon/version.rb @@ -13,7 +13,7 @@ module Mastodon end def patch - 1 + 3 end def flags diff --git a/lib/paperclip/attachment_extensions.rb b/lib/paperclip/attachment_extensions.rb index 271f8b603..786f558e9 100644 --- a/lib/paperclip/attachment_extensions.rb +++ b/lib/paperclip/attachment_extensions.rb @@ -6,6 +6,35 @@ module Paperclip instance_read(:meta) end + # monkey-patch to avoid unlinking too avoid unlinking source file too early + # see https://github.com/kreeti/kt-paperclip/issues/64 + def post_process_style(name, style) #:nodoc: + raise "Style #{name} has no processors defined." if style.processors.blank? + + intermediate_files = [] + original = @queued_for_write[:original] + # if we're processing the original, close + unlink the source tempfile + intermediate_files << original if name == :original + + @queued_for_write[name] = style.processors. + inject(original) do |file, processor| + file = Paperclip.processor(processor).make(file, style.processor_options, self) + intermediate_files << file unless file == original + file + end + + unadapted_file = @queued_for_write[name] + @queued_for_write[name] = Paperclip.io_adapters. + for(@queued_for_write[name], @options[:adapter_options]) + unadapted_file.close if unadapted_file.respond_to?(:close) + @queued_for_write[name] + rescue Paperclip::Errors::NotIdentifiedByImageMagickError => e + log("An error was received while processing: #{e.inspect}") + (@errors[:processing] ||= []) << e.message if @options[:whiny] + ensure + unlink_files(intermediate_files) + end + # We overwrite this method to support delayed processing in # Sidekiq. Since we process the original file to reduce disk # usage, and we still want to generate thumbnails straight diff --git a/lib/paperclip/media_type_spoof_detector_extensions.rb b/lib/paperclip/media_type_spoof_detector_extensions.rb deleted file mode 100644 index 43337cc68..000000000 --- a/lib/paperclip/media_type_spoof_detector_extensions.rb +++ /dev/null @@ -1,35 +0,0 @@ -# frozen_string_literal: true - -module Paperclip - module MediaTypeSpoofDetectorExtensions - def mapping_override_mismatch? - !Array(mapped_content_type).include?(calculated_content_type) && !Array(mapped_content_type).include?(type_from_mime_magic) - end - - def calculated_media_type_from_mime_magic - @calculated_media_type_from_mime_magic ||= type_from_mime_magic.split('/').first - end - - def calculated_type_mismatch? - !media_types_from_name.include?(calculated_media_type) && !media_types_from_name.include?(calculated_media_type_from_mime_magic) - end - - def type_from_mime_magic - @type_from_mime_magic ||= begin - begin - File.open(@file.path) do |file| - MimeMagic.by_magic(file)&.type || '' - end - rescue Errno::ENOENT - '' - end - end - end - - def type_from_file_command - @type_from_file_command ||= FileCommandContentTypeDetector.new(@file.path).detect - end - end -end - -Paperclip::MediaTypeSpoofDetector.prepend(Paperclip::MediaTypeSpoofDetectorExtensions) diff --git a/lib/paperclip/response_with_limit_adapter.rb b/lib/paperclip/response_with_limit_adapter.rb index 17a2abd25..deb89717a 100644 --- a/lib/paperclip/response_with_limit_adapter.rb +++ b/lib/paperclip/response_with_limit_adapter.rb @@ -17,9 +17,9 @@ module Paperclip def cache_current_values @original_filename = filename_from_content_disposition.presence || filename_from_path.presence || 'data' - @size = @target.response.content_length @tempfile = copy_to_tempfile(@target) @content_type = ContentTypeDetector.new(@tempfile.path).detect + @size = File.size(@tempfile) end def copy_to_tempfile(source) diff --git a/lib/paperclip/schema_extensions.rb b/lib/paperclip/schema_extensions.rb deleted file mode 100644 index 8d065676a..000000000 --- a/lib/paperclip/schema_extensions.rb +++ /dev/null @@ -1,37 +0,0 @@ -# frozen_string_literal: true - -# Monkey-patch various Paperclip methods for Ruby 3.0 compatibility - -module Paperclip - module Schema - module StatementsExtensions - def add_attachment(table_name, *attachment_names) - raise ArgumentError, 'Please specify attachment name in your add_attachment call in your migration.' if attachment_names.empty? - - options = attachment_names.extract_options! - - attachment_names.each do |attachment_name| - COLUMNS.each_pair do |column_name, column_type| - column_options = options.merge(options[column_name.to_sym] || {}) - add_column(table_name, "#{attachment_name}_#{column_name}", column_type, **column_options) - end - end - end - end - - module TableDefinitionExtensions - def attachment(*attachment_names) - options = attachment_names.extract_options! - attachment_names.each do |attachment_name| - COLUMNS.each_pair do |column_name, column_type| - column_options = options.merge(options[column_name.to_sym] || {}) - column("#{attachment_name}_#{column_name}", column_type, **column_options) - end - end - end - end - end -end - -Paperclip::Schema::Statements.prepend(Paperclip::Schema::StatementsExtensions) -Paperclip::Schema::TableDefinition.prepend(Paperclip::Schema::TableDefinitionExtensions) diff --git a/lib/paperclip/storage_extensions.rb b/lib/paperclip/storage_extensions.rb new file mode 100644 index 000000000..95c35641e --- /dev/null +++ b/lib/paperclip/storage_extensions.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +# Some S3-compatible providers might not actually be compatible with some APIs +# used by kt-paperclip, see https://github.com/mastodon/mastodon/issues/16822 +if ENV['S3_ENABLED'] == 'true' && ENV['S3_FORCE_SINGLE_REQUEST'] == 'true' + module Paperclip + module Storage + module S3Extensions + def copy_to_local_file(style, local_dest_path) + log("copying #{path(style)} to local file #{local_dest_path}") + s3_object(style).download_file(local_dest_path, { mode: 'single_request' }) + rescue Aws::Errors::ServiceError => e + warn("#{e} - cannot copy #{path(style)} to local file #{local_dest_path}") + false + end + end + end + end + + Paperclip::Storage::S3.prepend(Paperclip::Storage::S3Extensions) +end diff --git a/lib/paperclip/transcoder.rb b/lib/paperclip/transcoder.rb index e99704086..ec1305038 100644 --- a/lib/paperclip/transcoder.rb +++ b/lib/paperclip/transcoder.rb @@ -19,7 +19,7 @@ module Paperclip metadata = VideoMetadataExtractor.new(@file.path) unless metadata.valid? - log("Unsupported file #{@file.path}") + Paperclip.log("Unsupported file #{@file.path}") return File.open(@file.path) end diff --git a/lib/paperclip/url_generator_extensions.rb b/lib/paperclip/url_generator_extensions.rb index e1d6df2c2..a2cf5929a 100644 --- a/lib/paperclip/url_generator_extensions.rb +++ b/lib/paperclip/url_generator_extensions.rb @@ -2,16 +2,6 @@ module Paperclip module UrlGeneratorExtensions - # Monkey-patch Paperclip to use Addressable::URI's normalization instead - # of the long-deprecated URI.esacpe - def escape_url(url) - if url.respond_to?(:escape) - url.escape - else - Addressable::URI.parse(url).normalize.to_str.gsub(escape_regex) { |m| "%#{m.ord.to_s(16).upcase}" } - end - end - def for_as_default(style_name) attachment_options[:interpolator].interpolate(default_url, @attachment, style_name) end diff --git a/lib/paperclip/validation_extensions.rb b/lib/paperclip/validation_extensions.rb deleted file mode 100644 index 0df0434f6..000000000 --- a/lib/paperclip/validation_extensions.rb +++ /dev/null @@ -1,58 +0,0 @@ -# frozen_string_literal: true - -# Monkey-patch various Paperclip validators for Ruby 3.0 compatibility - -module Paperclip - module Validators - module AttachmentSizeValidatorExtensions - def validate_each(record, attr_name, _value) - base_attr_name = attr_name - attr_name = "#{attr_name}_file_size".to_sym - value = record.send(:read_attribute_for_validation, attr_name) - - if value.present? - options.slice(*Paperclip::Validators::AttachmentSizeValidator::AVAILABLE_CHECKS).each do |option, option_value| - option_value = option_value.call(record) if option_value.is_a?(Proc) - option_value = extract_option_value(option, option_value) - - next if value.send(Paperclip::Validators::AttachmentSizeValidator::CHECKS[option], option_value) - - error_message_key = options[:in] ? :in_between : option - [attr_name, base_attr_name].each do |error_attr_name| - record.errors.add(error_attr_name, error_message_key, **filtered_options(value).merge( - min: min_value_in_human_size(record), - max: max_value_in_human_size(record), - count: human_size(option_value) - )) - end - end - end - end - end - - module AttachmentContentTypeValidatorExtensions - def mark_invalid(record, attribute, types) - record.errors.add attribute, :invalid, **options.merge({ types: types.join(', ') }) - end - end - - module AttachmentPresenceValidatorExtensions - def validate_each(record, attribute, _value) - if record.send("#{attribute}_file_name").blank? - record.errors.add(attribute, :blank, **options) - end - end - end - - module AttachmentFileNameValidatorExtensions - def mark_invalid(record, attribute, patterns) - record.errors.add attribute, :invalid, options.merge({ names: patterns.join(', ') }) - end - end - end -end - -Paperclip::Validators::AttachmentSizeValidator.prepend(Paperclip::Validators::AttachmentSizeValidatorExtensions) -Paperclip::Validators::AttachmentContentTypeValidator.prepend(Paperclip::Validators::AttachmentContentTypeValidatorExtensions) -Paperclip::Validators::AttachmentPresenceValidator.prepend(Paperclip::Validators::AttachmentPresenceValidatorExtensions) -Paperclip::Validators::AttachmentFileNameValidator.prepend(Paperclip::Validators::AttachmentFileNameValidatorExtensions) diff --git a/lib/sidekiq_error_handler.rb b/lib/sidekiq_error_handler.rb new file mode 100644 index 000000000..358afd540 --- /dev/null +++ b/lib/sidekiq_error_handler.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +class SidekiqErrorHandler + BACKTRACE_LIMIT = 3 + + def call(*) + yield + rescue Mastodon::HostValidationError + # Do not retry + rescue => e + limit_backtrace_and_raise(e) + ensure + socket = Thread.current[:statsd_socket] + socket&.close + Thread.current[:statsd_socket] = nil + end + + private + + def limit_backtrace_and_raise(exception) + exception.set_backtrace(exception.backtrace.first(BACKTRACE_LIMIT)) + raise exception + end +end diff --git a/lib/tasks/db.rake b/lib/tasks/db.rake index b2a0d61de..a6b8c74cd 100644 --- a/lib/tasks/db.rake +++ b/lib/tasks/db.rake @@ -21,8 +21,8 @@ namespace :db do at_exit do unless %w(C POSIX).include?(ActiveRecord::Base.connection.select_one('SELECT datcollate FROM pg_database WHERE datname = current_database();')['datcollate']) warn <<~WARNING - Your database collation is susceptible to index corruption. - (This warning does not indicate that index corruption has occurred and can be ignored) + Your database collation may be susceptible to index corruption. + (This warning does not indicate that index corruption has occurred, and it can be ignored if you've previously checked for index corruption) (To learn more, visit: https://docs.joinmastodon.org/admin/troubleshooting/index-corruption/) WARNING end diff --git a/lib/tasks/mastodon.rake b/lib/tasks/mastodon.rake index 72bacb5eb..a89af6778 100644 --- a/lib/tasks/mastodon.rake +++ b/lib/tasks/mastodon.rake @@ -333,8 +333,12 @@ namespace :mastodon do prompt.say 'This configuration will be written to .env.production' if prompt.yes?('Save configuration?') + incompatible_syntax = false + env_contents = env.each_pair.map do |key, value| if value.is_a?(String) && value =~ /[\s\#\\"]/ + incompatible_syntax = true + if value =~ /[']/ value = value.to_s.gsub(/[\\"\$]/) { |x| "\\#{x}" } "#{key}=\"#{value}\"" @@ -346,12 +350,19 @@ namespace :mastodon do end end.join("\n") - File.write(Rails.root.join('.env.production'), "# Generated with mastodon:setup on #{Time.now.utc}\n\n" + env_contents + "\n") + generated_header = "# Generated with mastodon:setup on #{Time.now.utc}\n\n".dup + + if incompatible_syntax + generated_header << "# Some variables in this file will be interpreted differently whether you are\n" + generated_header << "# using docker-compose or not.\n\n" + end + + File.write(Rails.root.join('.env.production'), "#{generated_header}#{env_contents}\n") if using_docker prompt.ok 'Below is your configuration, save it to an .env.production file outside Docker:' prompt.say "\n" - prompt.say File.read(Rails.root.join('.env.production')) + prompt.say "#{generated_header}#{env.each_pair.map { |key, value| "#{key}=#{value}" }.join("\n")}" prompt.say "\n" prompt.ok 'It is also saved within this container so you can proceed with this wizard.' end @@ -430,7 +441,7 @@ namespace :mastodon do namespace :webpush do desc 'Generate VAPID key' - task generate_vapid_key: :environment do + task :generate_vapid_key do vapid_key = Webpush.generate_key puts "VAPID_PRIVATE_KEY=#{vapid_key.private_key}" puts "VAPID_PUBLIC_KEY=#{vapid_key.public_key}" diff --git a/lib/tasks/repo.rake b/lib/tasks/repo.rake index 86c358a94..bbf7f20ee 100644 --- a/lib/tasks/repo.rake +++ b/lib/tasks/repo.rake @@ -1,6 +1,6 @@ # frozen_string_literal: true -REPOSITORY_NAME = 'tootsuite/mastodon' +REPOSITORY_NAME = 'mastodon/mastodon' namespace :repo do desc 'Generate the AUTHORS.md file' @@ -34,7 +34,7 @@ namespace :repo do file << <<~FOOTER - This document is provided for informational purposes only. Since it is only updated once per release, the version you are looking at may be currently out of date. To see the full list of contributors, consider looking at the [git history](https://github.com/tootsuite/mastodon/graphs/contributors) instead. + This document is provided for informational purposes only. Since it is only updated once per release, the version you are looking at may be currently out of date. To see the full list of contributors, consider looking at the [git history](https://github.com/mastodon/mastodon/graphs/contributors) instead. FOOTER end @@ -96,7 +96,7 @@ namespace :repo do end.uniq.compact missing_available_locales = locales_in_files - I18n.available_locales - missing_locale_names = I18n.available_locales.reject { |locale| SettingsHelper::HUMAN_LOCALES.key?(locale) } + missing_locale_names = I18n.available_locales.reject { |locale| LanguagesHelper::HUMAN_LOCALES.key?(locale) } critical = false diff --git a/lib/tasks/tests.rake b/lib/tasks/tests.rake new file mode 100644 index 000000000..0f38b50e3 --- /dev/null +++ b/lib/tasks/tests.rake @@ -0,0 +1,181 @@ +# frozen_string_literal: true + +namespace :tests do + namespace :migrations do + desc 'Populate the database with test data for 2.0.0' + task populate_v2: :environment do + admin_key = OpenSSL::PKey::RSA.new(2048) + user_key = OpenSSL::PKey::RSA.new(2048) + remote_key = OpenSSL::PKey::RSA.new(2048) + remote_key2 = OpenSSL::PKey::RSA.new(2048) + remote_key3 = OpenSSL::PKey::RSA.new(2048) + admin_private_key = ActiveRecord::Base.connection.quote(admin_key.to_pem) + admin_public_key = ActiveRecord::Base.connection.quote(admin_key.public_key.to_pem) + user_private_key = ActiveRecord::Base.connection.quote(user_key.to_pem) + user_public_key = ActiveRecord::Base.connection.quote(user_key.public_key.to_pem) + remote_public_key = ActiveRecord::Base.connection.quote(remote_key.public_key.to_pem) + remote_public_key2 = ActiveRecord::Base.connection.quote(remote_key2.public_key.to_pem) + remote_public_key_ap = ActiveRecord::Base.connection.quote(remote_key3.public_key.to_pem) + local_domain = ActiveRecord::Base.connection.quote(Rails.configuration.x.local_domain) + + ActiveRecord::Base.connection.execute(<<~SQL) + -- accounts + + INSERT INTO "accounts" + (id, username, domain, private_key, public_key, created_at, updated_at) + VALUES + (1, 'admin', NULL, #{admin_private_key}, #{admin_public_key}, now(), now()), + (2, 'user', NULL, #{user_private_key}, #{user_public_key}, now(), now()); + + INSERT INTO "accounts" + (id, username, domain, private_key, public_key, created_at, updated_at, remote_url, salmon_url) + VALUES + (3, 'remote', 'remote.com', NULL, #{remote_public_key}, now(), now(), + 'https://remote.com/@remote', 'https://remote.com/salmon/1'), + (4, 'Remote', 'remote.com', NULL, #{remote_public_key}, now(), now(), + 'https://remote.com/@Remote', 'https://remote.com/salmon/1'), + (5, 'REMOTE', 'Remote.com', NULL, #{remote_public_key2}, now(), now(), + 'https://remote.com/stale/@REMOTE', 'https://remote.com/stale/salmon/1'); + + INSERT INTO "accounts" + (id, username, domain, private_key, public_key, created_at, updated_at, protocol, inbox_url, outbox_url, followers_url) + VALUES + (6, 'bob', 'activitypub.com', NULL, #{remote_public_key_ap}, now(), now(), + 1, 'https://activitypub.com/users/bob/inbox', 'https://activitypub.com/users/bob/outbox', 'https://activitypub.com/users/bob/followers'); + + INSERT INTO "accounts" + (id, username, domain, private_key, public_key, created_at, updated_at) + VALUES + (7, 'user', #{local_domain}, #{user_private_key}, #{user_public_key}, now(), now()), + (8, 'pt_user', NULL, #{user_private_key}, #{user_public_key}, now(), now()); + + -- users + + INSERT INTO "users" + (id, account_id, email, created_at, updated_at, admin) + VALUES + (1, 1, 'admin@localhost', now(), now(), true), + (2, 2, 'user@localhost', now(), now(), false); + + INSERT INTO "users" + (id, account_id, email, created_at, updated_at, admin, locale) + VALUES + (3, 7, 'ptuser@localhost', now(), now(), false, 'pt'); + + -- statuses + + INSERT INTO "statuses" + (id, account_id, text, created_at, updated_at) + VALUES + (1, 1, 'test', now(), now()), + (2, 1, '@remote@remote.com hello', now(), now()), + (3, 1, '@Remote@remote.com hello', now(), now()), + (4, 1, '@REMOTE@remote.com hello', now(), now()); + + INSERT INTO "statuses" + (id, account_id, text, created_at, updated_at, uri, local) + VALUES + (5, 1, 'activitypub status', now(), now(), 'https://localhost/users/admin/statuses/4', true); + + INSERT INTO "statuses" + (id, account_id, text, created_at, updated_at) + VALUES + (6, 3, 'test', now(), now()); + + INSERT INTO "statuses" + (id, account_id, text, created_at, updated_at, in_reply_to_id, in_reply_to_account_id) + VALUES + (7, 4, '@admin hello', now(), now(), 3, 1); + + INSERT INTO "statuses" + (id, account_id, text, created_at, updated_at) + VALUES + (8, 5, 'test', now(), now()); + + INSERT INTO "statuses" + (id, account_id, reblog_of_id, created_at, updated_at) + VALUES + (9, 1, 2, now(), now()); + + -- mentions (from previous statuses) + + INSERT INTO "mentions" + (status_id, account_id, created_at, updated_at) + VALUES + (2, 3, now(), now()), + (3, 4, now(), now()), + (4, 5, now(), now()); + + -- stream entries + + INSERT INTO "stream_entries" + (activity_id, account_id, activity_type, created_at, updated_at) + VALUES + (1, 1, 'status', now(), now()), + (2, 1, 'status', now(), now()), + (3, 1, 'status', now(), now()), + (4, 1, 'status', now(), now()), + (5, 1, 'status', now(), now()), + (6, 3, 'status', now(), now()), + (7, 4, 'status', now(), now()), + (8, 5, 'status', now(), now()), + (9, 1, 'status', now(), now()); + + + -- custom emoji + + INSERT INTO "custom_emojis" + (shortcode, created_at, updated_at) + VALUES + ('test', now(), now()), + ('Test', now(), now()), + ('blobcat', now(), now()); + + INSERT INTO "custom_emojis" + (shortcode, domain, uri, created_at, updated_at) + VALUES + ('blobcat', 'remote.org', 'https://remote.org/emoji/blobcat', now(), now()), + ('blobcat', 'Remote.org', 'https://remote.org/emoji/blobcat', now(), now()), + ('Blobcat', 'remote.org', 'https://remote.org/emoji/Blobcat', now(), now()); + + -- favourites + + INSERT INTO "favourites" + (account_id, status_id, created_at, updated_at) + VALUES + (1, 1, now(), now()), + (1, 7, now(), now()), + (4, 1, now(), now()), + (3, 1, now(), now()), + (5, 1, now(), now()); + + -- pinned statuses + + INSERT INTO "status_pins" + (account_id, status_id, created_at, updated_at) + VALUES + (1, 1, now(), now()), + (3, 6, now(), now()), + (4, 7, now(), now()); + + -- follows + + INSERT INTO "follows" + (account_id, target_account_id, created_at, updated_at) + VALUES + (1, 5, now(), now()), + (6, 2, now(), now()), + (5, 2, now(), now()), + (6, 1, now(), now()); + + -- follow requests + + INSERT INTO "follow_requests" + (account_id, target_account_id, created_at, updated_at) + VALUES + (2, 5, now(), now()), + (5, 1, now(), now()); + SQL + end + end +end |