From 23752639b20abd26eefb199172da9fc3ceae172e Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Thu, 23 Aug 2018 23:30:57 +0200 Subject: Bump version to 2.5.0rc1 (#8356) --- lib/mastodon/version.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb index 093485111..a490478cd 100644 --- a/lib/mastodon/version.rb +++ b/lib/mastodon/version.rb @@ -9,11 +9,11 @@ module Mastodon end def minor - 4 + 5 end def patch - 3 + 0 end def pre @@ -21,7 +21,7 @@ module Mastodon end def flags - '' + 'rc1' end def to_a -- cgit From 793eea29823a44fd4950f87898ecf0ff3b49351d Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sat, 25 Aug 2018 13:25:39 +0200 Subject: Add improved CLI interface for removing remote media (#8411) ./bin/tootctl media remove --days 7 --background Make the old rake task point to it --- app/workers/maintenance/destroy_media_worker.rb | 2 +- .../maintenance/redownload_account_media_worker.rb | 2 +- app/workers/maintenance/uncache_media_worker.rb | 2 +- bin/tootctl | 4 ++ lib/cli.rb | 11 +++++ lib/mastodon/media_cli.rb | 47 ++++++++++++++++++++++ lib/tasks/mastodon.rake | 11 ++--- 7 files changed, 68 insertions(+), 11 deletions(-) create mode 100755 bin/tootctl create mode 100644 lib/cli.rb create mode 100644 lib/mastodon/media_cli.rb (limited to 'lib') diff --git a/app/workers/maintenance/destroy_media_worker.rb b/app/workers/maintenance/destroy_media_worker.rb index 5f052983b..cde33d6d7 100644 --- a/app/workers/maintenance/destroy_media_worker.rb +++ b/app/workers/maintenance/destroy_media_worker.rb @@ -6,7 +6,7 @@ class Maintenance::DestroyMediaWorker sidekiq_options queue: 'pull' def perform(media_attachment_id) - media = MediaAttachment.find(media_attachment_id) + media = media_attachment_id.is_a?(MediaAttachment) ? media_attachment_id : MediaAttachment.find(media_attachment_id) media.destroy rescue ActiveRecord::RecordNotFound true diff --git a/app/workers/maintenance/redownload_account_media_worker.rb b/app/workers/maintenance/redownload_account_media_worker.rb index fc26815f2..6afbe6e19 100644 --- a/app/workers/maintenance/redownload_account_media_worker.rb +++ b/app/workers/maintenance/redownload_account_media_worker.rb @@ -6,7 +6,7 @@ class Maintenance::RedownloadAccountMediaWorker sidekiq_options queue: 'pull', retry: false def perform(account_id) - account = Account.find(account_id) + account = account_id.is_a?(Account) ? account_id : Account.find(account_id) account.reset_avatar! account.reset_header! account.save diff --git a/app/workers/maintenance/uncache_media_worker.rb b/app/workers/maintenance/uncache_media_worker.rb index 2d1a670a7..4bc62ef75 100644 --- a/app/workers/maintenance/uncache_media_worker.rb +++ b/app/workers/maintenance/uncache_media_worker.rb @@ -6,7 +6,7 @@ class Maintenance::UncacheMediaWorker sidekiq_options queue: 'pull' def perform(media_attachment_id) - media = MediaAttachment.find(media_attachment_id) + media = media_attachment_id.is_a?(MediaAttachment) ? media_attachment_id : MediaAttachment.find(media_attachment_id) return if media.file.blank? diff --git a/bin/tootctl b/bin/tootctl new file mode 100755 index 000000000..2fe02523a --- /dev/null +++ b/bin/tootctl @@ -0,0 +1,4 @@ +#!/usr/bin/env ruby +APP_PATH = File.expand_path('../config/application', __dir__) +require_relative '../lib/cli' +Mastodon::CLI.start(ARGV) diff --git a/lib/cli.rb b/lib/cli.rb new file mode 100644 index 000000000..7e82806b6 --- /dev/null +++ b/lib/cli.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +require 'thor' +require_relative 'mastodon/media_cli' + +module Mastodon + class CLI < Thor + desc 'media SUBCOMMAND ...ARGS', 'manage media files' + subcommand 'media', Mastodon::MediaCLI + end +end diff --git a/lib/mastodon/media_cli.rb b/lib/mastodon/media_cli.rb new file mode 100644 index 000000000..cc6ad07d9 --- /dev/null +++ b/lib/mastodon/media_cli.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +require_relative '../../config/boot' +require_relative '../../config/environment' + +# rubocop:disable Rails/Output + +module Mastodon + class MediaCLI < Thor + option :days, type: :numeric, default: 7 + option :background, type: :boolean, default: false + desc 'remove', 'remove remote media files' + long_desc <<-DESC + Removes locally cached copies of media attachments from other servers. + + The --days option specifies how old media attachments have to be before + they are removed. It defaults to 7 days. + + With the --background option, instead of deleting the files sequentially, + they will be queued into Sidekiq and the command will exit as soon as + possible. In Sidekiq they will be processed with higher concurrency, but + it may impact other operations of the Mastodon server, and it may overload + the underlying file storage. + DESC + def remove + time_ago = options[:days].days.ago + queued = 0 + + MediaAttachment.where.not(remote_url: '').where.not(file_file_name: nil).where('created_at < ?', time_ago).select(:id).reorder(nil).find_in_batches do |media_attachments| + if options[:background] + queued += media_attachments.size + Maintenance::UncacheMediaWorker.push_bulk(media_attachments.map(&:id)) + else + media_attachments.each do |m| + Maintenance::UncacheMediaWorker.new.perform(m) + print '.' + end + end + end + + puts + puts "Scheduled the deletion of #{queued} media attachments" if options[:background] + end + end +end + +# rubocop:enable Rails/Output diff --git a/lib/tasks/mastodon.rake b/lib/tasks/mastodon.rake index 191ce634c..9ce39335d 100644 --- a/lib/tasks/mastodon.rake +++ b/lib/tasks/mastodon.rake @@ -512,14 +512,9 @@ namespace :mastodon do desc 'Remove cached remote media attachments that are older than NUM_DAYS. By default 7 (week)' task remove_remote: :environment do - time_ago = ENV.fetch('NUM_DAYS') { 7 }.to_i.days.ago - nb_media_attachments = 0 - - MediaAttachment.where.not(remote_url: '').where.not(file_file_name: nil).where('created_at < ?', time_ago).select(:id).reorder(nil).find_in_batches do |media_attachments| - nb_media_attachments += media_attachments.length - Maintenance::UncacheMediaWorker.push_bulk(media_attachments.map(&:id)) - end - puts "Scheduled the deletion of #{nb_media_attachments} media attachments" + require_relative '../mastodon/media_cli' + cli = Mastodon::MediaCLI.new([], days: ENV['NUM_DAYS'] || 7) + cli.invoke(:remove) end desc 'Set unknown attachment type for remote-only attachments' -- cgit From 2bba6e582d4e8dcf119177e36f1e7fcf3873aeb6 Mon Sep 17 00:00:00 2001 From: M Somerville Date: Sat, 25 Aug 2018 12:27:08 +0100 Subject: Rename S3_CLOUDFRONT_HOST to S3_ALIAS_HOST. (#8423) Still check for S3_CLOUDFRONT_HOST for existing installs. --- .env.nanobox | 4 ++-- .env.production.sample | 4 ++-- config/initializers/paperclip.rb | 4 ++-- config/webpack/production.js | 4 ++-- lib/tasks/mastodon.rake | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/.env.nanobox b/.env.nanobox index 8e0af6a8a..b60b6ee68 100644 --- a/.env.nanobox +++ b/.env.nanobox @@ -136,8 +136,8 @@ SMTP_FROM_ADDRESS=notifications@${APP_NAME}.nanoapp.io # Defaults to 60 seconds. Set to 0 to disable # SWIFT_CACHE_TTL= -# Optional alias for S3 if you want to use Cloudfront or Cloudflare in front -# S3_CLOUDFRONT_HOST= +# Optional alias for S3 (e.g. to serve files on a custom domain, possibly using Cloudfront or Cloudflare) +# S3_ALIAS_HOST= # Streaming API integration # STREAMING_API_BASE_URL= diff --git a/.env.production.sample b/.env.production.sample index 349daedd8..d1164efdc 100644 --- a/.env.production.sample +++ b/.env.production.sample @@ -134,8 +134,8 @@ SMTP_FROM_ADDRESS=notifications@example.com # Defaults to 60 seconds. Set to 0 to disable # SWIFT_CACHE_TTL= -# Optional alias for S3 if you want to use Cloudfront or Cloudflare in front -# S3_CLOUDFRONT_HOST= +# Optional alias for S3 (e.g. to serve files on a custom domain, possibly using Cloudfront or Cloudflare) +# S3_ALIAS_HOST= # Streaming API integration # STREAMING_API_BASE_URL= diff --git a/config/initializers/paperclip.rb b/config/initializers/paperclip.rb index 59ab9b9a1..df0205879 100644 --- a/config/initializers/paperclip.rb +++ b/config/initializers/paperclip.rb @@ -47,10 +47,10 @@ if ENV['S3_ENABLED'] == 'true' Paperclip::Attachment.default_options[:url] = ':s3_path_url' end - if ENV.has_key?('S3_CLOUDFRONT_HOST') + if ENV.has_key?('S3_ALIAS_HOST') || ENV.has_key?('S3_CLOUDFRONT_HOST') Paperclip::Attachment.default_options.merge!( url: ':s3_alias_url', - s3_host_alias: ENV['S3_CLOUDFRONT_HOST'] + s3_host_alias: ENV['S3_ALIAS_HOST'] || ENV['S3_CLOUDFRONT_HOST'] ) end elsif ENV['SWIFT_ENABLED'] == 'true' diff --git a/config/webpack/production.js b/config/webpack/production.js index 4966807a1..27a78108b 100644 --- a/config/webpack/production.js +++ b/config/webpack/production.js @@ -23,8 +23,8 @@ try { let attachmentHost; if (process.env.S3_ENABLED === 'true') { - if (process.env.S3_CLOUDFRONT_HOST) { - attachmentHost = process.env.S3_CLOUDFRONT_HOST; + if (process.env.S3_ALIAS_HOST || process.env.S3_CLOUDFRONT_HOST) { + attachmentHost = process.env.S3_ALIAS_HOST || process.env.S3_CLOUDFRONT_HOST; } else { attachmentHost = process.env.S3_HOSTNAME || `s3-${process.env.S3_REGION || 'us-east-1'}.amazonaws.com`; } diff --git a/lib/tasks/mastodon.rake b/lib/tasks/mastodon.rake index 9ce39335d..519c21785 100644 --- a/lib/tasks/mastodon.rake +++ b/lib/tasks/mastodon.rake @@ -222,7 +222,7 @@ namespace :mastodon do end if prompt.yes?('Do you want to access the uploaded files from your own domain?') - env['S3_CLOUDFRONT_HOST'] = prompt.ask('Domain for uploaded files:') do |q| + env['S3_ALIAS_HOST'] = prompt.ask('Domain for uploaded files:') do |q| q.required true q.default "files.#{env['LOCAL_DOMAIN']}" q.modify :strip -- cgit From b378b4c5c7e99de2c14e39e2733a745689de8ad1 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sun, 26 Aug 2018 16:53:06 +0200 Subject: Add CLI interface for importing custom emoji (#8437) bin/tootctl emoji import PATH_TO_TAR Fix #8435 --- lib/cli.rb | 4 +++ lib/mastodon/cli_helper.rb | 8 +++++ lib/mastodon/emoji_cli.rb | 81 ++++++++++++++++++++++++++++++++++++++++++++++ lib/mastodon/media_cli.rb | 18 ++++++++--- 4 files changed, 106 insertions(+), 5 deletions(-) create mode 100644 lib/mastodon/cli_helper.rb create mode 100644 lib/mastodon/emoji_cli.rb (limited to 'lib') diff --git a/lib/cli.rb b/lib/cli.rb index 7e82806b6..c7dae0276 100644 --- a/lib/cli.rb +++ b/lib/cli.rb @@ -2,10 +2,14 @@ require 'thor' require_relative 'mastodon/media_cli' +require_relative 'mastodon/emoji_cli' module Mastodon class CLI < Thor desc 'media SUBCOMMAND ...ARGS', 'manage media files' subcommand 'media', Mastodon::MediaCLI + + desc 'emoji SUBCOMMAND ...ARGS', 'manage custom emoji' + subcommand 'emoji', Mastodon::EmojiCLI end end diff --git a/lib/mastodon/cli_helper.rb b/lib/mastodon/cli_helper.rb new file mode 100644 index 000000000..8c4d9731c --- /dev/null +++ b/lib/mastodon/cli_helper.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +dev_null = Logger.new('/dev/null') + +Rails.logger = dev_null +ActiveRecord::Base.logger = dev_null +HttpLog.configuration.logger = dev_null +Paperclip.options[:log] = false diff --git a/lib/mastodon/emoji_cli.rb b/lib/mastodon/emoji_cli.rb new file mode 100644 index 000000000..71f8b2cc7 --- /dev/null +++ b/lib/mastodon/emoji_cli.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +require 'rubygems/package' +require_relative '../../config/boot' +require_relative '../../config/environment' +require_relative 'cli_helper' + +# rubocop:disable Rails/Output + +module Mastodon + class EmojiCLI < Thor + option :prefix + option :suffix + option :overwrite, type: :boolean + option :unlisted, type: :boolean + desc 'import PATH', 'import emoji from a TAR archive at PATH' + long_desc <<-LONG_DESC + Imports custom emoji from a TAR archive specified by PATH. + + Existing emoji will be skipped unless the --overwrite option + is provided, in which case they will be overwritten. + + With the --prefix option, a prefix can be added to all + generated shortcodes. Likewise, the --suffix option controls + the suffix of all shortcodes. + + With the --unlisted option, the processed emoji will not be + visible in the emoji picker (but still usable via other means) + LONG_DESC + def import(path) + imported = 0 + skipped = 0 + failed = 0 + + Gem::Package::TarReader.new(Zlib::GzipReader.open(path)) do |tar| + tar.each do |entry| + next unless entry.file? && entry.full_name.end_with?('.png') + + shortcode = [options[:prefix], File.basename(entry.full_name, '.*'), options[:suffix]].compact.join + custom_emoji = CustomEmoji.local.find_by(shortcode: shortcode) + + if custom_emoji && !options[:overwrite] + skipped += 1 + next + end + + custom_emoji ||= CustomEmoji.new(shortcode: shortcode, domain: nil) + custom_emoji.image = StringIO.new(entry.read) + custom_emoji.image_file_name = File.basename(entry.full_name) + custom_emoji.visible_in_picker = !options[:unlisted] + + if custom_emoji.save + imported += 1 + else + failed += 1 + say('Failure/Error: ', :red) + say(entry.full_name) + say(' ' + custom_emoji.errors[:image].join(', '), :red) + end + end + end + + puts + say("Imported #{imported}, skipped #{skipped}, failed to import #{failed}", color(imported, skipped, failed)) + end + + private + + def color(green, _yellow, red) + if !green.zero? && red.zero? + :green + elsif red.zero? + :yellow + else + :red + end + end + end +end + +# rubocop:enable Rails/Output diff --git a/lib/mastodon/media_cli.rb b/lib/mastodon/media_cli.rb index cc6ad07d9..00bd662f4 100644 --- a/lib/mastodon/media_cli.rb +++ b/lib/mastodon/media_cli.rb @@ -2,6 +2,7 @@ require_relative '../../config/boot' require_relative '../../config/environment' +require_relative 'cli_helper' # rubocop:disable Rails/Output @@ -23,8 +24,9 @@ module Mastodon the underlying file storage. DESC def remove - time_ago = options[:days].days.ago - queued = 0 + time_ago = options[:days].days.ago + queued = 0 + processed = 0 MediaAttachment.where.not(remote_url: '').where.not(file_file_name: nil).where('created_at < ?', time_ago).select(:id).reorder(nil).find_in_batches do |media_attachments| if options[:background] @@ -33,13 +35,19 @@ module Mastodon else media_attachments.each do |m| Maintenance::UncacheMediaWorker.new.perform(m) - print '.' + say('.', :green, false) + processed += 1 end end end - puts - puts "Scheduled the deletion of #{queued} media attachments" if options[:background] + say + + if options[:background] + say("Scheduled the deletion of #{queued} media attachments", :green) + else + say("Removed #{processed} media attachments", :green) + end end end end -- cgit From b9c378848544b8691f9bc12ce718b818ad89dfaf Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sun, 26 Aug 2018 17:19:47 +0200 Subject: Fix NUM_DAYS in rake mastodon:media:remove_remote and print warning (#8462) Remove old deprecated rake tasks --- lib/tasks/mastodon.rake | 41 ++--------------------------------------- 1 file changed, 2 insertions(+), 39 deletions(-) (limited to 'lib') diff --git a/lib/tasks/mastodon.rake b/lib/tasks/mastodon.rake index 519c21785..7455478b6 100644 --- a/lib/tasks/mastodon.rake +++ b/lib/tasks/mastodon.rake @@ -394,12 +394,6 @@ namespace :mastodon do end end - desc 'Execute daily tasks (deprecated)' - task :daily do - # No-op - # All of these tasks are now executed via sidekiq-scheduler - end - desc 'Turn a user into an admin, identified by the USERNAME environment variable' task make_admin: :environment do include RoutingHelper @@ -494,12 +488,6 @@ namespace :mastodon do end namespace :media do - desc 'Removes media attachments that have not been assigned to any status for longer than a day (deprecated)' - task clear: :environment do - # No-op - # This task is now executed via sidekiq-scheduler - end - desc 'Remove media attachments attributed to silenced accounts' task remove_silenced: :environment do nb_media_attachments = 0 @@ -512,8 +500,9 @@ namespace :mastodon do desc 'Remove cached remote media attachments that are older than NUM_DAYS. By default 7 (week)' task remove_remote: :environment do + puts 'Please use `./bin/tootctl media remove --help` directly'.colorize(:yellow) require_relative '../mastodon/media_cli' - cli = Mastodon::MediaCLI.new([], days: ENV['NUM_DAYS'] || 7) + cli = Mastodon::MediaCLI.new([], days: (ENV['NUM_DAYS'] || 7).to_i) cli.invoke(:remove) end @@ -543,21 +532,9 @@ namespace :mastodon do task clear: :environment do Pubsubhubbub::UnsubscribeWorker.push_bulk(Account.remote.without_followers.where.not(subscription_expires_at: nil).pluck(:id)) end - - desc 'Re-subscribes to soon expiring PuSH subscriptions (deprecated)' - task refresh: :environment do - # No-op - # This task is now executed via sidekiq-scheduler - end end namespace :feeds do - desc 'Clear timelines of inactive users (deprecated)' - task clear: :environment do - # No-op - # This task is now executed via sidekiq-scheduler - end - desc 'Clear all timelines without regenerating them' task clear_all: :environment do Redis.current.keys('feed:*').each { |key| Redis.current.del(key) } @@ -571,21 +548,7 @@ namespace :mastodon do end end - namespace :emails do - desc 'Send out digest e-mails (deprecated)' - task digest: :environment do - # No-op - # This task is now executed via sidekiq-scheduler - end - end - namespace :users do - desc 'Clear out unconfirmed users (deprecated)' - task clear: :environment do - # No-op - # This task is now executed via sidekiq-scheduler - end - desc 'List e-mails of all admin users' task admins: :environment do puts 'Admin user emails:' -- cgit From da13fa50215c5153ee199eb8e70e7fa030523a00 Mon Sep 17 00:00:00 2001 From: Quint Guvernator Date: Sun, 26 Aug 2018 13:22:46 -0400 Subject: Fix low-hanging rubocop gripes (#8458) * rubocop: quit being so picky * rubocop: miscellany * rubocop: prefer present to blank --- .rubocop.yml | 6 +++++ .../api/v1/lists/accounts_controller.rb | 2 +- app/controllers/api/v1/lists_controller.rb | 2 +- app/lib/feed_manager.rb | 2 +- app/mailers/user_mailer.rb | 2 +- app/models/concerns/expireable.rb | 2 +- app/models/user.rb | 4 ++-- lib/tasks/db.rake | 2 +- lib/tasks/mastodon.rake | 28 ++++++++++------------ 9 files changed, 26 insertions(+), 24 deletions(-) (limited to 'lib') diff --git a/.rubocop.yml b/.rubocop.yml index 4f9e09b43..4ba5903bd 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -62,6 +62,9 @@ Metrics/ParameterLists: Metrics/PerceivedComplexity: Max: 20 +Naming/MemoizedInstanceVariableName: + Enabled: false + Rails: Enabled: true @@ -71,6 +74,9 @@ Rails/HasAndBelongsToMany: Rails/SkipsModelValidations: Enabled: false +Rails/HttpStatus: + Enabled: false + Style/ClassAndModuleChildren: Enabled: false diff --git a/app/controllers/api/v1/lists/accounts_controller.rb b/app/controllers/api/v1/lists/accounts_controller.rb index 19de56732..ec4477034 100644 --- a/app/controllers/api/v1/lists/accounts_controller.rb +++ b/app/controllers/api/v1/lists/accounts_controller.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true class Api::V1::Lists::AccountsController < Api::BaseController - before_action -> { doorkeeper_authorize! :read, :'read:lists' }, only: [:show] + before_action -> { doorkeeper_authorize! :read, :'read:lists' }, only: [:show] before_action -> { doorkeeper_authorize! :write, :'write:lists' }, except: [:show] before_action :require_user! diff --git a/app/controllers/api/v1/lists_controller.rb b/app/controllers/api/v1/lists_controller.rb index b42b8b971..054172bee 100644 --- a/app/controllers/api/v1/lists_controller.rb +++ b/app/controllers/api/v1/lists_controller.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true class Api::V1::ListsController < Api::BaseController - before_action -> { doorkeeper_authorize! :read, :'read:lists' }, only: [:index, :show] + before_action -> { doorkeeper_authorize! :read, :'read:lists' }, only: [:index, :show] before_action -> { doorkeeper_authorize! :write, :'write:lists' }, except: [:index, :show] before_action :require_user! diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb index 14cba70dc..b59a9f1cd 100644 --- a/app/lib/feed_manager.rb +++ b/app/lib/feed_manager.rb @@ -288,7 +288,7 @@ class FeedManager # remains in the set. We could pick a random element, but this # set should generally be small, and it seems ideal to show the # oldest potential such reblog. - other_reblog = redis.smembers(reblog_set_key).map(&:to_i).sort.first + other_reblog = redis.smembers(reblog_set_key).map(&:to_i).min redis.zadd(timeline_key, other_reblog, other_reblog) if other_reblog diff --git a/app/mailers/user_mailer.rb b/app/mailers/user_mailer.rb index 9848c34a2..aa76b4dfe 100644 --- a/app/mailers/user_mailer.rb +++ b/app/mailers/user_mailer.rb @@ -16,7 +16,7 @@ class UserMailer < Devise::Mailer return if @resource.disabled? I18n.with_locale(@resource.locale || I18n.default_locale) do - mail to: @resource.unconfirmed_email.blank? ? @resource.email : @resource.unconfirmed_email, + mail to: @resource.unconfirmed_email.presence || @resource.email, subject: I18n.t(@resource.pending_reconfirmation? ? 'devise.mailer.reconfirmation_instructions.subject' : 'devise.mailer.confirmation_instructions.subject', instance: @instance), template_name: @resource.pending_reconfirmation? ? 'reconfirmation_instructions' : 'confirmation_instructions' end diff --git a/app/models/concerns/expireable.rb b/app/models/concerns/expireable.rb index 444ccdfdb..2c0631476 100644 --- a/app/models/concerns/expireable.rb +++ b/app/models/concerns/expireable.rb @@ -9,7 +9,7 @@ module Expireable attr_reader :expires_in def expires_in=(interval) - self.expires_at = interval.to_i.seconds.from_now unless interval.blank? + self.expires_at = interval.to_i.seconds.from_now if interval.present? @expires_in = interval end diff --git a/app/models/user.rb b/app/models/user.rb index 75b7e9e7c..25f77ed14 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -98,7 +98,7 @@ class User < ApplicationRecord :reduce_motion, :system_font_ui, :noindex, :theme, :display_sensitive_media, :hide_network, :default_language, to: :settings, prefix: :setting, allow_nil: false - attr_accessor :invite_code + attr_reader :invite_code def pam_conflict(_) # block pam login tries on traditional account @@ -258,7 +258,7 @@ class User < ApplicationRecord end def invite_code=(code) - self.invite = Invite.find_by(code: code) unless code.blank? + self.invite = Invite.find_by(code: code) if code.present? @invite_code = code end diff --git a/lib/tasks/db.rake b/lib/tasks/db.rake index 32039c31d..b76e90131 100644 --- a/lib/tasks/db.rake +++ b/lib/tasks/db.rake @@ -18,7 +18,7 @@ def each_schema_load_environment # needing to do the same, and we can't even use the same method # to do it. - if Rails.env == 'development' + if Rails.env.development? test_conf = ActiveRecord::Base.configurations['test'] if test_conf['database']&.present? diff --git a/lib/tasks/mastodon.rake b/lib/tasks/mastodon.rake index 7455478b6..649a22a0b 100644 --- a/lib/tasks/mastodon.rake +++ b/lib/tasks/mastodon.rake @@ -280,14 +280,14 @@ namespace :mastodon do begin ActionMailer::Base.smtp_settings = { - :port => env['SMTP_PORT'], - :address => env['SMTP_SERVER'], - :user_name => env['SMTP_LOGIN'].presence, - :password => env['SMTP_PASSWORD'].presence, - :domain => env['LOCAL_DOMAIN'], - :authentication => env['SMTP_AUTH_METHOD'] == 'none' ? nil : env['SMTP_AUTH_METHOD'] || :plain, - :openssl_verify_mode => env['SMTP_OPENSSL_VERIFY_MODE'], - :enable_starttls_auto => true, + port: env['SMTP_PORT'], + address: env['SMTP_SERVER'], + user_name: env['SMTP_LOGIN'].presence, + password: env['SMTP_PASSWORD'].presence, + domain: env['LOCAL_DOMAIN'], + authentication: env['SMTP_AUTH_METHOD'] == 'none' ? nil : env['SMTP_AUTH_METHOD'] || :plain, + openssl_verify_mode: env['SMTP_OPENSSL_VERIFY_MODE'], + enable_starttls_auto: true, } ActionMailer::Base.default_options = { @@ -326,13 +326,11 @@ namespace :mastodon do if prompt.yes?('Prepare the database now?') prompt.say 'Running `RAILS_ENV=production rails db:setup` ...' - prompt.say "\n" + prompt.say "\n\n" if cmd.run!({ RAILS_ENV: 'production', SAFETY_ASSURED: 1 }, :rails, 'db:setup').failure? - prompt.say "\n" prompt.error 'That failed! Perhaps your configuration is not right' else - prompt.say "\n" prompt.ok 'Done!' end end @@ -343,13 +341,11 @@ namespace :mastodon do if prompt.yes?('Compile the assets now?') prompt.say 'Running `RAILS_ENV=production rails assets:precompile` ...' - prompt.say "\n" + prompt.say "\n\n" if cmd.run!({ RAILS_ENV: 'production' }, :rails, 'assets:precompile').failure? - prompt.say "\n" prompt.error 'That failed! Maybe you need swap space?' else - prompt.say "\n" prompt.say 'Done!' end end @@ -715,10 +711,10 @@ namespace :mastodon do pastel = Pastel.new duplicate_masters.each do |account| - puts pastel.yellow("First of their name: ") + pastel.bold(account.username) + " (#{admin_account_url(account.id)})" + puts pastel.yellow('First of their name: ') + pastel.bold(account.username) + " (#{admin_account_url(account.id)})" Account.where('lower(username) = ?', account.username.downcase).where.not(id: account.id).each do |duplicate| - puts " " + pastel.red("Duplicate: ") + admin_account_url(duplicate.id) + puts ' ' + pastel.red('Duplicate: ') + admin_account_url(duplicate.id) end end end -- cgit From cabdbb7f9c1df8007749d07a2e186bb3ad35f62b Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sun, 26 Aug 2018 20:21:03 +0200 Subject: Add CLI task for rotating keys (#8466) * If an Update is signed with known key, skip re-following procedure Because it means the remote actor did *not* lose their database * Add CLI method for rotating keys bin/tootctl accounts rotate [USERNAME] Generates a new RSA key per account and sends out an Update activity signed with the old key. * Key rotation: Space out Update fan-outs every 5 minutes per 1000 accounts * Skip suspended accounts in key rotation --- app/lib/activitypub/activity/update.rb | 2 +- app/lib/activitypub/linked_data_signature.rb | 5 +- app/lib/request.rb | 5 +- .../activitypub/process_account_service.rb | 5 +- app/workers/activitypub/delivery_worker.rb | 5 +- .../activitypub/update_distribution_worker.rb | 5 +- lib/cli.rb | 9 ++-- lib/mastodon/accounts_cli.rb | 55 ++++++++++++++++++++++ lib/mastodon/emoji_cli.rb | 2 +- lib/mastodon/media_cli.rb | 2 +- 10 files changed, 79 insertions(+), 16 deletions(-) create mode 100644 lib/mastodon/accounts_cli.rb (limited to 'lib') diff --git a/app/lib/activitypub/activity/update.rb b/app/lib/activitypub/activity/update.rb index aa5907f03..6eebc3b5c 100644 --- a/app/lib/activitypub/activity/update.rb +++ b/app/lib/activitypub/activity/update.rb @@ -11,6 +11,6 @@ class ActivityPub::Activity::Update < ActivityPub::Activity def update_account return if @account.uri != object_uri - ActivityPub::ProcessAccountService.new.call(@account.username, @account.domain, @object) + ActivityPub::ProcessAccountService.new.call(@account.username, @account.domain, @object, signed_with_known_key: true) end end diff --git a/app/lib/activitypub/linked_data_signature.rb b/app/lib/activitypub/linked_data_signature.rb index 16142a6ff..f52a8f406 100644 --- a/app/lib/activitypub/linked_data_signature.rb +++ b/app/lib/activitypub/linked_data_signature.rb @@ -32,7 +32,7 @@ class ActivityPub::LinkedDataSignature end end - def sign!(creator) + def sign!(creator, sign_with: nil) options = { 'type' => 'RsaSignature2017', 'creator' => [ActivityPub::TagManager.instance.uri_for(creator), '#main-key'].join, @@ -42,8 +42,9 @@ class ActivityPub::LinkedDataSignature options_hash = hash(options.without('type', 'id', 'signatureValue').merge('@context' => CONTEXT)) document_hash = hash(@json.without('signature')) to_be_signed = options_hash + document_hash + keypair = sign_with.present? ? OpenSSL::PKey::RSA.new(sign_with) : creator.keypair - signature = Base64.strict_encode64(creator.keypair.sign(OpenSSL::Digest::SHA256.new, to_be_signed)) + signature = Base64.strict_encode64(keypair.sign(OpenSSL::Digest::SHA256.new, to_be_signed)) @json.merge('signature' => options.merge('signatureValue' => signature)) end diff --git a/app/lib/request.rb b/app/lib/request.rb index 576ed23ca..21bdaa700 100644 --- a/app/lib/request.rb +++ b/app/lib/request.rb @@ -22,10 +22,11 @@ class Request set_digest! if options.key?(:body) end - def on_behalf_of(account, key_id_format = :acct) + def on_behalf_of(account, key_id_format = :acct, sign_with: nil) raise ArgumentError unless account.local? @account = account + @keypair = sign_with.present? ? OpenSSL::PKey::RSA.new(sign_with) : @account.keypair @key_id_format = key_id_format self @@ -70,7 +71,7 @@ class Request def signature algorithm = 'rsa-sha256' - signature = Base64.strict_encode64(@account.keypair.sign(OpenSSL::Digest::SHA256.new, signed_string)) + signature = Base64.strict_encode64(@keypair.sign(OpenSSL::Digest::SHA256.new, signed_string)) "keyId=\"#{key_id}\",algorithm=\"#{algorithm}\",headers=\"#{signed_headers}\",signature=\"#{signature}\"" end diff --git a/app/services/activitypub/process_account_service.rb b/app/services/activitypub/process_account_service.rb index ac19bf933..670a0e4d6 100644 --- a/app/services/activitypub/process_account_service.rb +++ b/app/services/activitypub/process_account_service.rb @@ -5,9 +5,10 @@ class ActivityPub::ProcessAccountService < BaseService # Should be called with confirmed valid JSON # and WebFinger-resolved username and domain - def call(username, domain, json) + def call(username, domain, json, options = {}) return if json['inbox'].blank? || unsupported_uri_scheme?(json['id']) + @options = options @json = json @uri = @json['id'] @username = username @@ -31,7 +32,7 @@ class ActivityPub::ProcessAccountService < BaseService return if @account.nil? after_protocol_change! if protocol_changed? - after_key_change! if key_changed? + after_key_change! if key_changed? && !@options[:signed_with_known_key] check_featured_collection! if @account.featured_collection_url.present? @account diff --git a/app/workers/activitypub/delivery_worker.rb b/app/workers/activitypub/delivery_worker.rb index 323a9f85b..adbb496d9 100644 --- a/app/workers/activitypub/delivery_worker.rb +++ b/app/workers/activitypub/delivery_worker.rb @@ -10,7 +10,8 @@ class ActivityPub::DeliveryWorker HEADERS = { 'Content-Type' => 'application/activity+json' }.freeze - def perform(json, source_account_id, inbox_url) + def perform(json, source_account_id, inbox_url, options = {}) + @options = options.with_indifferent_access @json = json @source_account = Account.find(source_account_id) @inbox_url = inbox_url @@ -27,7 +28,7 @@ class ActivityPub::DeliveryWorker def build_request request = Request.new(:post, @inbox_url, body: @json) - request.on_behalf_of(@source_account, :uri) + request.on_behalf_of(@source_account, :uri, sign_with: @options[:sign_with]) request.add_headers(HEADERS) end diff --git a/app/workers/activitypub/update_distribution_worker.rb b/app/workers/activitypub/update_distribution_worker.rb index bbda69305..b9e5ff064 100644 --- a/app/workers/activitypub/update_distribution_worker.rb +++ b/app/workers/activitypub/update_distribution_worker.rb @@ -5,7 +5,8 @@ class ActivityPub::UpdateDistributionWorker sidekiq_options queue: 'push' - def perform(account_id) + def perform(account_id, options = {}) + @options = options.with_indifferent_access @account = Account.find(account_id) ActivityPub::DeliveryWorker.push_bulk(inboxes) do |inbox_url| @@ -26,7 +27,7 @@ class ActivityPub::UpdateDistributionWorker end def signed_payload - @signed_payload ||= Oj.dump(ActivityPub::LinkedDataSignature.new(payload).sign!(@account)) + @signed_payload ||= Oj.dump(ActivityPub::LinkedDataSignature.new(payload).sign!(@account, sign_with: @options[:sign_with])) end def payload diff --git a/lib/cli.rb b/lib/cli.rb index c7dae0276..60bff4147 100644 --- a/lib/cli.rb +++ b/lib/cli.rb @@ -3,13 +3,16 @@ require 'thor' require_relative 'mastodon/media_cli' require_relative 'mastodon/emoji_cli' - +require_relative 'mastodon/accounts_cli' module Mastodon class CLI < Thor - desc 'media SUBCOMMAND ...ARGS', 'manage media files' + desc 'media SUBCOMMAND ...ARGS', 'Manage media files' subcommand 'media', Mastodon::MediaCLI - desc 'emoji SUBCOMMAND ...ARGS', 'manage custom emoji' + desc 'emoji SUBCOMMAND ...ARGS', 'Manage custom emoji' subcommand 'emoji', Mastodon::EmojiCLI + + desc 'accounts SUBCOMMAND ...ARGS', 'Manage accounts' + subcommand 'accounts', Mastodon::AccountsCLI end end diff --git a/lib/mastodon/accounts_cli.rb b/lib/mastodon/accounts_cli.rb new file mode 100644 index 000000000..83b69549d --- /dev/null +++ b/lib/mastodon/accounts_cli.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +require 'rubygems/package' +require_relative '../../config/boot' +require_relative '../../config/environment' +require_relative 'cli_helper' + +module Mastodon + class AccountsCLI < Thor + option :all, type: :boolean + desc 'rotate [USERNAME]', 'Generate and broadcast new keys' + long_desc <<-LONG_DESC + Generate and broadcast new RSA keys as part of security + maintenance. + + With the --all option, all local accounts will be subject + to the rotation. Otherwise, and by default, only a single + account specified by the USERNAME argument will be + processed. + LONG_DESC + def rotate(username = nil) + if options[:all] + processed = 0 + delay = 0 + + Account.local.without_suspended.find_in_batches do |accounts| + accounts.each do |account| + rotate_keys_for_account(account, delay) + processed += 1 + say('.', :green, false) + end + + delay += 5.minutes + end + + say + say("OK, rotated keys for #{processed} accounts", :green) + elsif username.present? + rotate_keys_for_account(Account.find_local(username)) + say('OK', :green) + else + say('No account(s) given', :red) + end + end + + private + + def rotate_keys_for_account(account, delay = 0) + old_key = account.private_key + new_key = OpenSSL::PKey::RSA.new(2048).to_pem + account.update(private_key: new_key) + ActivityPub::UpdateDistributionWorker.perform_in(delay, account.id, sign_with: old_key) + end + end +end diff --git a/lib/mastodon/emoji_cli.rb b/lib/mastodon/emoji_cli.rb index 71f8b2cc7..0a773c771 100644 --- a/lib/mastodon/emoji_cli.rb +++ b/lib/mastodon/emoji_cli.rb @@ -13,7 +13,7 @@ module Mastodon option :suffix option :overwrite, type: :boolean option :unlisted, type: :boolean - desc 'import PATH', 'import emoji from a TAR archive at PATH' + desc 'import PATH', 'Import emoji from a TAR archive at PATH' long_desc <<-LONG_DESC Imports custom emoji from a TAR archive specified by PATH. diff --git a/lib/mastodon/media_cli.rb b/lib/mastodon/media_cli.rb index 00bd662f4..ee28270da 100644 --- a/lib/mastodon/media_cli.rb +++ b/lib/mastodon/media_cli.rb @@ -10,7 +10,7 @@ module Mastodon class MediaCLI < Thor option :days, type: :numeric, default: 7 option :background, type: :boolean, default: false - desc 'remove', 'remove remote media files' + desc 'remove', 'Remove remote media files' long_desc <<-DESC Removes locally cached copies of media attachments from other servers. -- cgit