From 793eea29823a44fd4950f87898ecf0ff3b49351d Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sat, 25 Aug 2018 13:25:39 +0200 Subject: Add improved CLI interface for removing remote media (#8411) ./bin/tootctl media remove --days 7 --background Make the old rake task point to it --- lib/cli.rb | 11 +++++++++++ lib/mastodon/media_cli.rb | 47 +++++++++++++++++++++++++++++++++++++++++++++++ lib/tasks/mastodon.rake | 11 +++-------- 3 files changed, 61 insertions(+), 8 deletions(-) create mode 100644 lib/cli.rb create mode 100644 lib/mastodon/media_cli.rb (limited to 'lib') diff --git a/lib/cli.rb b/lib/cli.rb new file mode 100644 index 000000000..7e82806b6 --- /dev/null +++ b/lib/cli.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +require 'thor' +require_relative 'mastodon/media_cli' + +module Mastodon + class CLI < Thor + desc 'media SUBCOMMAND ...ARGS', 'manage media files' + subcommand 'media', Mastodon::MediaCLI + end +end diff --git a/lib/mastodon/media_cli.rb b/lib/mastodon/media_cli.rb new file mode 100644 index 000000000..cc6ad07d9 --- /dev/null +++ b/lib/mastodon/media_cli.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +require_relative '../../config/boot' +require_relative '../../config/environment' + +# rubocop:disable Rails/Output + +module Mastodon + class MediaCLI < Thor + option :days, type: :numeric, default: 7 + option :background, type: :boolean, default: false + desc 'remove', 'remove remote media files' + long_desc <<-DESC + Removes locally cached copies of media attachments from other servers. + + The --days option specifies how old media attachments have to be before + they are removed. It defaults to 7 days. + + With the --background option, instead of deleting the files sequentially, + they will be queued into Sidekiq and the command will exit as soon as + possible. In Sidekiq they will be processed with higher concurrency, but + it may impact other operations of the Mastodon server, and it may overload + the underlying file storage. + DESC + def remove + time_ago = options[:days].days.ago + queued = 0 + + MediaAttachment.where.not(remote_url: '').where.not(file_file_name: nil).where('created_at < ?', time_ago).select(:id).reorder(nil).find_in_batches do |media_attachments| + if options[:background] + queued += media_attachments.size + Maintenance::UncacheMediaWorker.push_bulk(media_attachments.map(&:id)) + else + media_attachments.each do |m| + Maintenance::UncacheMediaWorker.new.perform(m) + print '.' + end + end + end + + puts + puts "Scheduled the deletion of #{queued} media attachments" if options[:background] + end + end +end + +# rubocop:enable Rails/Output diff --git a/lib/tasks/mastodon.rake b/lib/tasks/mastodon.rake index 191ce634c..9ce39335d 100644 --- a/lib/tasks/mastodon.rake +++ b/lib/tasks/mastodon.rake @@ -512,14 +512,9 @@ namespace :mastodon do desc 'Remove cached remote media attachments that are older than NUM_DAYS. By default 7 (week)' task remove_remote: :environment do - time_ago = ENV.fetch('NUM_DAYS') { 7 }.to_i.days.ago - nb_media_attachments = 0 - - MediaAttachment.where.not(remote_url: '').where.not(file_file_name: nil).where('created_at < ?', time_ago).select(:id).reorder(nil).find_in_batches do |media_attachments| - nb_media_attachments += media_attachments.length - Maintenance::UncacheMediaWorker.push_bulk(media_attachments.map(&:id)) - end - puts "Scheduled the deletion of #{nb_media_attachments} media attachments" + require_relative '../mastodon/media_cli' + cli = Mastodon::MediaCLI.new([], days: ENV['NUM_DAYS'] || 7) + cli.invoke(:remove) end desc 'Set unknown attachment type for remote-only attachments' -- cgit From 2bba6e582d4e8dcf119177e36f1e7fcf3873aeb6 Mon Sep 17 00:00:00 2001 From: M Somerville Date: Sat, 25 Aug 2018 12:27:08 +0100 Subject: Rename S3_CLOUDFRONT_HOST to S3_ALIAS_HOST. (#8423) Still check for S3_CLOUDFRONT_HOST for existing installs. --- .env.nanobox | 4 ++-- .env.production.sample | 4 ++-- config/initializers/paperclip.rb | 4 ++-- config/webpack/production.js | 4 ++-- lib/tasks/mastodon.rake | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) (limited to 'lib') diff --git a/.env.nanobox b/.env.nanobox index 8e0af6a8a..b60b6ee68 100644 --- a/.env.nanobox +++ b/.env.nanobox @@ -136,8 +136,8 @@ SMTP_FROM_ADDRESS=notifications@${APP_NAME}.nanoapp.io # Defaults to 60 seconds. Set to 0 to disable # SWIFT_CACHE_TTL= -# Optional alias for S3 if you want to use Cloudfront or Cloudflare in front -# S3_CLOUDFRONT_HOST= +# Optional alias for S3 (e.g. to serve files on a custom domain, possibly using Cloudfront or Cloudflare) +# S3_ALIAS_HOST= # Streaming API integration # STREAMING_API_BASE_URL= diff --git a/.env.production.sample b/.env.production.sample index 349daedd8..d1164efdc 100644 --- a/.env.production.sample +++ b/.env.production.sample @@ -134,8 +134,8 @@ SMTP_FROM_ADDRESS=notifications@example.com # Defaults to 60 seconds. Set to 0 to disable # SWIFT_CACHE_TTL= -# Optional alias for S3 if you want to use Cloudfront or Cloudflare in front -# S3_CLOUDFRONT_HOST= +# Optional alias for S3 (e.g. to serve files on a custom domain, possibly using Cloudfront or Cloudflare) +# S3_ALIAS_HOST= # Streaming API integration # STREAMING_API_BASE_URL= diff --git a/config/initializers/paperclip.rb b/config/initializers/paperclip.rb index 59ab9b9a1..df0205879 100644 --- a/config/initializers/paperclip.rb +++ b/config/initializers/paperclip.rb @@ -47,10 +47,10 @@ if ENV['S3_ENABLED'] == 'true' Paperclip::Attachment.default_options[:url] = ':s3_path_url' end - if ENV.has_key?('S3_CLOUDFRONT_HOST') + if ENV.has_key?('S3_ALIAS_HOST') || ENV.has_key?('S3_CLOUDFRONT_HOST') Paperclip::Attachment.default_options.merge!( url: ':s3_alias_url', - s3_host_alias: ENV['S3_CLOUDFRONT_HOST'] + s3_host_alias: ENV['S3_ALIAS_HOST'] || ENV['S3_CLOUDFRONT_HOST'] ) end elsif ENV['SWIFT_ENABLED'] == 'true' diff --git a/config/webpack/production.js b/config/webpack/production.js index 4966807a1..27a78108b 100644 --- a/config/webpack/production.js +++ b/config/webpack/production.js @@ -23,8 +23,8 @@ try { let attachmentHost; if (process.env.S3_ENABLED === 'true') { - if (process.env.S3_CLOUDFRONT_HOST) { - attachmentHost = process.env.S3_CLOUDFRONT_HOST; + if (process.env.S3_ALIAS_HOST || process.env.S3_CLOUDFRONT_HOST) { + attachmentHost = process.env.S3_ALIAS_HOST || process.env.S3_CLOUDFRONT_HOST; } else { attachmentHost = process.env.S3_HOSTNAME || `s3-${process.env.S3_REGION || 'us-east-1'}.amazonaws.com`; } diff --git a/lib/tasks/mastodon.rake b/lib/tasks/mastodon.rake index 9ce39335d..519c21785 100644 --- a/lib/tasks/mastodon.rake +++ b/lib/tasks/mastodon.rake @@ -222,7 +222,7 @@ namespace :mastodon do end if prompt.yes?('Do you want to access the uploaded files from your own domain?') - env['S3_CLOUDFRONT_HOST'] = prompt.ask('Domain for uploaded files:') do |q| + env['S3_ALIAS_HOST'] = prompt.ask('Domain for uploaded files:') do |q| q.required true q.default "files.#{env['LOCAL_DOMAIN']}" q.modify :strip -- cgit