From a6b44401156524ef70a5917d17c93886c5e20a0d Mon Sep 17 00:00:00 2001 From: Clar Fon Date: Fri, 26 Jul 2019 01:57:27 -0400 Subject: Remove pre from version, add extra suffix variable (#11407) --- lib/mastodon/version.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb index 47eac2432..3db57ceaa 100644 --- a/lib/mastodon/version.rb +++ b/lib/mastodon/version.rb @@ -16,20 +16,20 @@ module Mastodon 2 end - def pre - nil + def flags + '' end - def flags + def suffix '' end def to_a - [major, minor, patch, pre].compact + [major, minor, patch].compact end def to_s - [to_a.join('.'), flags].join + [to_a.join('.'), flags, suffix].join end def repository -- cgit From 0d80f686d81e4a12db1ac54b8f31f69f375dc8dc Mon Sep 17 00:00:00 2001 From: mayaeh Date: Sun, 28 Jul 2019 20:48:19 +0900 Subject: Add `tootctl preview_cards remove` (#11320) * Add `tootctl preview_cards remove` * fix code style * Remove `Scheduler::PreviewCardsCleanupScheduler` file * fix code style again Add exclude case where image_file_name is blank * Added a function to output confirmation if the specified number of days is less than 2 weeks --- .../scheduler/preview_cards_cleanup_scheduler.rb | 22 ----- config/sidekiq.yml | 3 - lib/cli.rb | 4 + lib/mastodon/preview_cards_cli.rb | 94 ++++++++++++++++++++++ 4 files changed, 98 insertions(+), 25 deletions(-) delete mode 100644 app/workers/scheduler/preview_cards_cleanup_scheduler.rb create mode 100644 lib/mastodon/preview_cards_cli.rb (limited to 'lib') diff --git a/app/workers/scheduler/preview_cards_cleanup_scheduler.rb b/app/workers/scheduler/preview_cards_cleanup_scheduler.rb deleted file mode 100644 index 2b38792f0..000000000 --- a/app/workers/scheduler/preview_cards_cleanup_scheduler.rb +++ /dev/null @@ -1,22 +0,0 @@ -# frozen_string_literal: true - -class Scheduler::PreviewCardsCleanupScheduler - include Sidekiq::Worker - - sidekiq_options unique: :until_executed, retry: 0 - - def perform - Maintenance::UncachePreviewWorker.push_bulk(recent_link_preview_cards.pluck(:id)) - Maintenance::UncachePreviewWorker.push_bulk(older_preview_cards.pluck(:id)) - end - - private - - def recent_link_preview_cards - PreviewCard.where(type: :link).where('updated_at < ?', 1.month.ago) - end - - def older_preview_cards - PreviewCard.where('updated_at < ?', 6.months.ago) - end -end diff --git a/config/sidekiq.yml b/config/sidekiq.yml index 7f41b6607..6ebe450b0 100644 --- a/config/sidekiq.yml +++ b/config/sidekiq.yml @@ -24,9 +24,6 @@ ip_cleanup_scheduler: cron: '<%= Random.rand(0..59) %> <%= Random.rand(3..5) %> * * *' class: Scheduler::IpCleanupScheduler - preview_cards_cleanup_scheduler: - cron: '<%= Random.rand(0..59) %> <%= Random.rand(3..5) %> * * *' - class: Scheduler::PreviewCardsCleanupScheduler email_scheduler: cron: '0 10 * * 2' class: Scheduler::EmailScheduler diff --git a/lib/cli.rb b/lib/cli.rb index be276583d..fbdf49fc3 100644 --- a/lib/cli.rb +++ b/lib/cli.rb @@ -9,6 +9,7 @@ require_relative 'mastodon/search_cli' require_relative 'mastodon/settings_cli' require_relative 'mastodon/statuses_cli' require_relative 'mastodon/domains_cli' +require_relative 'mastodon/preview_cards_cli' require_relative 'mastodon/cache_cli' require_relative 'mastodon/version' @@ -42,6 +43,9 @@ module Mastodon desc 'domains SUBCOMMAND ...ARGS', 'Manage account domains' subcommand 'domains', Mastodon::DomainsCLI + desc 'preview_cards SUBCOMMAND ...ARGS', 'Manage preview cards' + subcommand 'preview_cards', Mastodon::PreviewCardsCLI + desc 'cache SUBCOMMAND ...ARGS', 'Manage cache' subcommand 'cache', Mastodon::CacheCLI diff --git a/lib/mastodon/preview_cards_cli.rb b/lib/mastodon/preview_cards_cli.rb new file mode 100644 index 000000000..465fe7d0b --- /dev/null +++ b/lib/mastodon/preview_cards_cli.rb @@ -0,0 +1,94 @@ +# frozen_string_literal: true + +require 'tty-prompt' +require_relative '../../config/boot' +require_relative '../../config/environment' +require_relative 'cli_helper' + +module Mastodon + class PreviewCardsCLI < Thor + include ActionView::Helpers::NumberHelper + + def self.exit_on_failure? + true + end + + option :days, type: :numeric, default: 180 + option :background, type: :boolean, default: false + option :verbose, type: :boolean, default: false + option :dry_run, type: :boolean, default: false + option :link, type: :boolean, default: false + desc 'remove', 'Remove preview cards' + long_desc <<-DESC + Removes locally thumbnails for previews. + + The --days option specifies how old preview cards have to be before + they are removed. It defaults to 180 days. + + With the --background option, instead of deleting the files sequentially, + they will be queued into Sidekiq and the command will exit as soon as + possible. In Sidekiq they will be processed with higher concurrency, but + it may impact other operations of the Mastodon server, and it may overload + the underlying file storage. + + With the --dry-run option, no work will be done. + + With the --verbose option, when preview cards are processed sequentially in the + foreground, the IDs of the preview cards will be printed. + + With the --link option, delete only link-type preview cards. + DESC + def remove + prompt = TTY::Prompt.new + time_ago = options[:days].days.ago + queued = 0 + processed = 0 + size = 0 + dry_run = options[:dry_run] ? '(DRY RUN)' : '' + link = options[:link] ? 'link-type ' : '' + scope = PreviewCard.where.not(image_file_name: nil) + scope = scope.where.not(image_file_name: '') + scope = scope.where(type: :link) if options[:link] + scope = scope.where('updated_at < ?', time_ago) + + if time_ago > 2.weeks.ago + prompt.say "\n" + prompt.say('The preview cards less than the past two weeks will not be re-acquired even when needed.') + prompt.say "\n" + + unless prompt.yes?('Are you sure you want to delete the preview cards?', default: false) + prompt.say "\n" + prompt.warn 'Nothing execute. Bye!' + prompt.say "\n" + exit(1) + end + end + + if options[:background] + scope.select(:id, :image_file_size).reorder(nil).find_in_batches do |preview_cards| + queued += preview_cards.size + size += preview_cards.reduce(0) { |sum, p| sum + (p.image_file_size || 0) } + Maintenance::UncachePreviewWorker.push_bulk(preview_cards.map(&:id)) unless options[:dry_run] + end + + else + scope.select(:id, :image_file_size).reorder(nil).find_in_batches do |preview_cards| + preview_cards.each do |p| + size += p.image_file_size || 0 + Maintenance::UncachePreviewWorker.new.perform(p.id) unless options[:dry_run] + options[:verbose] ? say(p.id) : say('.', :green, false) + processed += 1 + end + end + end + + say + + if options[:background] + say("Scheduled the deletion of #{queued} #{link}preview cards (approx. #{number_to_human_size(size)}) #{dry_run}", :green, true) + else + say("Removed #{processed} #{link}preview cards (approx. #{number_to_human_size(size)}) #{dry_run}", :green, true) + end + end + end +end -- cgit From 15de24a425cb07efd58db5299b043e4110fa138f Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Sun, 28 Jul 2019 13:48:43 +0200 Subject: Bump json-ld-preloaded from 3.0.2 to 3.0.3 (#11316) * Bump json-ld-preloaded from 3.0.2 to 3.0.3 Bumps [json-ld-preloaded](https://github.com/ruby-rdf/json-ld-preloaded) from 3.0.2 to 3.0.3. - [Release notes](https://github.com/ruby-rdf/json-ld-preloaded/releases) - [Commits](https://github.com/ruby-rdf/json-ld-preloaded/compare/3.0.2...3.0.3) Signed-off-by: dependabot-preview[bot] * use json-ld edge --- Gemfile | 2 +- Gemfile.lock | 23 +++++++++++++++++------ app/helpers/jsonld_helper.rb | 2 +- lib/json_ld/security.rb | 5 +++-- 4 files changed, 22 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/Gemfile b/Gemfile index 96eb44af7..04ebc4cd8 100644 --- a/Gemfile +++ b/Gemfile @@ -92,7 +92,7 @@ gem 'tzinfo-data', '~> 1.2019' gem 'webpacker', '~> 4.0' gem 'webpush' -gem 'json-ld', '~> 3.0' +gem 'json-ld', git: 'https://github.com/ruby-rdf/json-ld.git', ref: '345b7a5733308af827e8491d284dbafa9128d7a2' gem 'json-ld-preloaded', '~> 3.0' gem 'rdf-normalize', '~> 0.3' diff --git a/Gemfile.lock b/Gemfile.lock index 5adae6f9c..8da261639 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -5,6 +5,19 @@ GIT specs: posix-spawn (0.3.13) +GIT + remote: https://github.com/ruby-rdf/json-ld.git + revision: 345b7a5733308af827e8491d284dbafa9128d7a2 + ref: 345b7a5733308af827e8491d284dbafa9128d7a2 + specs: + json-ld (3.0.2) + htmlentities (~> 4.3) + json-canonicalization (~> 0.1) + link_header (~> 0.0, >= 0.0.8) + multi_json (~> 1.13) + rack (>= 1.6, < 3.0) + rdf (~> 3.0, >= 3.0.8) + GIT remote: https://github.com/tmm1/http_parser.rb revision: 54b17ba8c7d8d20a16dfc65d1775241833219cf2 @@ -297,10 +310,8 @@ GEM jaro_winkler (1.5.3) jmespath (1.4.0) json (2.2.0) - json-ld (3.0.2) - multi_json (~> 1.12) - rdf (>= 2.2.8, < 4.0) - json-ld-preloaded (3.0.2) + json-canonicalization (0.1.0) + json-ld-preloaded (3.0.3) json-ld (~> 3.0) multi_json (~> 1.12) rdf (~> 3.0) @@ -479,7 +490,7 @@ GEM thor (>= 0.19.0, < 2.0) rainbow (3.0.0) rake (12.3.2) - rdf (3.0.9) + rdf (3.0.12) hamster (~> 3.0) link_header (~> 0.0, >= 0.0.8) rdf-normalize (0.3.3) @@ -700,7 +711,7 @@ DEPENDENCIES i18n-tasks (~> 0.9) idn-ruby iso-639 - json-ld (~> 3.0) + json-ld! json-ld-preloaded (~> 3.0) kaminari (~> 1.1) letter_opener (~> 1.7) diff --git a/app/helpers/jsonld_helper.rb b/app/helpers/jsonld_helper.rb index 83a5b2462..1c473efa3 100644 --- a/app/helpers/jsonld_helper.rb +++ b/app/helpers/jsonld_helper.rb @@ -130,7 +130,7 @@ module JsonLdHelper end end - doc = JSON::LD::API::RemoteDocument.new(url, json) + doc = JSON::LD::API::RemoteDocument.new(json, documentUrl: url) block_given? ? yield(doc) : doc end diff --git a/lib/json_ld/security.rb b/lib/json_ld/security.rb index 1230206f0..a6fbce95f 100644 --- a/lib/json_ld/security.rb +++ b/lib/json_ld/security.rb @@ -1,9 +1,9 @@ # -*- encoding: utf-8 -*- # frozen_string_literal: true -# This file generated automatically from https://w3id.org/security/v1 +# This file generated automatically from http://w3id.org/security/v1 require 'json/ld' class JSON::LD::Context - add_preloaded("https://w3id.org/security/v1") do + add_preloaded("http://w3id.org/security/v1") do new(processingMode: "json-ld-1.0", term_definitions: { "CryptographicKey" => TermDefinition.new("CryptographicKey", id: "https://w3id.org/security#Key", simple: true), "EcdsaKoblitzSignature2016" => TermDefinition.new("EcdsaKoblitzSignature2016", id: "https://w3id.org/security#EcdsaKoblitzSignature2016", simple: true), @@ -47,4 +47,5 @@ class JSON::LD::Context "xsd" => TermDefinition.new("xsd", id: "http://www.w3.org/2001/XMLSchema#", simple: true, prefix: true) }) end + alias_preloaded("https://w3id.org/security/v1", "http://w3id.org/security/v1") end -- cgit