From c4fbfaf0b85a589fd669d097471be6bfb82620c6 Mon Sep 17 00:00:00 2001 From: Jeong Arm Date: Sun, 6 Oct 2019 11:30:07 +0900 Subject: Remove unused option from tootctl accounts cull (#12074) --- lib/mastodon/accounts_cli.rb | 1 - 1 file changed, 1 deletion(-) (limited to 'lib') diff --git a/lib/mastodon/accounts_cli.rb b/lib/mastodon/accounts_cli.rb index a09a6ab04..6dbb75689 100644 --- a/lib/mastodon/accounts_cli.rb +++ b/lib/mastodon/accounts_cli.rb @@ -211,7 +211,6 @@ module Mastodon end option :concurrency, type: :numeric, default: 5, aliases: [:c] - option :verbose, type: :boolean, aliases: [:v] option :dry_run, type: :boolean desc 'cull', 'Remove remote accounts that no longer exist' long_desc <<-LONG_DESC -- cgit From 38b6c34e32e2624b1c110809428642dcd1f69c82 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Mon, 7 Oct 2019 04:24:05 +0200 Subject: Fix issues with tootctl's parallelization and progress reporting (#12093) --- lib/mastodon/cli_helper.rb | 36 +++++++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 11 deletions(-) (limited to 'lib') diff --git a/lib/mastodon/cli_helper.rb b/lib/mastodon/cli_helper.rb index da7348349..c2950dffa 100644 --- a/lib/mastodon/cli_helper.rb +++ b/lib/mastodon/cli_helper.rb @@ -15,6 +15,11 @@ module Mastodon end def parallelize_with_progress(scope) + if options[:concurrency] < 1 + say('Cannot run with this concurrency setting, must be at least 1', :red) + exit(1) + end + ActiveRecord::Base.configurations[Rails.env]['pool'] = options[:concurrency] progress = create_progress_bar(scope.count) @@ -27,17 +32,26 @@ module Mastodon items.each do |item| futures << Concurrent::Future.execute(executor: pool) do - ActiveRecord::Base.connection_pool.with_connection do - begin - progress.log("Processing #{item.id}") if options[:verbose] - - result = yield(item) - aggregate.increment(result) if result.is_a?(Integer) - rescue => e - progress.log pastel.red("Error processing #{item.id}: #{e}") - ensure - progress.increment + begin + if !progress.total.nil? && progress.progress + 1 > progress.total + # The number of items has changed between start and now, + # since there is no good way to predict the final count from + # here, just change the progress bar to an indeterminate one + + progress.total = nil end + + progress.log("Processing #{item.id}") if options[:verbose] + + result = ActiveRecord::Base.connection_pool.with_connection do + yield(item) + end + + aggregate.increment(result) if result.is_a?(Integer) + rescue => e + progress.log pastel.red("Error processing #{item.id}: #{e}") + ensure + progress.increment end end end @@ -46,7 +60,7 @@ module Mastodon futures.map(&:value) end - progress.finish + progress.stop [total.value, aggregate.value] end -- cgit From 6c9b4f6b7710cbbdafcceafb59dfef0be6baba65 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Mon, 7 Oct 2019 05:05:02 +0200 Subject: Fix tootctl not allocating enough database connections for main thread (#12097) --- lib/mastodon/cli_helper.rb | 2 +- lib/mastodon/feeds_cli.rb | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/mastodon/cli_helper.rb b/lib/mastodon/cli_helper.rb index c2950dffa..ec4d9a81e 100644 --- a/lib/mastodon/cli_helper.rb +++ b/lib/mastodon/cli_helper.rb @@ -20,7 +20,7 @@ module Mastodon exit(1) end - ActiveRecord::Base.configurations[Rails.env]['pool'] = options[:concurrency] + ActiveRecord::Base.configurations[Rails.env]['pool'] = options[:concurrency] + 1 progress = create_progress_bar(scope.count) pool = Concurrent::FixedThreadPool.new(options[:concurrency]) diff --git a/lib/mastodon/feeds_cli.rb b/lib/mastodon/feeds_cli.rb index ea7c90dff..578ea15c5 100644 --- a/lib/mastodon/feeds_cli.rb +++ b/lib/mastodon/feeds_cli.rb @@ -27,7 +27,6 @@ module Mastodon dry_run = options[:dry_run] ? '(DRY RUN)' : '' if options[:all] || username.nil? - processed, = parallelize_with_progress(Account.joins(:user).merge(User.active)) do |account| PrecomputeFeedService.new.call(account) unless options[:dry_run] end -- cgit From 0aaa3afc2df2cc2fa8cdd277cf91dcfabf45afcd Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Mon, 7 Oct 2019 20:04:56 +0200 Subject: Add `tootctl media usage` command (#12115) --- lib/mastodon/media_cli.rb | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'lib') diff --git a/lib/mastodon/media_cli.rb b/lib/mastodon/media_cli.rb index ec2f36c30..bb97751d5 100644 --- a/lib/mastodon/media_cli.rb +++ b/lib/mastodon/media_cli.rb @@ -97,5 +97,17 @@ module Mastodon say("Downloaded #{processed} media attachments (approx. #{number_to_human_size(aggregate)})#{dry_run}", :green, true) end + + desc 'usage', 'Calculate disk space consumed by Mastodon' + def usage + say("Attachments:\t#{number_to_human_size(MediaAttachment.sum(:file_file_size))} (#{number_to_human_size(MediaAttachment.where(account: Account.local).sum(:file_file_size))} local)") + say("Custom emoji:\t#{number_to_human_size(CustomEmoji.sum(:image_file_size))} (#{number_to_human_size(CustomEmoji.local.sum(:image_file_size))} local)") + say("Preview cards:\t#{number_to_human_size(PreviewCard.sum(:image_file_size))}") + say("Avatars:\t#{number_to_human_size(Account.sum(:avatar_file_size))} (#{number_to_human_size(Account.local.sum(:avatar_file_size))} local)") + say("Headers:\t#{number_to_human_size(Account.sum(:header_file_size))} (#{number_to_human_size(Account.local.sum(:header_file_size))} local)") + say("Backups:\t#{number_to_human_size(Backup.sum(:dump_file_size))}") + say("Imports:\t#{number_to_human_size(Import.sum(:data_file_size))}") + say("Settings:\t#{number_to_human_size(SiteUpload.sum(:file_file_size))}") + end end end -- cgit From 4a98e77d0e7fec15e1830059a9a56bf56aa391db Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 8 Oct 2019 05:59:10 +0200 Subject: Change `tootctl media refresh` to skip already downloaded attachments (#12118) --- lib/mastodon/media_cli.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/mastodon/media_cli.rb b/lib/mastodon/media_cli.rb index bb97751d5..e48175134 100644 --- a/lib/mastodon/media_cli.rb +++ b/lib/mastodon/media_cli.rb @@ -50,6 +50,7 @@ module Mastodon option :concurrency, type: :numeric, default: 5, aliases: [:c] option :verbose, type: :boolean, default: false, aliases: [:v] option :dry_run, type: :boolean, default: false + option :force, type: :boolean, default: false desc 'refresh', 'Fetch remote media files' long_desc <<-DESC Re-downloads media attachments from other servers. You must specify the @@ -62,6 +63,9 @@ module Mastodon using username@domain handle of the account. Use the --domain option to download attachments from a specific domain. + + By default, attachments that are believed to be already downloaded will + not be re-downloaded. To force re-download of every URL, use --force. DESC def refresh dry_run = options[:dry_run] ? ' (DRY RUN)' : '' @@ -85,7 +89,7 @@ module Mastodon end processed, aggregate = parallelize_with_progress(scope) do |media_attachment| - next if media_attachment.remote_url.blank? + next if media_attachment.remote_url.blank? || (!options[:force] && media_attachment.file_file_name.present?) unless options[:dry_run] media_attachment.reset_file! -- cgit From e6d111f38b95b08a2880390b1519655b54ac2fd0 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Wed, 9 Oct 2019 07:36:57 +0200 Subject: Bump version to 3.0.1 (#12116) --- CHANGELOG.md | 30 ++++++++++++++++++++++++++++++ lib/mastodon/version.rb | 2 +- 2 files changed, 31 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b0d23a22..47a466294 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,36 @@ Changelog All notable changes to this project will be documented in this file. +## [3.0.1] - 2019-10-09 +### Added + +- Add `tootctl media usage` command ([Gargron](https://github.com/tootsuite/mastodon/pull/12115)) +- Add admin setting to auto-approve trending hashtags ([Gargron](https://github.com/tootsuite/mastodon/pull/12122)) + +### Changed + +- Change `tootctl media refresh` to skip already downloaded attachments ([Gargron](https://github.com/tootsuite/mastodon/pull/12118)) + +### Removed + +- Remove auto-silence behaviour from spam check ([Gargron](https://github.com/tootsuite/mastodon/pull/12117)) +- Remove HTML `lang` attribute from individual statuses in web UI ([Gargron](https://github.com/tootsuite/mastodon/pull/12124)) +- Remove fallback to long description on sidebar and meta description ([Gargron](https://github.com/tootsuite/mastodon/pull/12119)) + +### Fixed + +- Fix attachment not being re-downloaded even if file is not stored ([Gargron](https://github.com/tootsuite/mastodon/pull/12125)) +- Fix old migration trying to use new column due to default status scope ([Gargron](https://github.com/tootsuite/mastodon/pull/12095)) +- Fix column back button missing for not found accounts ([trwnh](https://github.com/tootsuite/mastodon/pull/12094)) +- Fix issues with tootctl's parallelization and progress reporting ([Gargron](https://github.com/tootsuite/mastodon/pull/12093), [Gargron](https://github.com/tootsuite/mastodon/pull/12097)) +- Fix existing user records with now-renamed `pt` locale ([Gargron](https://github.com/tootsuite/mastodon/pull/12092)) +- Fix hashtag timeline REST API accepting too many hashtags ([Gargron](https://github.com/tootsuite/mastodon/pull/12091)) +- Fix `GET /api/v1/instance` REST APIs being unavailable in secure mode ([Gargron](https://github.com/tootsuite/mastodon/pull/12089)) +- Fix performance of home feed regeneration and merging ([Gargron](https://github.com/tootsuite/mastodon/pull/12084)) +- Fix ffmpeg performance issues due to stdout buffer overflow ([hugogameiro](https://github.com/tootsuite/mastodon/pull/12088)) +- Fix S3 adapter retrying failing uploads with exponential backoff ([Gargron](https://github.com/tootsuite/mastodon/pull/12085)) +- Fix `tootctl accounts cull` advertising unused option flag ([Kjwon15](https://github.com/tootsuite/mastodon/pull/12074)) + ## [3.0.0] - 2019-10-03 ### Added diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb index a0f4678ec..f3ead6d8d 100644 --- a/lib/mastodon/version.rb +++ b/lib/mastodon/version.rb @@ -13,7 +13,7 @@ module Mastodon end def patch - 0 + 1 end def flags -- cgit From 45eccaf8c530c91ff7b0c599bc2e783fe9eb1e6b Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Thu, 10 Oct 2019 06:48:53 +0200 Subject: Fix preloaded JSON-LD context for identity not being used (#12138) Regression from #11316 --- config/initializers/json_ld.rb | 1 + lib/json_ld/identity.rb | 87 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+) create mode 100644 lib/json_ld/identity.rb (limited to 'lib') diff --git a/config/initializers/json_ld.rb b/config/initializers/json_ld.rb index d5575d135..3ed3c4b31 100644 --- a/config/initializers/json_ld.rb +++ b/config/initializers/json_ld.rb @@ -1,3 +1,4 @@ # frozen_string_literal: true require_relative '../../lib/json_ld/security' +require_relative '../../lib/json_ld/identity' diff --git a/lib/json_ld/identity.rb b/lib/json_ld/identity.rb new file mode 100644 index 000000000..4fb3f8e9d --- /dev/null +++ b/lib/json_ld/identity.rb @@ -0,0 +1,87 @@ +# -*- encoding: utf-8 -*- +# frozen_string_literal: true +# This file generated automatically from http://w3id.org/identity/v1 +require 'json/ld' +class JSON::LD::Context + add_preloaded("http://w3id.org/identity/v1") do + new(term_definitions: { + "Credential" => TermDefinition.new("Credential", id: "https://w3id.org/credentials#Credential", simple: true), + "CryptographicKey" => TermDefinition.new("CryptographicKey", id: "https://w3id.org/security#Key", simple: true), + "CryptographicKeyCredential" => TermDefinition.new("CryptographicKeyCredential", id: "https://w3id.org/credentials#CryptographicKeyCredential", simple: true), + "EncryptedMessage" => TermDefinition.new("EncryptedMessage", id: "https://w3id.org/security#EncryptedMessage", simple: true), + "GraphSignature2012" => TermDefinition.new("GraphSignature2012", id: "https://w3id.org/security#GraphSignature2012", simple: true), + "Group" => TermDefinition.new("Group", id: "https://www.w3.org/ns/activitystreams#Group", simple: true), + "Identity" => TermDefinition.new("Identity", id: "https://w3id.org/identity#Identity", simple: true), + "LinkedDataSignature2015" => TermDefinition.new("LinkedDataSignature2015", id: "https://w3id.org/security#LinkedDataSignature2015", simple: true), + "Organization" => TermDefinition.new("Organization", id: "http://schema.org/Organization", simple: true), + "Person" => TermDefinition.new("Person", id: "http://schema.org/Person", simple: true), + "PostalAddress" => TermDefinition.new("PostalAddress", id: "http://schema.org/PostalAddress", simple: true), + "about" => TermDefinition.new("about", id: "http://schema.org/about", type_mapping: "@id"), + "accessControl" => TermDefinition.new("accessControl", id: "https://w3id.org/permissions#accessControl", type_mapping: "@id"), + "address" => TermDefinition.new("address", id: "http://schema.org/address", type_mapping: "@id"), + "addressCountry" => TermDefinition.new("addressCountry", id: "http://schema.org/addressCountry", simple: true), + "addressLocality" => TermDefinition.new("addressLocality", id: "http://schema.org/addressLocality", simple: true), + "addressRegion" => TermDefinition.new("addressRegion", id: "http://schema.org/addressRegion", simple: true), + "cipherAlgorithm" => TermDefinition.new("cipherAlgorithm", id: "https://w3id.org/security#cipherAlgorithm", simple: true), + "cipherData" => TermDefinition.new("cipherData", id: "https://w3id.org/security#cipherData", simple: true), + "cipherKey" => TermDefinition.new("cipherKey", id: "https://w3id.org/security#cipherKey", simple: true), + "claim" => TermDefinition.new("claim", id: "https://w3id.org/credentials#claim", type_mapping: "@id"), + "comment" => TermDefinition.new("comment", id: "http://www.w3.org/2000/01/rdf-schema#comment", simple: true), + "created" => TermDefinition.new("created", id: "http://purl.org/dc/terms/created", type_mapping: "http://www.w3.org/2001/XMLSchema#dateTime"), + "creator" => TermDefinition.new("creator", id: "http://purl.org/dc/terms/creator", type_mapping: "@id"), + "cred" => TermDefinition.new("cred", id: "https://w3id.org/credentials#", simple: true, prefix: true), + "credential" => TermDefinition.new("credential", id: "https://w3id.org/credentials#credential", type_mapping: "@id"), + "dc" => TermDefinition.new("dc", id: "http://purl.org/dc/terms/", simple: true, prefix: true), + "description" => TermDefinition.new("description", id: "http://schema.org/description", simple: true), + "digestAlgorithm" => TermDefinition.new("digestAlgorithm", id: "https://w3id.org/security#digestAlgorithm", simple: true), + "digestValue" => TermDefinition.new("digestValue", id: "https://w3id.org/security#digestValue", simple: true), + "domain" => TermDefinition.new("domain", id: "https://w3id.org/security#domain", simple: true), + "email" => TermDefinition.new("email", id: "http://schema.org/email", simple: true), + "expires" => TermDefinition.new("expires", id: "https://w3id.org/security#expiration", type_mapping: "http://www.w3.org/2001/XMLSchema#dateTime"), + "familyName" => TermDefinition.new("familyName", id: "http://schema.org/familyName", simple: true), + "givenName" => TermDefinition.new("givenName", id: "http://schema.org/givenName", simple: true), + "id" => TermDefinition.new("id", id: "@id", simple: true), + "identity" => TermDefinition.new("identity", id: "https://w3id.org/identity#", simple: true, prefix: true), + "identityService" => TermDefinition.new("identityService", id: "https://w3id.org/identity#identityService", type_mapping: "@id"), + "idp" => TermDefinition.new("idp", id: "https://w3id.org/identity#idp", type_mapping: "@id"), + "image" => TermDefinition.new("image", id: "http://schema.org/image", type_mapping: "@id"), + "initializationVector" => TermDefinition.new("initializationVector", id: "https://w3id.org/security#initializationVector", simple: true), + "issued" => TermDefinition.new("issued", id: "https://w3id.org/credentials#issued", type_mapping: "http://www.w3.org/2001/XMLSchema#dateTime"), + "issuer" => TermDefinition.new("issuer", id: "https://w3id.org/credentials#issuer", type_mapping: "@id"), + "label" => TermDefinition.new("label", id: "http://www.w3.org/2000/01/rdf-schema#label", simple: true), + "member" => TermDefinition.new("member", id: "http://schema.org/member", type_mapping: "@id"), + "memberOf" => TermDefinition.new("memberOf", id: "http://schema.org/memberOf", type_mapping: "@id"), + "name" => TermDefinition.new("name", id: "http://schema.org/name", simple: true), + "nonce" => TermDefinition.new("nonce", id: "https://w3id.org/security#nonce", simple: true), + "normalizationAlgorithm" => TermDefinition.new("normalizationAlgorithm", id: "https://w3id.org/security#normalizationAlgorithm", simple: true), + "owner" => TermDefinition.new("owner", id: "https://w3id.org/security#owner", type_mapping: "@id"), + "password" => TermDefinition.new("password", id: "https://w3id.org/security#password", simple: true), + "paymentProcessor" => TermDefinition.new("paymentProcessor", id: "https://w3id.org/payswarm#processor", simple: true), + "perm" => TermDefinition.new("perm", id: "https://w3id.org/permissions#", simple: true, prefix: true), + "postalCode" => TermDefinition.new("postalCode", id: "http://schema.org/postalCode", simple: true), + "preferences" => TermDefinition.new("preferences", id: "https://w3id.org/payswarm#preferences", type_mapping: "@vocab"), + "privateKey" => TermDefinition.new("privateKey", id: "https://w3id.org/security#privateKey", type_mapping: "@id"), + "privateKeyPem" => TermDefinition.new("privateKeyPem", id: "https://w3id.org/security#privateKeyPem", simple: true), + "ps" => TermDefinition.new("ps", id: "https://w3id.org/payswarm#", simple: true, prefix: true), + "publicKey" => TermDefinition.new("publicKey", id: "https://w3id.org/security#publicKey", type_mapping: "@id"), + "publicKeyPem" => TermDefinition.new("publicKeyPem", id: "https://w3id.org/security#publicKeyPem", simple: true), + "publicKeyService" => TermDefinition.new("publicKeyService", id: "https://w3id.org/security#publicKeyService", type_mapping: "@id"), + "rdf" => TermDefinition.new("rdf", id: "http://www.w3.org/1999/02/22-rdf-syntax-ns#", simple: true, prefix: true), + "rdfs" => TermDefinition.new("rdfs", id: "http://www.w3.org/2000/01/rdf-schema#", simple: true, prefix: true), + "recipient" => TermDefinition.new("recipient", id: "https://w3id.org/credentials#recipient", type_mapping: "@id"), + "revoked" => TermDefinition.new("revoked", id: "https://w3id.org/security#revoked", type_mapping: "http://www.w3.org/2001/XMLSchema#dateTime"), + "schema" => TermDefinition.new("schema", id: "http://schema.org/", simple: true, prefix: true), + "sec" => TermDefinition.new("sec", id: "https://w3id.org/security#", simple: true, prefix: true), + "signature" => TermDefinition.new("signature", id: "https://w3id.org/security#signature", simple: true), + "signatureAlgorithm" => TermDefinition.new("signatureAlgorithm", id: "https://w3id.org/security#signatureAlgorithm", simple: true), + "signatureValue" => TermDefinition.new("signatureValue", id: "https://w3id.org/security#signatureValue", simple: true), + "streetAddress" => TermDefinition.new("streetAddress", id: "http://schema.org/streetAddress", simple: true), + "title" => TermDefinition.new("title", id: "http://purl.org/dc/terms/title", simple: true), + "type" => TermDefinition.new("type", id: "@type", simple: true), + "url" => TermDefinition.new("url", id: "http://schema.org/url", type_mapping: "@id"), + "writePermission" => TermDefinition.new("writePermission", id: "https://w3id.org/permissions#writePermission", type_mapping: "@id"), + "xsd" => TermDefinition.new("xsd", id: "http://www.w3.org/2001/XMLSchema#", simple: true, prefix: true) + }) + end + alias_preloaded("https://w3id.org/identity/v1", "http://w3id.org/identity/v1") +end -- cgit