From 370e38ee15da6ff0b85c048486b56b173c1d4b02 Mon Sep 17 00:00:00 2001 From: tateisu Date: Thu, 25 Jun 2020 19:17:10 +0900 Subject: Add tootctl email-domain-blocks (#13589) * Add tootctl email_domains (block|unblock) * fix codeclimate issues. * fix codeclimate issues. * fix codeclimate issues. * add list subcommand, remove log_action. * fix codeclimate issues. * filter duplicate hostnames,ips before block * rebase from currnet master branch. rename email_domains_cli.rb to email_domain_blocks_cli.rb . rename Mastodon::EmailDomainsCLI to Mastodon::EmailDomainBlocksCLI . rename command email_domains to email-domain-blocks . (Thor recognizes both of - and _ ) rename subcommand block to add . rename subcommand unblock to remove . change the color in list subcommand to while for domain or cyan for childlen. don't use include() in list subcommand. suppress console output about succeeded entry. add console output about count of processed/skipped. remove capitalization in subcommand description. remove long_desc in subcommand 'remove'. remove duplicate where in subcommand 'remove'. * fix codeclimate issue. --- lib/cli.rb | 4 + lib/mastodon/email_domain_blocks_cli.rb | 133 ++++++++++++++++++++++++++++++++ 2 files changed, 137 insertions(+) create mode 100644 lib/mastodon/email_domain_blocks_cli.rb (limited to 'lib') diff --git a/lib/cli.rb b/lib/cli.rb index 313a36a3d..7cab0d5a1 100644 --- a/lib/cli.rb +++ b/lib/cli.rb @@ -12,6 +12,7 @@ require_relative 'mastodon/domains_cli' require_relative 'mastodon/preview_cards_cli' require_relative 'mastodon/cache_cli' require_relative 'mastodon/upgrade_cli' +require_relative 'mastodon/email_domain_blocks_cli' require_relative 'mastodon/version' module Mastodon @@ -53,6 +54,9 @@ module Mastodon desc 'upgrade SUBCOMMAND ...ARGS', 'Various version upgrade utilities' subcommand 'upgrade', Mastodon::UpgradeCLI + desc 'email-domain-blocks SUBCOMMAND ...ARGS', 'Manage E-mail domain blocks' + subcommand 'email_domain_blocks', Mastodon::EmailDomainBlocksCLI + option :dry_run, type: :boolean desc 'self-destruct', 'Erase the server from the federation' long_desc <<~LONG_DESC diff --git a/lib/mastodon/email_domain_blocks_cli.rb b/lib/mastodon/email_domain_blocks_cli.rb new file mode 100644 index 000000000..8b468ed15 --- /dev/null +++ b/lib/mastodon/email_domain_blocks_cli.rb @@ -0,0 +1,133 @@ +# frozen_string_literal: true + +require 'concurrent' +require_relative '../../config/boot' +require_relative '../../config/environment' +require_relative 'cli_helper' + +module Mastodon + class EmailDomainBlocksCLI < Thor + include CLIHelper + + def self.exit_on_failure? + true + end + + desc 'list', 'list E-mail domain blocks' + long_desc <<-LONG_DESC + list up all E-mail domain blocks. + LONG_DESC + def list + EmailDomainBlock.where(parent_id: nil).order(id: 'DESC').find_each do |entry| + say(entry.domain.to_s, :white) + EmailDomainBlock.where(parent_id: entry.id).order(id: 'DESC').find_each do |child| + say(" #{child.domain}", :cyan) + end + end + end + + option :with_dns_records, type: :boolean + desc 'add [DOMAIN...]', 'add E-mail domain blocks' + long_desc <<-LONG_DESC + add E-mail domain blocks from a given DOMAIN. + + When the --with-dns-records option is given, An attempt to resolve the + given domain's DNS records will be made and the results will also be + blacklisted. + LONG_DESC + def add(*domains) + if domains.empty? + say('No domain(s) given', :red) + exit(1) + end + + skipped = 0 + processed = 0 + + domains.each do |domain| + if EmailDomainBlock.where(domain: domain).exists? + say("#{domain} is already blocked.", :yellow) + skipped += 1 + next + end + + email_domain_block = EmailDomainBlock.new(domain: domain, with_dns_records: options[:with_dns_records] || false) + email_domain_block.save! + processed += 1 + + next unless email_domain_block.with_dns_records? + + hostnames = [] + ips = [] + + Resolv::DNS.open do |dns| + dns.timeouts = 1 + hostnames = dns.getresources(email_domain_block.domain, Resolv::DNS::Resource::IN::MX).to_a.map { |e| e.exchange.to_s } + + ([email_domain_block.domain] + hostnames).uniq.each do |hostname| + ips.concat(dns.getresources(hostname, Resolv::DNS::Resource::IN::A).to_a.map { |e| e.address.to_s }) + ips.concat(dns.getresources(hostname, Resolv::DNS::Resource::IN::AAAA).to_a.map { |e| e.address.to_s }) + end + end + + (hostnames + ips).uniq.each do |hostname| + another_email_domain_block = EmailDomainBlock.new(domain: hostname, parent: email_domain_block) + if EmailDomainBlock.where(domain: hostname).exists? + say("#{hostname} is already blocked.", :yellow) + skipped += 1 + next + end + another_email_domain_block.save! + processed += 1 + end + end + + say("Added #{processed}, skipped #{skipped}", color(processed, 0)) + end + + desc 'remove [DOMAIN...]', 'remove E-mail domain blocks' + def remove(*domains) + if domains.empty? + say('No domain(s) given', :red) + exit(1) + end + + skipped = 0 + processed = 0 + failed = 0 + + domains.each do |domain| + entry = EmailDomainBlock.find_by(domain: domain) + if entry.nil? + say("#{domain} is not yet blocked.", :yellow) + skipped += 1 + next + end + + children_count = EmailDomainBlock.where(parent_id: entry.id).count + + result = entry.destroy + if result + processed += 1 + children_count + else + say("#{domain} was not unblocked. 'destroy' returns false.", :red) + failed += 1 + end + end + + say("Removed #{processed}, skipped #{skipped}, failed #{failed}", color(processed, failed)) + end + + private + + def color(processed, failed) + if !processed.zero? && failed.zero? + :green + elsif failed.zero? + :yellow + else + :red + end + end + end +end -- cgit From 4662afe0756f821ce5febb90dee96887b4c247bb Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Fri, 26 Jun 2020 21:28:40 +0200 Subject: Fix help text around `tootctl email_domain_blocks` (#14147) --- lib/cli.rb | 2 +- lib/mastodon/email_domain_blocks_cli.rb | 31 ++++++++++++++++++------------- 2 files changed, 19 insertions(+), 14 deletions(-) (limited to 'lib') diff --git a/lib/cli.rb b/lib/cli.rb index 7cab0d5a1..9162144cc 100644 --- a/lib/cli.rb +++ b/lib/cli.rb @@ -54,7 +54,7 @@ module Mastodon desc 'upgrade SUBCOMMAND ...ARGS', 'Various version upgrade utilities' subcommand 'upgrade', Mastodon::UpgradeCLI - desc 'email-domain-blocks SUBCOMMAND ...ARGS', 'Manage E-mail domain blocks' + desc 'email_domain_blocks SUBCOMMAND ...ARGS', 'Manage e-mail domain blocks' subcommand 'email_domain_blocks', Mastodon::EmailDomainBlocksCLI option :dry_run, type: :boolean diff --git a/lib/mastodon/email_domain_blocks_cli.rb b/lib/mastodon/email_domain_blocks_cli.rb index 8b468ed15..7fe1efaaa 100644 --- a/lib/mastodon/email_domain_blocks_cli.rb +++ b/lib/mastodon/email_domain_blocks_cli.rb @@ -13,13 +13,11 @@ module Mastodon true end - desc 'list', 'list E-mail domain blocks' - long_desc <<-LONG_DESC - list up all E-mail domain blocks. - LONG_DESC + desc 'list', 'List blocked e-mail domains' def list EmailDomainBlock.where(parent_id: nil).order(id: 'DESC').find_each do |entry| say(entry.domain.to_s, :white) + EmailDomainBlock.where(parent_id: entry.id).order(id: 'DESC').find_each do |child| say(" #{child.domain}", :cyan) end @@ -27,13 +25,17 @@ module Mastodon end option :with_dns_records, type: :boolean - desc 'add [DOMAIN...]', 'add E-mail domain blocks' + desc 'add DOMAIN...', 'Block e-mail domain(s)' long_desc <<-LONG_DESC - add E-mail domain blocks from a given DOMAIN. - - When the --with-dns-records option is given, An attempt to resolve the - given domain's DNS records will be made and the results will also be - blacklisted. + Blocking an e-mail domain prevents users from signing up + with e-mail addresses from that domain. You can provide one or + multiple domains to the command. + + When the --with-dns-records option is given, an attempt to resolve the + given domains' DNS records will be made and the results (A, AAAA and MX) will + also be blocked. This can be helpful if you are blocking an e-mail server that + has many different domains pointing to it as it allows you to essentially block + it at the root. LONG_DESC def add(*domains) if domains.empty? @@ -72,11 +74,13 @@ module Mastodon (hostnames + ips).uniq.each do |hostname| another_email_domain_block = EmailDomainBlock.new(domain: hostname, parent: email_domain_block) + if EmailDomainBlock.where(domain: hostname).exists? say("#{hostname} is already blocked.", :yellow) skipped += 1 next end + another_email_domain_block.save! processed += 1 end @@ -85,7 +89,7 @@ module Mastodon say("Added #{processed}, skipped #{skipped}", color(processed, 0)) end - desc 'remove [DOMAIN...]', 'remove E-mail domain blocks' + desc 'remove DOMAIN...', 'Remove e-mail domain blocks' def remove(*domains) if domains.empty? say('No domain(s) given', :red) @@ -98,6 +102,7 @@ module Mastodon domains.each do |domain| entry = EmailDomainBlock.find_by(domain: domain) + if entry.nil? say("#{domain} is not yet blocked.", :yellow) skipped += 1 @@ -105,12 +110,12 @@ module Mastodon end children_count = EmailDomainBlock.where(parent_id: entry.id).count - result = entry.destroy + if result processed += 1 + children_count else - say("#{domain} was not unblocked. 'destroy' returns false.", :red) + say("#{domain} could not be unblocked.", :red) failed += 1 end end -- cgit From 8c04e37b033ff35174753fb82faa1cd7bed110da Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sat, 27 Jun 2020 20:20:11 +0200 Subject: Remove the terms blacklist and whitelist from UX (#14149) Localization strings: - "Whitelist mode" -> "Limited federation mode" - "Blacklist e-mail domain" -> "Block e-mail domain" - "Whitelist domain" -> "Allow domain for federation" ...And so on Environment variables (backwards-compatible): - `WHITELIST_MODE` -> `LIMITED_FEDERATION_MODE` - `EMAIL_DOMAIN_BLACKLIST` -> `EMAIL_DOMAIN_DENYLIST` - `EMAIL_DOMAIN_WHITELIST` -> `EMAIL_DOMAIN_ALLOWLIST` tootctl: - `tootctl domains purge --whitelist-mode` -> `tootctl domains purge --limited-federation-mode` Removed badly maintained and no longer relevant .env.production.sample file --- .env.production.sample | 262 -------------------------------- config/initializers/2_whitelist_mode.rb | 2 +- config/initializers/blacklists.rb | 4 +- config/locales/en.yml | 34 ++--- lib/mastodon/domains_cli.rb | 8 +- 5 files changed, 24 insertions(+), 286 deletions(-) delete mode 100644 .env.production.sample (limited to 'lib') diff --git a/.env.production.sample b/.env.production.sample deleted file mode 100644 index e041e0a04..000000000 --- a/.env.production.sample +++ /dev/null @@ -1,262 +0,0 @@ -# Service dependencies -# You may set REDIS_URL instead for more advanced options -# You may also set REDIS_NAMESPACE to share Redis between multiple Mastodon servers -REDIS_HOST=redis -REDIS_PORT=6379 -# You may set DATABASE_URL instead for more advanced options -DB_HOST=db -DB_USER=postgres -DB_NAME=postgres -DB_PASS= -DB_PORT=5432 -# Optional ElasticSearch configuration -# You may also set ES_PREFIX to share the same cluster between multiple Mastodon servers (falls back to REDIS_NAMESPACE if not set) -# ES_ENABLED=true -# ES_HOST=es -# ES_PORT=9200 - -# Federation -# Note: Changing LOCAL_DOMAIN at a later time will cause unwanted side effects, including breaking all existing federation. -# LOCAL_DOMAIN should *NOT* contain the protocol part of the domain e.g https://example.com. -LOCAL_DOMAIN=example.com - -# Changing LOCAL_HTTPS in production is no longer supported. (Mastodon will always serve https:// links) - -# Use this only if you need to run mastodon on a different domain than the one used for federation. -# You can read more about this option on https://github.com/tootsuite/documentation/blob/master/Running-Mastodon/Serving_a_different_domain.md -# DO *NOT* USE THIS UNLESS YOU KNOW *EXACTLY* WHAT YOU ARE DOING. -# WEB_DOMAIN=mastodon.example.com - -# Use this if you want to have several aliases handler@example1.com -# handler@example2.com etc. for the same user. LOCAL_DOMAIN should not -# be added. Comma separated values -# ALTERNATE_DOMAINS=example1.com,example2.com - -# Application secrets -# Generate each with the `RAILS_ENV=production bundle exec rake secret` task (`docker-compose run --rm web bundle exec rake secret` if you use docker compose) -SECRET_KEY_BASE= -OTP_SECRET= - -# VAPID keys (used for push notifications -# You can generate the keys using the following command (first is the private key, second is the public one) -# You should only generate this once per instance. If you later decide to change it, all push subscription will -# be invalidated, requiring the users to access the website again to resubscribe. -# -# Generate with `RAILS_ENV=production bundle exec rake mastodon:webpush:generate_vapid_key` task (`docker-compose run --rm web bundle exec rake mastodon:webpush:generate_vapid_key` if you use docker compose) -# -# For more information visit https://rossta.net/blog/using-the-web-push-api-with-vapid.html -VAPID_PRIVATE_KEY= -VAPID_PUBLIC_KEY= - -# Registrations -# Single user mode will disable registrations and redirect frontpage to the first profile -# SINGLE_USER_MODE=true -# Prevent registrations with following e-mail domains -# EMAIL_DOMAIN_BLACKLIST=example1.com|example2.de|etc -# Only allow registrations with the following e-mail domains -# EMAIL_DOMAIN_WHITELIST=example1.com|example2.de|etc - -# Optionally change default language -# DEFAULT_LOCALE=de - -# E-mail configuration -# Note: Mailgun and SparkPost (https://sparkpo.st/smtp) each have good free tiers -# If you want to use an SMTP server without authentication (e.g local Postfix relay) -# then set SMTP_AUTH_METHOD and SMTP_OPENSSL_VERIFY_MODE to 'none' and -# *comment* SMTP_LOGIN and SMTP_PASSWORD (leaving them blank is not enough). -SMTP_SERVER=smtp.mailgun.org -SMTP_PORT=587 -SMTP_LOGIN= -SMTP_PASSWORD= -SMTP_FROM_ADDRESS=notifications@example.com -#SMTP_REPLY_TO= -#SMTP_DOMAIN= # defaults to LOCAL_DOMAIN -#SMTP_DELIVERY_METHOD=smtp # delivery method can also be sendmail -#SMTP_AUTH_METHOD=plain -#SMTP_CA_FILE=/etc/ssl/certs/ca-certificates.crt -#SMTP_OPENSSL_VERIFY_MODE=peer -#SMTP_ENABLE_STARTTLS_AUTO=true -#SMTP_TLS=true - -# Optional user upload path and URL (images, avatars). Default is :rails_root/public/system. If you set this variable, you are responsible for making your HTTP server (eg. nginx) serve these files. -# PAPERCLIP_ROOT_PATH=/var/lib/mastodon/public-system -# PAPERCLIP_ROOT_URL=/system - -# Optional asset host for multi-server setups -# The asset host must allow cross origin request from WEB_DOMAIN or LOCAL_DOMAIN -# if WEB_DOMAIN is not set. For example, the server may have the -# following header field: -# Access-Control-Allow-Origin: https://example.com/ -# CDN_HOST=https://assets.example.com - -# S3 (optional) -# The attachment host must allow cross origin request from WEB_DOMAIN or -# LOCAL_DOMAIN if WEB_DOMAIN is not set. For example, the server may have the -# following header field: -# Access-Control-Allow-Origin: https://192.168.1.123:9000/ -# S3_ENABLED=true -# S3_BUCKET= -# AWS_ACCESS_KEY_ID= -# AWS_SECRET_ACCESS_KEY= -# S3_REGION= -# S3_PROTOCOL=http -# S3_HOSTNAME=192.168.1.123:9000 - -# S3 (Minio Config (optional) Please check Minio instance for details) -# The attachment host must allow cross origin request - see the description -# above. -# S3_ENABLED=true -# S3_BUCKET= -# AWS_ACCESS_KEY_ID= -# AWS_SECRET_ACCESS_KEY= -# S3_REGION= -# S3_PROTOCOL=https -# S3_HOSTNAME= -# S3_ENDPOINT= -# S3_SIGNATURE_VERSION= - -# Google Cloud Storage (optional) -# Use S3 compatible API. Since GCS does not support Multipart Upload, -# increase the value of S3_MULTIPART_THRESHOLD to disable Multipart Upload. -# The attachment host must allow cross origin request - see the description -# above. -# S3_ENABLED=true -# AWS_ACCESS_KEY_ID= -# AWS_SECRET_ACCESS_KEY= -# S3_REGION= -# S3_PROTOCOL=https -# S3_HOSTNAME=storage.googleapis.com -# S3_ENDPOINT=https://storage.googleapis.com -# S3_MULTIPART_THRESHOLD=52428801 # 50.megabytes - -# Swift (optional) -# The attachment host must allow cross origin request - see the description -# above. -# SWIFT_ENABLED=true -# SWIFT_USERNAME= -# For Keystone V3, the value for SWIFT_TENANT should be the project name -# SWIFT_TENANT= -# SWIFT_PASSWORD= -# Some OpenStack V3 providers require PROJECT_ID (optional) -# SWIFT_PROJECT_ID= -# Keystone V2 and V3 URLs are supported. Use a V3 URL if possible to avoid -# issues with token rate-limiting during high load. -# SWIFT_AUTH_URL= -# SWIFT_CONTAINER= -# SWIFT_OBJECT_URL= -# SWIFT_REGION= -# Defaults to 'default' -# SWIFT_DOMAIN_NAME= -# Defaults to 60 seconds. Set to 0 to disable -# SWIFT_CACHE_TTL= - -# Optional alias for S3 (e.g. to serve files on a custom domain, possibly using Cloudfront or Cloudflare) -# S3_ALIAS_HOST= - -# Streaming API integration -# STREAMING_API_BASE_URL= - -# Advanced settings -# If you need to use pgBouncer, you need to disable prepared statements: -# PREPARED_STATEMENTS=false - -# Cluster number setting for streaming API server. -# If you comment out following line, cluster number will be `numOfCpuCores - 1`. -STREAMING_CLUSTER_NUM=1 - -# Docker mastodon user -# If you use Docker, you may want to assign UID/GID manually. -# UID=1000 -# GID=1000 - -# LDAP authentication (optional) -# LDAP_ENABLED=true -# LDAP_HOST=localhost -# LDAP_PORT=389 -# LDAP_METHOD=simple_tls -# LDAP_BASE= -# LDAP_BIND_DN= -# LDAP_PASSWORD= -# LDAP_UID=cn -# LDAP_MAIL=mail -# LDAP_SEARCH_FILTER=(|(%{uid}=%{email})(%{mail}=%{email})) -# LDAP_UID_CONVERSION_ENABLED=true -# LDAP_UID_CONVERSION_SEARCH=., - -# LDAP_UID_CONVERSION_REPLACE=_ - -# PAM authentication (optional) -# PAM authentication uses for the email generation the "email" pam variable -# and optional as fallback PAM_DEFAULT_SUFFIX -# The pam environment variable "email" is provided by: -# https://github.com/devkral/pam_email_extractor -# PAM_ENABLED=true -# Fallback email domain for email address generation (LOCAL_DOMAIN by default) -# PAM_EMAIL_DOMAIN=example.com -# Name of the pam service (pam "auth" section is evaluated) -# PAM_DEFAULT_SERVICE=rpam -# Name of the pam service used for checking if an user can register (pam "account" section is evaluated) (nil (disabled) by default) -# PAM_CONTROLLED_SERVICE=rpam - -# Global OAuth settings (optional) : -# If you have only one strategy, you may want to enable this -# OAUTH_REDIRECT_AT_SIGN_IN=true - -# Optional CAS authentication (cf. omniauth-cas) : -# CAS_ENABLED=true -# CAS_URL=https://sso.myserver.com/ -# CAS_HOST=sso.myserver.com/ -# CAS_PORT=443 -# CAS_SSL=true -# CAS_VALIDATE_URL= -# CAS_CALLBACK_URL= -# CAS_LOGOUT_URL= -# CAS_LOGIN_URL= -# CAS_UID_FIELD='user' -# CAS_CA_PATH= -# CAS_DISABLE_SSL_VERIFICATION=false -# CAS_UID_KEY='user' -# CAS_NAME_KEY='name' -# CAS_EMAIL_KEY='email' -# CAS_NICKNAME_KEY='nickname' -# CAS_FIRST_NAME_KEY='firstname' -# CAS_LAST_NAME_KEY='lastname' -# CAS_LOCATION_KEY='location' -# CAS_IMAGE_KEY='image' -# CAS_PHONE_KEY='phone' - -# Optional SAML authentication (cf. omniauth-saml) -# SAML_ENABLED=true -# SAML_ACS_URL=http://localhost:3000/auth/auth/saml/callback -# SAML_ISSUER=https://example.com -# SAML_IDP_SSO_TARGET_URL=https://idp.testshib.org/idp/profile/SAML2/Redirect/SSO -# SAML_IDP_CERT= -# SAML_IDP_CERT_FINGERPRINT= -# SAML_NAME_IDENTIFIER_FORMAT= -# SAML_CERT= -# SAML_PRIVATE_KEY= -# SAML_SECURITY_WANT_ASSERTION_SIGNED=true -# SAML_SECURITY_WANT_ASSERTION_ENCRYPTED=true -# SAML_SECURITY_ASSUME_EMAIL_IS_VERIFIED=true -# SAML_ATTRIBUTES_STATEMENTS_UID="urn:oid:0.9.2342.19200300.100.1.1" -# SAML_ATTRIBUTES_STATEMENTS_EMAIL="urn:oid:1.3.6.1.4.1.5923.1.1.1.6" -# SAML_ATTRIBUTES_STATEMENTS_FULL_NAME="urn:oid:2.16.840.1.113730.3.1.241" -# SAML_ATTRIBUTES_STATEMENTS_FIRST_NAME="urn:oid:2.5.4.42" -# SAML_ATTRIBUTES_STATEMENTS_LAST_NAME="urn:oid:2.5.4.4" -# SAML_UID_ATTRIBUTE="urn:oid:0.9.2342.19200300.100.1.1" -# SAML_ATTRIBUTES_STATEMENTS_VERIFIED= -# SAML_ATTRIBUTES_STATEMENTS_VERIFIED_EMAIL= - -# Use HTTP proxy for outgoing request (optional) -# http_proxy=http://gateway.local:8118 -# Access control for hidden service. -# ALLOW_ACCESS_TO_HIDDEN_SERVICE=true - -# Authorized fetch mode (optional) -# Require remote servers to authentify when fetching toots, see -# https://docs.joinmastodon.org/admin/config/#authorized_fetch -# AUTHORIZED_FETCH=true - -# Whitelist mode (optional) -# Only allow federation with whitelisted domains, see -# https://docs.joinmastodon.org/admin/config/#whitelist_mode -# WHITELIST_MODE=true diff --git a/config/initializers/2_whitelist_mode.rb b/config/initializers/2_whitelist_mode.rb index a17ad07a2..1cc6a8e72 100644 --- a/config/initializers/2_whitelist_mode.rb +++ b/config/initializers/2_whitelist_mode.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true Rails.application.configure do - config.x.whitelist_mode = ENV['WHITELIST_MODE'] == 'true' + config.x.whitelist_mode = (ENV['LIMITED_FEDERATION_MODE'] || ENV['WHITELIST_MODE']) == 'true' end diff --git a/config/initializers/blacklists.rb b/config/initializers/blacklists.rb index 020d84f56..0e3339c98 100644 --- a/config/initializers/blacklists.rb +++ b/config/initializers/blacklists.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true Rails.application.configure do - config.x.email_domains_blacklist = ENV.fetch('EMAIL_DOMAIN_BLACKLIST') { 'mvrht.com' } - config.x.email_domains_whitelist = ENV.fetch('EMAIL_DOMAIN_WHITELIST') { '' } + config.x.email_domains_blacklist = (ENV['EMAIL_DOMAIN_DENYLIST'] || ENV['EMAIL_DOMAIN_BLACKLIST']) || '' + config.x.email_domains_whitelist = (ENV['EMAIL_DOMAIN_ALLOWLIST'] || ENV['EMAIL_DOMAIN_WHITELIST']) || '' end diff --git a/config/locales/en.yml b/config/locales/en.yml index 52692129e..7fc58643f 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -95,7 +95,7 @@ en: delete: Delete destroyed_msg: Moderation note successfully destroyed! accounts: - add_email_domain_block: Blacklist e-mail domain + add_email_domain_block: Block e-mail domain approve: Approve approve_all: Approve all are_you_sure: Are you sure? @@ -196,7 +196,7 @@ en: username: Username warn: Warn web: Web - whitelisted: Whitelisted + whitelisted: Allowed for federation action_logs: action_types: assigned_to_self_report: Assign Report @@ -241,15 +241,15 @@ en: create_account_warning: "%{name} sent a warning to %{target}" create_announcement: "%{name} created new announcement %{target}" create_custom_emoji: "%{name} uploaded new emoji %{target}" - create_domain_allow: "%{name} whitelisted domain %{target}" + create_domain_allow: "%{name} allowed federation with domain %{target}" create_domain_block: "%{name} blocked domain %{target}" - create_email_domain_block: "%{name} blacklisted e-mail domain %{target}" + create_email_domain_block: "%{name} blocked e-mail domain %{target}" demote_user: "%{name} demoted user %{target}" destroy_announcement: "%{name} deleted announcement %{target}" destroy_custom_emoji: "%{name} destroyed emoji %{target}" - destroy_domain_allow: "%{name} removed domain %{target} from whitelist" + destroy_domain_allow: "%{name} disallowed federation with domain %{target}" destroy_domain_block: "%{name} unblocked domain %{target}" - destroy_email_domain_block: "%{name} whitelisted e-mail domain %{target}" + destroy_email_domain_block: "%{name} unblocked e-mail domain %{target}" destroy_status: "%{name} removed status by %{target}" disable_2fa_user: "%{name} disabled two factor requirement for user %{target}" disable_custom_emoji: "%{name} disabled emoji %{target}" @@ -350,12 +350,12 @@ en: week_interactions: interactions this week week_users_active: active this week week_users_new: users this week - whitelist_mode: Whitelist mode + whitelist_mode: Limited federation mode domain_allows: - add_new: Whitelist domain - created_msg: Domain has been successfully whitelisted - destroyed_msg: Domain has been removed from the whitelist - undo: Remove from whitelist + add_new: Allow federation with domain + created_msg: Domain has been successfully allowed for federation + destroyed_msg: Domain has been disallowed from federation + undo: Disallow federation with domain domain_blocks: add_new: Add new domain block created_msg: Domain block is now being processed @@ -398,16 +398,16 @@ en: view: View domain block email_domain_blocks: add_new: Add new - created_msg: Successfully added e-mail domain to blacklist + created_msg: Successfully blocked e-mail domain delete: Delete - destroyed_msg: Successfully deleted e-mail domain from blacklist + destroyed_msg: Successfully unblocked e-mail domain domain: Domain - empty: No e-mail domains currently blacklisted. + empty: No e-mail domains currently blocked. from_html: from %{domain} new: create: Add domain - title: New e-mail blacklist entry - title: E-mail blacklist + title: Block new e-mail domain + title: Blocked e-mail domains instances: by_domain: Domain delivery_available: Delivery is available @@ -451,7 +451,7 @@ en: pending: Waiting for relay's approval save_and_enable: Save and enable setup: Setup a relay connection - signatures_not_enabled: Relays will not work correctly while secure mode or whitelist mode is enabled + signatures_not_enabled: Relays will not work correctly while secure mode or limited federation mode is enabled status: Status title: Relays report_notes: diff --git a/lib/mastodon/domains_cli.rb b/lib/mastodon/domains_cli.rb index b5435bb5e..558737c27 100644 --- a/lib/mastodon/domains_cli.rb +++ b/lib/mastodon/domains_cli.rb @@ -16,22 +16,22 @@ module Mastodon option :concurrency, type: :numeric, default: 5, aliases: [:c] option :verbose, type: :boolean, aliases: [:v] option :dry_run, type: :boolean - option :whitelist_mode, type: :boolean + option :limited_federation_mode, type: :boolean desc 'purge [DOMAIN...]', 'Remove accounts from a DOMAIN without a trace' long_desc <<-LONG_DESC Remove all accounts from a given DOMAIN without leaving behind any records. Unlike a suspension, if the DOMAIN still exists in the wild, it means the accounts could return if they are resolved again. - When the --whitelist-mode option is given, instead of purging accounts - from a single domain, all accounts from domains that are not whitelisted + When the --limited-federation-mode option is given, instead of purging accounts + from a single domain, all accounts from domains that have not been explicitly allowed are removed from the database. LONG_DESC def purge(*domains) dry_run = options[:dry_run] ? ' (DRY RUN)' : '' scope = begin - if options[:whitelist_mode] + if options[:limited_federation_mode] Account.remote.where.not(domain: DomainAllow.pluck(:domain)) elsif !domains.empty? Account.remote.where(domain: domains) -- cgit From 64aac3073340dbc92c33f5f1c6f76dcafa77a450 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Mon, 29 Jun 2020 13:56:55 +0200 Subject: Add customizable thumbnails for audio and video attachments (#14145) - Change audio files to not be stripped of metadata - Automatically extract cover art from audio if it exists - Add `thumbnail` parameter to `POST /api/v1/media`, `POST /api/v2/media` and `PUT /api/v1/media/:id` - Add `icon` to represent it in attachments in ActivityPub - Fix `preview_url` containing URL of missing missing image when there is no thumbnail instead of null - Fix duration of audio not being displayed on public pages until the file is loaded --- app/controllers/api/v1/media_controller.rb | 2 +- app/controllers/media_proxy_controller.rb | 4 +- app/controllers/settings/pictures_controller.rb | 13 +-- app/javascript/mastodon/components/status.js | 3 +- app/javascript/mastodon/features/audio/index.js | 42 +++++--- .../features/status/components/detailed_status.js | 3 +- app/lib/activitypub/activity/create.rb | 12 ++- app/models/concerns/remotable.rb | 29 +++--- app/models/media_attachment.rb | 108 ++++++++++++++------- app/serializers/activitypub/note_serializer.rb | 10 ++ .../rest/media_attachment_serializer.rb | 4 +- .../activitypub/process_account_service.rb | 4 +- app/views/statuses/_detailed_status.html.haml | 2 +- app/views/statuses/_simple_status.html.haml | 2 +- app/workers/post_process_media_worker.rb | 2 +- app/workers/redownload_media_worker.rb | 3 +- ...0_add_thumbnail_columns_to_media_attachments.rb | 11 +++ db/schema.rb | 7 +- lib/mastodon/media_cli.rb | 10 +- lib/paperclip/attachment_extensions.rb | 2 +- lib/paperclip/image_extractor.rb | 49 ++++++++++ lib/paperclip/type_corrector.rb | 10 +- spec/models/concerns/remotable_spec.rb | 53 +++------- 23 files changed, 247 insertions(+), 138 deletions(-) create mode 100644 db/migrate/20200627125810_add_thumbnail_columns_to_media_attachments.rb create mode 100644 lib/paperclip/image_extractor.rb (limited to 'lib') diff --git a/app/controllers/api/v1/media_controller.rb b/app/controllers/api/v1/media_controller.rb index 0bb3d0d27..a2a919a3e 100644 --- a/app/controllers/api/v1/media_controller.rb +++ b/app/controllers/api/v1/media_controller.rb @@ -39,7 +39,7 @@ class Api::V1::MediaController < Api::BaseController end def media_attachment_params - params.permit(:file, :description, :focus) + params.permit(:file, :thumbnail, :description, :focus) end def file_type_error diff --git a/app/controllers/media_proxy_controller.rb b/app/controllers/media_proxy_controller.rb index 014b89de1..a8261ec2b 100644 --- a/app/controllers/media_proxy_controller.rb +++ b/app/controllers/media_proxy_controller.rb @@ -28,8 +28,8 @@ class MediaProxyController < ApplicationController private def redownload! - @media_attachment.file_remote_url = @media_attachment.remote_url - @media_attachment.created_at = Time.now.utc + @media_attachment.download_file! + @media_attachment.created_at = Time.now.utc @media_attachment.save! end diff --git a/app/controllers/settings/pictures_controller.rb b/app/controllers/settings/pictures_controller.rb index 73926707b..df2a6eed3 100644 --- a/app/controllers/settings/pictures_controller.rb +++ b/app/controllers/settings/pictures_controller.rb @@ -7,13 +7,8 @@ module Settings before_action :set_picture def destroy - if valid_picture - account_params = { - @picture => nil, - (@picture + '_remote_url') => nil, - } - - msg = UpdateAccountService.new.call(@account, account_params) ? I18n.t('generic.changes_saved_msg') : nil + if valid_picture? + msg = I18n.t('generic.changes_saved_msg') if UpdateAccountService.new.call(@account, { @picture => nil, "#{@picture}_remote_url" => '' }) redirect_to settings_profile_path, notice: msg, status: 303 else bad_request @@ -30,8 +25,8 @@ module Settings @picture = params[:id] end - def valid_picture - @picture == 'avatar' || @picture == 'header' + def valid_picture? + %w(avatar header).include?(@picture) end end end diff --git a/app/javascript/mastodon/components/status.js b/app/javascript/mastodon/components/status.js index 2dc961936..827b69500 100644 --- a/app/javascript/mastodon/components/status.js +++ b/app/javascript/mastodon/components/status.js @@ -352,7 +352,8 @@ class Status extends ImmutablePureComponent { this.handlePosterLoad(img); - img.src = this.props.poster; + if (!this.props.blurhash) { + const img = new Image(); + img.crossOrigin = 'anonymous'; + img.onload = () => this.handlePosterLoad(img); + img.src = this.props.poster; + } else { + this._setColorScheme(); + this._decodeBlurhash(); + } } componentDidUpdate (prevProps, prevState) { - if (prevProps.poster !== this.props.poster) { + if (prevProps.poster !== this.props.poster && !this.props.blurhash) { const img = new Image(); img.crossOrigin = 'anonymous'; img.onload = () => this.handlePosterLoad(img); img.src = this.props.poster; } - if (prevState.blurhash !== this.state.blurhash) { - const context = this.blurhashCanvas.getContext('2d'); - const pixels = decode(this.state.blurhash, 32, 32); - const outputImageData = new ImageData(pixels, 32, 32); - - context.putImageData(outputImageData, 0, 0); + if (prevState.blurhash !== this.state.blurhash || prevProps.blurhash !== this.props.blurhash) { + this._setColorScheme(); + this._decodeBlurhash(); } this._clear(); this._draw(); } + _decodeBlurhash () { + const context = this.blurhashCanvas.getContext('2d'); + const pixels = decode(this.props.blurhash || this.state.blurhash, 32, 32); + const outputImageData = new ImageData(pixels, 32, 32); + + context.putImageData(outputImageData, 0, 0); + } + componentWillUnmount () { window.removeEventListener('scroll', this.handleScroll); window.removeEventListener('resize', this.handleResize); @@ -415,7 +426,7 @@ class Audio extends React.PureComponent { } handlePosterLoad = image => { - const canvas = document.createElement('canvas'); + const canvas = document.createElement('canvas'); const context = canvas.getContext('2d'); canvas.width = image.width; @@ -425,10 +436,15 @@ class Audio extends React.PureComponent { const inputImageData = context.getImageData(0, 0, image.width, image.height); const blurhash = encode(inputImageData.data, image.width, image.height, 4, 4); + + this.setState({ blurhash }); + } + + _setColorScheme () { + const blurhash = this.props.blurhash || this.state.blurhash; const averageColor = decodeRGB(decode83(blurhash.slice(2, 6))); this.setState({ - blurhash, color: adjustColor(averageColor), darkText: luma(averageColor) >= 165, }); diff --git a/app/javascript/mastodon/features/status/components/detailed_status.js b/app/javascript/mastodon/features/status/components/detailed_status.js index 935e4207e..f7d0c9bd4 100644 --- a/app/javascript/mastodon/features/status/components/detailed_status.js +++ b/app/javascript/mastodon/features/status/components/detailed_status.js @@ -125,7 +125,8 @@ class DetailedStatus extends ImmutablePureComponent { src={attachment.get('url')} alt={attachment.get('description')} duration={attachment.getIn(['meta', 'original', 'duration'], 0)} - poster={status.getIn(['account', 'avatar_static'])} + poster={attachment.get('preview_url') || status.getIn(['account', 'avatar_static'])} + blurhash={attachment.get('blurhash')} height={150} /> ); diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index 3509a6c40..d3d460551 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -238,12 +238,13 @@ class ActivityPub::Activity::Create < ActivityPub::Activity begin href = Addressable::URI.parse(attachment['url']).normalize.to_s - media_attachment = MediaAttachment.create(account: @account, remote_url: href, description: attachment['summary'].presence || attachment['name'].presence, focus: attachment['focalPoint'], blurhash: supported_blurhash?(attachment['blurhash']) ? attachment['blurhash'] : nil) + media_attachment = MediaAttachment.create(account: @account, remote_url: href, thumbnail_remote_url: icon_url_from_attachment(attachment), description: attachment['summary'].presence || attachment['name'].presence, focus: attachment['focalPoint'], blurhash: supported_blurhash?(attachment['blurhash']) ? attachment['blurhash'] : nil) media_attachments << media_attachment next if unsupported_media_type?(attachment['mediaType']) || skip_download? - media_attachment.file_remote_url = href + media_attachment.download_file! + media_attachment.download_thumbnail! media_attachment.save rescue Mastodon::UnexpectedResponseError, HTTP::TimeoutError, HTTP::ConnectionError, OpenSSL::SSL::SSLError RedownloadMediaWorker.perform_in(rand(30..600).seconds, media_attachment.id) @@ -256,6 +257,13 @@ class ActivityPub::Activity::Create < ActivityPub::Activity media_attachments end + def icon_url_from_attachment(attachment) + url = attachment['icon'].is_a?(Hash) ? attachment['icon']['url'] : attachment['icon'] + Addressable::URI.parse(url).normalize.to_s if url.present? + rescue Addressable::URI::InvalidURIError + nil + end + def process_poll return unless @object['type'] == 'Question' && (@object['anyOf'].is_a?(Array) || @object['oneOf'].is_a?(Array)) diff --git a/app/models/concerns/remotable.rb b/app/models/concerns/remotable.rb index c728a460e..6fc1dcc26 100644 --- a/app/models/concerns/remotable.rb +++ b/app/models/concerns/remotable.rb @@ -4,12 +4,12 @@ module Remotable extend ActiveSupport::Concern class_methods do - def remotable_attachment(attachment_name, limit, suppress_errors: true) - attribute_name = "#{attachment_name}_remote_url".to_sym - method_name = "#{attribute_name}=".to_sym - alt_method_name = "reset_#{attachment_name}!".to_sym + def remotable_attachment(attachment_name, limit, suppress_errors: true, download_on_assign: true, attribute_name: nil) + attribute_name ||= "#{attachment_name}_remote_url".to_sym + + define_method("download_#{attachment_name}!") do + url = self[attribute_name] - define_method method_name do |url| return if url.blank? begin @@ -18,7 +18,7 @@ module Remotable return end - return if !%w(http https).include?(parsed_url.scheme) || parsed_url.host.blank? || (self[attribute_name] == url && send("#{attachment_name}_file_name").present?) + return if !%w(http https).include?(parsed_url.scheme) || parsed_url.host.blank? begin Request.new(:get, url).perform do |response| @@ -36,10 +36,8 @@ module Remotable basename = SecureRandom.hex(8) - send("#{attachment_name}_file_name=", basename + extname) - send("#{attachment_name}=", StringIO.new(response.body_with_limit(limit))) - - self[attribute_name] = url if has_attribute?(attribute_name) + public_send("#{attachment_name}_file_name=", basename + extname) + public_send("#{attachment_name}=", StringIO.new(response.body_with_limit(limit))) end rescue Mastodon::UnexpectedResponseError, HTTP::TimeoutError, HTTP::ConnectionError, OpenSSL::SSL::SSLError => e Rails.logger.debug "Error fetching remote #{attachment_name}: #{e}" @@ -50,14 +48,15 @@ module Remotable end end - define_method alt_method_name do - url = self[attribute_name] + define_method("#{attribute_name}=") do |url| + return if self[attribute_name] == url && public_send("#{attachment_name}_file_name").present? - return if url.blank? + self[attribute_name] = url - self[attribute_name] = '' - send(method_name, url) + public_send("download_#{attachment_name}!") if download_on_assign end + + alias_method("reset_#{attachment_name}!", "download_#{attachment_name}!") end end diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb index d44467009..f67566a18 100644 --- a/app/models/media_attachment.rb +++ b/app/models/media_attachment.rb @@ -21,6 +21,11 @@ # blurhash :string # processing :integer # file_storage_schema_version :integer +# thumbnail_file_name :string +# thumbnail_content_type :string +# thumbnail_file_size :integer +# thumbnail_updated_at :datetime +# thumbnail_remote_url :string # class MediaAttachment < ApplicationRecord @@ -49,13 +54,13 @@ class MediaAttachment < ApplicationRecord original: { pixels: 1_638_400, # 1280x1280px file_geometry_parser: FastGeometryParser, - }, + }.freeze, small: { pixels: 160_000, # 400x400px file_geometry_parser: FastGeometryParser, blurhash: BLURHASH_OPTIONS, - }, + }.freeze, }.freeze VIDEO_FORMAT = { @@ -74,14 +79,14 @@ class MediaAttachment < ApplicationRecord 'frames:v' => 60 * 60 * 3, 'crf' => 18, 'map_metadata' => '-1', - }, - }, + }.freeze, + }.freeze, }.freeze VIDEO_PASSTHROUGH_OPTIONS = { - video_codecs: ['h264'], - audio_codecs: ['aac', nil], - colorspaces: ['yuv420p'], + video_codecs: ['h264'].freeze, + audio_codecs: ['aac', nil].freeze, + colorspaces: ['yuv420p'].freeze, options: { format: 'mp4', convert_options: { @@ -90,9 +95,9 @@ class MediaAttachment < ApplicationRecord 'map_metadata' => '-1', 'c:v' => 'copy', 'c:a' => 'copy', - }, - }, - }, + }.freeze, + }.freeze, + }.freeze, }.freeze VIDEO_STYLES = { @@ -101,15 +106,15 @@ class MediaAttachment < ApplicationRecord output: { 'loglevel' => 'fatal', vf: 'scale=\'min(400\, iw):min(400\, ih)\':force_original_aspect_ratio=decrease', - }, - }, + }.freeze, + }.freeze, format: 'png', time: 0, file_geometry_parser: FastGeometryParser, blurhash: BLURHASH_OPTIONS, - }, + }.freeze, - original: VIDEO_FORMAT.merge(passthrough_options: VIDEO_PASSTHROUGH_OPTIONS), + original: VIDEO_FORMAT.merge(passthrough_options: VIDEO_PASSTHROUGH_OPTIONS).freeze, }.freeze AUDIO_STYLES = { @@ -119,16 +124,23 @@ class MediaAttachment < ApplicationRecord convert_options: { output: { 'loglevel' => 'fatal', - 'map_metadata' => '-1', 'q:a' => 2, - }, - }, - }, + }.freeze, + }.freeze, + }.freeze, }.freeze VIDEO_CONVERTED_STYLES = { - small: VIDEO_STYLES[:small], - original: VIDEO_FORMAT, + small: VIDEO_STYLES[:small].freeze, + original: VIDEO_FORMAT.freeze, + }.freeze + + THUMBNAIL_STYLES = { + original: IMAGE_STYLES[:small].freeze, + }.freeze + + GLOBAL_CONVERT_OPTIONS = { + all: '-quality 90 -strip +set modify-date +set create-date', }.freeze IMAGE_LIMIT = 10.megabytes @@ -144,18 +156,28 @@ class MediaAttachment < ApplicationRecord has_attached_file :file, styles: ->(f) { file_styles f }, processors: ->(f) { file_processors f }, - convert_options: { all: '-quality 90 -strip +set modify-date +set create-date' } + convert_options: GLOBAL_CONVERT_OPTIONS validates_attachment_content_type :file, content_type: IMAGE_MIME_TYPES + VIDEO_MIME_TYPES + AUDIO_MIME_TYPES validates_attachment_size :file, less_than: IMAGE_LIMIT, unless: :larger_media_format? validates_attachment_size :file, less_than: VIDEO_LIMIT, if: :larger_media_format? - remotable_attachment :file, VIDEO_LIMIT, suppress_errors: false + remotable_attachment :file, VIDEO_LIMIT, suppress_errors: false, download_on_assign: false, attribute_name: :remote_url + + has_attached_file :thumbnail, + styles: THUMBNAIL_STYLES, + processors: [:lazy_thumbnail, :blurhash_transcoder], + convert_options: GLOBAL_CONVERT_OPTIONS + + validates_attachment_content_type :thumbnail, content_type: IMAGE_MIME_TYPES + validates_attachment_size :thumbnail, less_than: IMAGE_LIMIT + remotable_attachment :thumbnail, IMAGE_LIMIT, suppress_errors: true, download_on_assign: false include Attachmentable validates :account, presence: true validates :description, length: { maximum: MAX_DESCRIPTION_LENGTH }, if: :local? validates :file, presence: true, if: :local? + validates :thumbnail, absence: true, if: -> { local? && !audio_or_video? } scope :attached, -> { where.not(status_id: nil).or(where.not(scheduled_status_id: nil)) } scope :unattached, -> { where(status_id: nil, scheduled_status_id: nil) } @@ -215,16 +237,21 @@ class MediaAttachment < ApplicationRecord @delay_processing end + def delay_processing_for_attachment?(attachment_name) + @delay_processing && attachment_name == :file + end + after_commit :enqueue_processing, on: :create after_commit :reset_parent_cache, on: :update before_create :prepare_description, unless: :local? before_create :set_shortcode before_create :set_processing - before_create :set_meta - before_post_process :set_type_and_extension - before_post_process :check_video_dimensions + after_post_process :set_meta + + before_file_post_process :set_type_and_extension + before_file_post_process :check_video_dimensions class << self def supported_mime_types @@ -237,25 +264,25 @@ class MediaAttachment < ApplicationRecord private - def file_styles(f) - if f.instance.file_content_type == 'image/gif' || VIDEO_CONVERTIBLE_MIME_TYPES.include?(f.instance.file_content_type) + def file_styles(attachment) + if attachment.instance.file_content_type == 'image/gif' || VIDEO_CONVERTIBLE_MIME_TYPES.include?(attachment.instance.file_content_type) VIDEO_CONVERTED_STYLES - elsif IMAGE_MIME_TYPES.include?(f.instance.file_content_type) + elsif IMAGE_MIME_TYPES.include?(attachment.instance.file_content_type) IMAGE_STYLES - elsif VIDEO_MIME_TYPES.include?(f.instance.file_content_type) + elsif VIDEO_MIME_TYPES.include?(attachment.instance.file_content_type) VIDEO_STYLES else AUDIO_STYLES end end - def file_processors(f) - if f.file_content_type == 'image/gif' + def file_processors(instance) + if instance.file_content_type == 'image/gif' [:gif_transcoder, :blurhash_transcoder] - elsif VIDEO_MIME_TYPES.include?(f.file_content_type) + elsif VIDEO_MIME_TYPES.include?(instance.file_content_type) [:video_transcoder, :blurhash_transcoder, :type_corrector] - elsif AUDIO_MIME_TYPES.include?(f.file_content_type) - [:transcoder, :type_corrector] + elsif AUDIO_MIME_TYPES.include?(instance.file_content_type) + [:image_extractor, :transcoder, :type_corrector] else [:lazy_thumbnail, :blurhash_transcoder, :type_corrector] end @@ -298,7 +325,7 @@ class MediaAttachment < ApplicationRecord def check_video_dimensions return unless (video? || gifv?) && file.queued_for_write[:original].present? - movie = FFMPEG::Movie.new(file.queued_for_write[:original].path) + movie = ffmpeg_data(file.queued_for_write[:original].path) return unless movie.valid? @@ -317,6 +344,8 @@ class MediaAttachment < ApplicationRecord meta[style] = style == :small || image? ? image_geometry(file) : video_metadata(file) end + meta[:small] = image_geometry(thumbnail.queued_for_write[:original]) if thumbnail.queued_for_write.key?(:original) + meta end @@ -334,7 +363,7 @@ class MediaAttachment < ApplicationRecord end def video_metadata(file) - movie = FFMPEG::Movie.new(file.path) + movie = ffmpeg_data(file.path) return {} unless movie.valid? @@ -347,6 +376,13 @@ class MediaAttachment < ApplicationRecord }.compact end + # We call this method about 3 different times on potentially different + # paths but ultimately the same file, so it makes sense to memoize the + # result while disregarding the path + def ffmpeg_data(path = nil) + @ffmpeg_data ||= FFMPEG::Movie.new(path) + end + def enqueue_processing PostProcessMediaWorker.perform_async(id) if delay_processing? end diff --git a/app/serializers/activitypub/note_serializer.rb b/app/serializers/activitypub/note_serializer.rb index 110621a28..f26fd93a4 100644 --- a/app/serializers/activitypub/note_serializer.rb +++ b/app/serializers/activitypub/note_serializer.rb @@ -167,6 +167,8 @@ class ActivityPub::NoteSerializer < ActivityPub::Serializer attributes :type, :media_type, :url, :name, :blurhash attribute :focal_point, if: :focal_point? + has_one :icon, serializer: ActivityPub::ImageSerializer, if: :thumbnail? + def type 'Document' end @@ -190,6 +192,14 @@ class ActivityPub::NoteSerializer < ActivityPub::Serializer def focal_point [object.file.meta['focus']['x'], object.file.meta['focus']['y']] end + + def icon + object.thumbnail + end + + def thumbnail? + object.thumbnail.present? + end end class MentionSerializer < ActivityPub::Serializer diff --git a/app/serializers/rest/media_attachment_serializer.rb b/app/serializers/rest/media_attachment_serializer.rb index cc10e3001..e65f7acf1 100644 --- a/app/serializers/rest/media_attachment_serializer.rb +++ b/app/serializers/rest/media_attachment_serializer.rb @@ -28,7 +28,9 @@ class REST::MediaAttachmentSerializer < ActiveModel::Serializer def preview_url if object.needs_redownload? media_proxy_url(object.id, :small) - else + elsif object.thumbnail.present? + full_asset_url(object.thumbnail.url(:original)) + elsif object.file.styles.key?(:small) full_asset_url(object.file.url(:small)) end end diff --git a/app/services/activitypub/process_account_service.rb b/app/services/activitypub/process_account_service.rb index f4276cece..85b915ec6 100644 --- a/app/services/activitypub/process_account_service.rb +++ b/app/services/activitypub/process_account_service.rb @@ -89,8 +89,8 @@ class ActivityPub::ProcessAccountService < BaseService end def set_fetchable_attributes! - @account.avatar_remote_url = image_url('icon') unless skip_download? - @account.header_remote_url = image_url('image') unless skip_download? + @account.avatar_remote_url = image_url('icon') || '' unless skip_download? + @account.header_remote_url = image_url('image') || '' unless skip_download? @account.public_key = public_key || '' @account.statuses_count = outbox_total_items if outbox_total_items.present? @account.following_count = following_total_items if following_total_items.present? diff --git a/app/views/statuses/_detailed_status.html.haml b/app/views/statuses/_detailed_status.html.haml index 684dd08d1..d10017db9 100644 --- a/app/views/statuses/_detailed_status.html.haml +++ b/app/views/statuses/_detailed_status.html.haml @@ -33,7 +33,7 @@ = render partial: 'statuses/attachment_list', locals: { attachments: status.media_attachments } - elsif status.media_attachments.first.audio? - audio = status.media_attachments.first - = react_component :audio, src: audio.file.url(:original), poster: full_asset_url(status.account.avatar_static_url), width: 670, height: 380, alt: audio.description, duration: audio.file.meta.dig(:original, :duration) do + = react_component :audio, src: audio.file.url(:original), poster: audio.thumbnail.present? ? audio.thumbnail.url : status.account.avatar_static_url, blurhash: audio.blurhash, width: 670, height: 380, alt: audio.description, duration: audio.file.meta.dig('original', 'duration') do = render partial: 'statuses/attachment_list', locals: { attachments: status.media_attachments } - else = react_component :media_gallery, height: 380, sensitive: status.sensitive?, standalone: true, autoplay: autoplay, media: status.media_attachments.map { |a| ActiveModelSerializers::SerializableResource.new(a, serializer: REST::MediaAttachmentSerializer).as_json } do diff --git a/app/views/statuses/_simple_status.html.haml b/app/views/statuses/_simple_status.html.haml index 06dc5ff93..ab09dfe45 100644 --- a/app/views/statuses/_simple_status.html.haml +++ b/app/views/statuses/_simple_status.html.haml @@ -39,7 +39,7 @@ = render partial: 'statuses/attachment_list', locals: { attachments: status.media_attachments } - elsif status.media_attachments.first.audio? - audio = status.media_attachments.first - = react_component :audio, src: audio.file.url(:original), poster: full_asset_url(status.account.avatar_static_url), width: 610, height: 343, alt: audio.description, duration: audio.file.meta.dig(:original, :duration) do + = react_component :audio, src: audio.file.url(:original), poster: audio.thumbnail.present? ? audio.thumbnail.url : status.account.avatar_static_url, blurhash: audio.blurhash, width: 610, height: 343, alt: audio.description, duration: audio.file.meta.dig('original', 'duration') do = render partial: 'statuses/attachment_list', locals: { attachments: status.media_attachments } - else = react_component :media_gallery, height: 343, sensitive: status.sensitive?, autoplay: autoplay, media: status.media_attachments.map { |a| ActiveModelSerializers::SerializableResource.new(a, serializer: REST::MediaAttachmentSerializer).as_json } do diff --git a/app/workers/post_process_media_worker.rb b/app/workers/post_process_media_worker.rb index 73f9ae2bf..a904f35b1 100644 --- a/app/workers/post_process_media_worker.rb +++ b/app/workers/post_process_media_worker.rb @@ -32,7 +32,7 @@ class PostProcessMediaWorker media_attachment.file.reprocess!(:original) media_attachment.processing = :complete - media_attachment.file_meta = previous_meta + media_attachment.file_meta = previous_meta.merge(media_attachment.file_meta).with_indifferent_access.slice(:focus, :original, :small) media_attachment.save rescue ActiveRecord::RecordNotFound true diff --git a/app/workers/redownload_media_worker.rb b/app/workers/redownload_media_worker.rb index 071501a49..0638cd0f0 100644 --- a/app/workers/redownload_media_worker.rb +++ b/app/workers/redownload_media_worker.rb @@ -11,7 +11,8 @@ class RedownloadMediaWorker return if media_attachment.remote_url.blank? - media_attachment.file_remote_url = media_attachment.remote_url + media_attachment.download_file! + media_attachment.download_thumbnail! media_attachment.save rescue ActiveRecord::RecordNotFound true diff --git a/db/migrate/20200627125810_add_thumbnail_columns_to_media_attachments.rb b/db/migrate/20200627125810_add_thumbnail_columns_to_media_attachments.rb new file mode 100644 index 000000000..f9c87a53c --- /dev/null +++ b/db/migrate/20200627125810_add_thumbnail_columns_to_media_attachments.rb @@ -0,0 +1,11 @@ +class AddThumbnailColumnsToMediaAttachments < ActiveRecord::Migration[5.2] + def up + add_attachment :media_attachments, :thumbnail + add_column :media_attachments, :thumbnail_remote_url, :string + end + + def down + remove_attachment :media_attachments, :thumbnail + remove_column :media_attachments, :thumbnail_remote_url + end +end diff --git a/db/schema.rb b/db/schema.rb index d603d98d7..277eccba7 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2020_06_20_164023) do +ActiveRecord::Schema.define(version: 2020_06_27_125810) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -489,6 +489,11 @@ ActiveRecord::Schema.define(version: 2020_06_20_164023) do t.string "blurhash" t.integer "processing" t.integer "file_storage_schema_version" + t.string "thumbnail_file_name" + t.string "thumbnail_content_type" + t.integer "thumbnail_file_size" + t.datetime "thumbnail_updated_at" + t.string "thumbnail_remote_url" t.index ["account_id"], name: "index_media_attachments_on_account_id" t.index ["scheduled_status_id"], name: "index_media_attachments_on_scheduled_status_id" t.index ["shortcode"], name: "index_media_attachments_on_shortcode", unique: true diff --git a/lib/mastodon/media_cli.rb b/lib/mastodon/media_cli.rb index c95f3410a..2a4e3e379 100644 --- a/lib/mastodon/media_cli.rb +++ b/lib/mastodon/media_cli.rb @@ -31,10 +31,11 @@ module Mastodon processed, aggregate = parallelize_with_progress(MediaAttachment.cached.where.not(remote_url: '').where('created_at < ?', time_ago)) do |media_attachment| next if media_attachment.file.blank? - size = media_attachment.file_file_size + size = media_attachment.file_file_size + (media_attachment.thumbnail_file_size || 0) unless options[:dry_run] media_attachment.file.destroy + media_attachment.thumbnail.destroy media_attachment.save end @@ -227,11 +228,12 @@ module Mastodon next if media_attachment.remote_url.blank? || (!options[:force] && media_attachment.file_file_name.present?) unless options[:dry_run] - media_attachment.file_remote_url = media_attachment.remote_url + media_attachment.reset_file! + media_attachment.reset_thumbnail! media_attachment.save end - media_attachment.file_file_size + media_attachment.file_file_size + (media_attachment.thumbnail_file_size || 0) end say("Downloaded #{processed} media attachments (approx. #{number_to_human_size(aggregate)})#{dry_run}", :green, true) @@ -239,7 +241,7 @@ module Mastodon desc 'usage', 'Calculate disk space consumed by Mastodon' def usage - say("Attachments:\t#{number_to_human_size(MediaAttachment.sum(:file_file_size))} (#{number_to_human_size(MediaAttachment.where(account: Account.local).sum(:file_file_size))} local)") + say("Attachments:\t#{number_to_human_size(MediaAttachment.sum(Arel.sql('COALESCE(file_file_size, 0) + COALESCE(thumbnail_file_size, 0)')))} (#{number_to_human_size(MediaAttachment.where(account: Account.local).sum(Arel.sql('COALESCE(file_file_size, 0) + COALESCE(thumbnail_file_size, 0)')))} local)") say("Custom emoji:\t#{number_to_human_size(CustomEmoji.sum(:image_file_size))} (#{number_to_human_size(CustomEmoji.local.sum(:image_file_size))} local)") say("Preview cards:\t#{number_to_human_size(PreviewCard.sum(:image_file_size))}") say("Avatars:\t#{number_to_human_size(Account.sum(:avatar_file_size))} (#{number_to_human_size(Account.local.sum(:avatar_file_size))} local)") diff --git a/lib/paperclip/attachment_extensions.rb b/lib/paperclip/attachment_extensions.rb index f3e51dbd3..93df0a326 100644 --- a/lib/paperclip/attachment_extensions.rb +++ b/lib/paperclip/attachment_extensions.rb @@ -7,7 +7,7 @@ module Paperclip # usage, and we still want to generate thumbnails straight # away, it's the only style we need to exclude def process_style?(style_name, style_args) - if style_name == :original && instance.respond_to?(:delay_processing?) && instance.delay_processing? + if style_name == :original && instance.respond_to?(:delay_processing_for_attachment?) && instance.delay_processing_for_attachment?(name) false else style_args.empty? || style_args.include?(style_name) diff --git a/lib/paperclip/image_extractor.rb b/lib/paperclip/image_extractor.rb new file mode 100644 index 000000000..114852e8b --- /dev/null +++ b/lib/paperclip/image_extractor.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +require 'mime/types/columnar' + +module Paperclip + class ImageExtractor < Paperclip::Processor + IMAGE_EXTRACTION_OPTIONS = { + convert_options: { + output: { + 'loglevel' => 'fatal', + vf: 'scale=\'min(400\, iw):min(400\, ih)\':force_original_aspect_ratio=decrease', + }.freeze, + }.freeze, + format: 'png', + time: -1, + file_geometry_parser: FastGeometryParser, + }.freeze + + def make + return @file unless options[:style] == :original + + image = begin + begin + Paperclip::Transcoder.make(file, IMAGE_EXTRACTION_OPTIONS.dup, attachment) + rescue Paperclip::Error, ::Av::CommandError + nil + end + end + + unless image.nil? + begin + attachment.instance.thumbnail = image if image.size.positive? + ensure + # Paperclip does not automatically delete the source file of + # a new attachment while working on copies of it, so we need + # to make sure it's cleaned up + + begin + FileUtils.rm(image) + rescue Errno::ENOENT + nil + end + end + end + + @file + end + end +end diff --git a/lib/paperclip/type_corrector.rb b/lib/paperclip/type_corrector.rb index 0b0c10a56..17e2fc5da 100644 --- a/lib/paperclip/type_corrector.rb +++ b/lib/paperclip/type_corrector.rb @@ -5,13 +5,15 @@ require 'mime/types/columnar' module Paperclip class TypeCorrector < Paperclip::Processor def make - target_extension = options[:format] - extension = File.extname(attachment.instance.file_file_name) + return @file unless options[:format] + + target_extension = '.' + options[:format] + extension = File.extname(attachment.instance_read(:file_name)) return @file unless options[:style] == :original && target_extension && extension != target_extension - attachment.instance.file_content_type = options[:content_type] || attachment.instance.file_content_type - attachment.instance.file_file_name = File.basename(attachment.instance.file_file_name, '.*') + '.' + target_extension + attachment.instance_write(:content_type, options[:content_type] || attachment.instance_read(:content_type)) + attachment.instance_write(:file_name, File.basename(attachment.instance_read(:file_name), '.*') + target_extension) @file end diff --git a/spec/models/concerns/remotable_spec.rb b/spec/models/concerns/remotable_spec.rb index 99a60cbf6..6957b044f 100644 --- a/spec/models/concerns/remotable_spec.rb +++ b/spec/models/concerns/remotable_spec.rb @@ -58,7 +58,11 @@ RSpec.describe Remotable do expect(foo).to respond_to(:reset_hoge!) end - describe '#hoge_remote_url' do + it 'defines a method #download_hoge!' do + expect(foo).to respond_to(:download_hoge!) + end + + describe '#hoge_remote_url=' do before do request end @@ -138,8 +142,8 @@ RSpec.describe Remotable do let(:code) { 500 } it 'calls not send' do - expect(foo).not_to receive(:send).with("#{hoge}=", any_args) - expect(foo).not_to receive(:send).with("#{hoge}_file_name=", any_args) + expect(foo).not_to receive(:public_send).with("#{hoge}=", any_args) + expect(foo).not_to receive(:public_send).with("#{hoge}_file_name=", any_args) foo.hoge_remote_url = url end end @@ -159,26 +163,14 @@ RSpec.describe Remotable do allow(SecureRandom).to receive(:hex).and_return(basename) allow(StringIO).to receive(:new).with(anything).and_return(string_io) - expect(foo).to receive(:send).with("#{hoge}=", string_io) - expect(foo).to receive(:send).with("#{hoge}_file_name=", basename + extname) - foo.hoge_remote_url = url - end - end + expect(foo).to receive(:public_send).with("download_#{hoge}!") - context 'if has_attribute?' do - it 'calls foo[attribute_name] = url' do - allow(foo).to receive(:has_attribute?).with(attribute_name).and_return(true) - expect(foo).to receive('[]=').with(attribute_name, url) foo.hoge_remote_url = url - end - end - context 'unless has_attribute?' do - it 'calls not foo[attribute_name] = url' do - allow(foo).to receive(:has_attribute?) - .with(attribute_name).and_return(false) - expect(foo).not_to receive('[]=').with(attribute_name, url) - foo.hoge_remote_url = url + expect(foo).to receive(:public_send).with("#{hoge}=", string_io) + expect(foo).to receive(:public_send).with("#{hoge}_file_name=", basename + extname) + + foo.download_hoge! end end end @@ -205,26 +197,5 @@ RSpec.describe Remotable do end end end - - describe '#reset_hoge!' do - context 'if url.blank?' do - it 'returns nil, without clearing foo[attribute_name] and calling #hoge_remote_url=' do - url = nil - expect(foo).not_to receive(:send).with(:hoge_remote_url=, url) - foo[attribute_name] = url - expect(foo.reset_hoge!).to be_nil - expect(foo[attribute_name]).to be_nil - end - end - - context 'unless url.blank?' do - it 'clears foo[attribute_name] and calls #hoge_remote_url=' do - foo[attribute_name] = url - expect(foo).to receive(:send).with(:hoge_remote_url=, url) - foo.reset_hoge! - expect(foo[attribute_name]).to be '' - end - end - end end end -- cgit