From 8f23726918fd8ded18ce9e55e6199df87abcc7cf Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Thu, 20 Jun 2019 10:52:36 +0200 Subject: Fix converted media being saved with original extension and mime type (#11130) --- lib/paperclip/type_corrector.rb | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 lib/paperclip/type_corrector.rb (limited to 'lib') diff --git a/lib/paperclip/type_corrector.rb b/lib/paperclip/type_corrector.rb new file mode 100644 index 000000000..0b0c10a56 --- /dev/null +++ b/lib/paperclip/type_corrector.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +require 'mime/types/columnar' + +module Paperclip + class TypeCorrector < Paperclip::Processor + def make + target_extension = options[:format] + extension = File.extname(attachment.instance.file_file_name) + + return @file unless options[:style] == :original && target_extension && extension != target_extension + + attachment.instance.file_content_type = options[:content_type] || attachment.instance.file_content_type + attachment.instance.file_file_name = File.basename(attachment.instance.file_file_name, '.*') + '.' + target_extension + + @file + end + end +end -- cgit From 84f945d64c00c73c35a3f9c23b895a4288ff2592 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sat, 22 Jun 2019 01:51:27 +0200 Subject: Bump version to 2.9.1 (#11143) --- CHANGELOG.md | 25 +++++++++++++++++++++++++ app/javascript/mastodon/locales/ar.json | 2 +- lib/mastodon/version.rb | 2 +- 3 files changed, 27 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/CHANGELOG.md b/CHANGELOG.md index c89f35cdf..196a83d67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,31 @@ Changelog All notable changes to this project will be documented in this file. +## [2.9.1] - 2019-06-22 +### Added + +- Add moderation API ([Gargron](https://github.com/tootsuite/mastodon/pull/9387)) +- Add audio uploads ([Gargron](https://github.com/tootsuite/mastodon/pull/11123), [Gargron](https://github.com/tootsuite/mastodon/pull/11141)) + +### Changed + +- Change domain blocks to automatically support subdomains ([Gargron](https://github.com/tootsuite/mastodon/pull/11138)) +- Change Nanobox configuration to bring it up to date ([danhunsaker](https://github.com/tootsuite/mastodon/pull/11083)) + +### Removed + +- Remove expensive counters from federation page in admin UI ([Gargron](https://github.com/tootsuite/mastodon/pull/11139)) + +### Fixed + +- Fix converted media being saved with original extension and mime type ([Gargron](https://github.com/tootsuite/mastodon/pull/11130)) +- Fix layout of identity proofs settings ([acid-chicken](https://github.com/tootsuite/mastodon/pull/11126)) +- Fix active scope only returning suspended users ([ThibG](https://github.com/tootsuite/mastodon/pull/11111)) +- Fix sanitizer making block level elements unreadable ([Gargron](https://github.com/tootsuite/mastodon/pull/10836)) +- Fix label for site theme not being translated in admin UI ([palindromordnilap](https://github.com/tootsuite/mastodon/pull/11121)) +- Fix statuses not being filtered irreversibly in web UI under some circumstances ([ThibG](https://github.com/tootsuite/mastodon/pull/11113)) +- Fix scrolling behaviour in compose form ([ThibG](https://github.com/tootsuite/mastodon/pull/11093)) + ## [2.9.0] - 2019-06-13 ### Added diff --git a/app/javascript/mastodon/locales/ar.json b/app/javascript/mastodon/locales/ar.json index 1586580a8..d05c61f98 100644 --- a/app/javascript/mastodon/locales/ar.json +++ b/app/javascript/mastodon/locales/ar.json @@ -369,7 +369,7 @@ "trends.count_by_accounts": "{count} {rawCount, plural, one {person} آخرون {people}} يتحدثون", "ui.beforeunload": "سوف تفقد مسودتك إن تركت ماستدون.", "upload_area.title": "اسحب ثم أفلت للرفع", - "upload_button.label": "إضافة وسائط (JPEG، PNG، GIF، WebM، MP4، MOV)", + "upload_button.label": "إضافة وسائط ({formats})", "upload_error.limit": "لقد تم بلوغ الحد الأقصى المسموح به لإرسال الملفات.", "upload_error.poll": "لا يمكن إدراج ملفات في استطلاعات الرأي.", "upload_form.description": "وصف للمعاقين بصريا", diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb index 02b831084..25dde5644 100644 --- a/lib/mastodon/version.rb +++ b/lib/mastodon/version.rb @@ -13,7 +13,7 @@ module Mastodon end def patch - 0 + 1 end def pre -- cgit From b5c772c3d42610894bd274de633acad492b53cdb Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sat, 22 Jun 2019 17:28:26 +0200 Subject: Bump version to 2.9.2 (#11152) --- CHANGELOG.md | 14 ++++++++++++++ lib/mastodon/version.rb | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/CHANGELOG.md b/CHANGELOG.md index 196a83d67..5ea7e417b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,20 @@ Changelog All notable changes to this project will be documented in this file. +## [2.9.2] - 2019-06-22 +### Added + +- Add `short_description` and `approval_required` to `GET /api/v1/instance` (#11146) + +### Changed + +- Change camera icon to paperclip icon in upload form (#11149) + +### Fixed + +- Fix audio-only OGG and WebM files not being processed as such (#11151) +- Fix audio not being downloaded from remote servers (#11145) + ## [2.9.1] - 2019-06-22 ### Added diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb index 25dde5644..47eac2432 100644 --- a/lib/mastodon/version.rb +++ b/lib/mastodon/version.rb @@ -13,7 +13,7 @@ module Mastodon end def patch - 1 + 2 end def pre -- cgit From e64e6a03dd1e0978fee48f0596dcfbc7fd29958f Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Fri, 28 Jun 2019 15:54:10 +0200 Subject: Add categories for custom emojis (#11196) Fix #7940 --- app/controllers/api/v1/custom_emojis_controller.rb | 2 +- .../compose/components/emoji_picker_dropdown.js | 30 ++++++++++++---------- app/javascript/mastodon/features/emoji/emoji.js | 3 +++ app/models/custom_emoji.rb | 2 ++ app/models/custom_emoji_category.rb | 15 +++++++++++ app/serializers/rest/custom_emoji_serializer.rb | 10 ++++++++ ...0190627222225_create_custom_emoji_categories.rb | 9 +++++++ ...90627222826_add_category_id_to_custom_emojis.rb | 5 ++++ db/schema.rb | 10 +++++++- lib/mastodon/emoji_cli.rb | 6 +++++ .../custom_emoji_category_fabricator.rb | 3 +++ spec/models/custom_emoji_category_spec.rb | 5 ++++ yarn.lock | 4 +-- 13 files changed, 86 insertions(+), 18 deletions(-) create mode 100644 app/models/custom_emoji_category.rb create mode 100644 db/migrate/20190627222225_create_custom_emoji_categories.rb create mode 100644 db/migrate/20190627222826_add_category_id_to_custom_emojis.rb create mode 100644 spec/fabricators/custom_emoji_category_fabricator.rb create mode 100644 spec/models/custom_emoji_category_spec.rb (limited to 'lib') diff --git a/app/controllers/api/v1/custom_emojis_controller.rb b/app/controllers/api/v1/custom_emojis_controller.rb index 1bb19a09d..b6877fb3c 100644 --- a/app/controllers/api/v1/custom_emojis_controller.rb +++ b/app/controllers/api/v1/custom_emojis_controller.rb @@ -7,7 +7,7 @@ class Api::V1::CustomEmojisController < Api::BaseController def index render_cached_json('api:v1:custom_emojis', expires_in: 1.minute) do - ActiveModelSerializers::SerializableResource.new(CustomEmoji.local.where(disabled: false), each_serializer: REST::CustomEmojiSerializer) + ActiveModelSerializers::SerializableResource.new(CustomEmoji.local.where(disabled: false).includes(:category), each_serializer: REST::CustomEmojiSerializer) end end end diff --git a/app/javascript/mastodon/features/compose/components/emoji_picker_dropdown.js b/app/javascript/mastodon/features/compose/components/emoji_picker_dropdown.js index c1429c756..e57c3c20c 100644 --- a/app/javascript/mastodon/features/compose/components/emoji_picker_dropdown.js +++ b/app/javascript/mastodon/features/compose/components/emoji_picker_dropdown.js @@ -6,7 +6,7 @@ import Overlay from 'react-overlays/lib/Overlay'; import classNames from 'classnames'; import ImmutablePropTypes from 'react-immutable-proptypes'; import detectPassiveEvents from 'detect-passive-events'; -import { buildCustomEmojis } from '../../emoji/emoji'; +import { buildCustomEmojis, categoriesFromEmojis } from '../../emoji/emoji'; const messages = defineMessages({ emoji: { id: 'emoji_button.label', defaultMessage: 'Insert emoji' }, @@ -31,19 +31,6 @@ let EmojiPicker, Emoji; // load asynchronously const backgroundImageFn = () => `${assetHost}/emoji/sheet_10.png`; const listenerOptions = detectPassiveEvents.hasSupport ? { passive: true } : false; -const categoriesSort = [ - 'recent', - 'custom', - 'people', - 'nature', - 'foods', - 'activity', - 'places', - 'objects', - 'symbols', - 'flags', -]; - class ModifierPickerMenu extends React.PureComponent { static propTypes = { @@ -241,8 +228,23 @@ class EmojiPickerMenu extends React.PureComponent { } const title = intl.formatMessage(messages.emoji); + const { modifierOpen } = this.state; + const categoriesSort = [ + 'recent', + 'people', + 'nature', + 'foods', + 'activity', + 'places', + 'objects', + 'symbols', + 'flags', + ]; + + categoriesSort.splice(1, 0, ...Array.from(categoriesFromEmojis(custom_emojis)).sort()); + return (
{ keywords: [name], imageUrl: url, custom: true, + customCategory: emoji.get('category'), }); }); return emojis; }; + +export const categoriesFromEmojis = customEmojis => customEmojis.reduce((set, emoji) => set.add(emoji.get('category') ? `custom-${emoji.get('category')}` : 'custom'), new Set()); diff --git a/app/models/custom_emoji.rb b/app/models/custom_emoji.rb index e73cd9bd2..514cf4845 100644 --- a/app/models/custom_emoji.rb +++ b/app/models/custom_emoji.rb @@ -16,6 +16,7 @@ # uri :string # image_remote_url :string # visible_in_picker :boolean default(TRUE), not null +# category_id :bigint(8) # class CustomEmoji < ApplicationRecord @@ -27,6 +28,7 @@ class CustomEmoji < ApplicationRecord :(#{SHORTCODE_RE_FRAGMENT}): (?=[^[:alnum:]:]|$)/x + belongs_to :category, class_name: 'CustomEmojiCategory', optional: true has_one :local_counterpart, -> { where(domain: nil) }, class_name: 'CustomEmoji', primary_key: :shortcode, foreign_key: :shortcode has_attached_file :image, styles: { static: { format: 'png', convert_options: '-coalesce -strip' } } diff --git a/app/models/custom_emoji_category.rb b/app/models/custom_emoji_category.rb new file mode 100644 index 000000000..7d8c0ee2d --- /dev/null +++ b/app/models/custom_emoji_category.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +# == Schema Information +# +# Table name: custom_emoji_categories +# +# id :bigint(8) not null, primary key +# name :string +# created_at :datetime not null +# updated_at :datetime not null +# + +class CustomEmojiCategory < ApplicationRecord + has_many :emojis, class_name: 'CustomEmoji', foreign_key: 'category_id', inverse_of: :category +end diff --git a/app/serializers/rest/custom_emoji_serializer.rb b/app/serializers/rest/custom_emoji_serializer.rb index 65686a866..aff58e4d9 100644 --- a/app/serializers/rest/custom_emoji_serializer.rb +++ b/app/serializers/rest/custom_emoji_serializer.rb @@ -5,6 +5,8 @@ class REST::CustomEmojiSerializer < ActiveModel::Serializer attributes :shortcode, :url, :static_url, :visible_in_picker + attribute :category, if: :category_loaded? + def url full_asset_url(object.image.url) end @@ -12,4 +14,12 @@ class REST::CustomEmojiSerializer < ActiveModel::Serializer def static_url full_asset_url(object.image.url(:static)) end + + def category + object.category.name + end + + def category_loaded? + object.association(:category).loaded? && object.category.present? + end end diff --git a/db/migrate/20190627222225_create_custom_emoji_categories.rb b/db/migrate/20190627222225_create_custom_emoji_categories.rb new file mode 100644 index 000000000..4713793e6 --- /dev/null +++ b/db/migrate/20190627222225_create_custom_emoji_categories.rb @@ -0,0 +1,9 @@ +class CreateCustomEmojiCategories < ActiveRecord::Migration[5.2] + def change + create_table :custom_emoji_categories do |t| + t.string :name, index: { unique: true } + + t.timestamps + end + end +end diff --git a/db/migrate/20190627222826_add_category_id_to_custom_emojis.rb b/db/migrate/20190627222826_add_category_id_to_custom_emojis.rb new file mode 100644 index 000000000..873b4d05f --- /dev/null +++ b/db/migrate/20190627222826_add_category_id_to_custom_emojis.rb @@ -0,0 +1,5 @@ +class AddCategoryIdToCustomEmojis < ActiveRecord::Migration[5.2] + def change + add_column :custom_emojis, :category_id, :bigint + end +end diff --git a/db/schema.rb b/db/schema.rb index f633f4e3f..09e6c9fae 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2019_05_29_143559) do +ActiveRecord::Schema.define(version: 2019_06_27_222826) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -208,6 +208,13 @@ ActiveRecord::Schema.define(version: 2019_05_29_143559) do t.index ["uri"], name: "index_conversations_on_uri", unique: true end + create_table "custom_emoji_categories", force: :cascade do |t| + t.string "name" + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["name"], name: "index_custom_emoji_categories_on_name", unique: true + end + create_table "custom_emojis", force: :cascade do |t| t.string "shortcode", default: "", null: false t.string "domain" @@ -221,6 +228,7 @@ ActiveRecord::Schema.define(version: 2019_05_29_143559) do t.string "uri" t.string "image_remote_url" t.boolean "visible_in_picker", default: true, null: false + t.bigint "category_id" t.index ["shortcode", "domain"], name: "index_custom_emojis_on_shortcode_and_domain", unique: true end diff --git a/lib/mastodon/emoji_cli.rb b/lib/mastodon/emoji_cli.rb index 97a822e45..beac1b1fd 100644 --- a/lib/mastodon/emoji_cli.rb +++ b/lib/mastodon/emoji_cli.rb @@ -15,6 +15,7 @@ module Mastodon option :suffix option :overwrite, type: :boolean option :unlisted, type: :boolean + option :category desc 'import PATH', 'Import emoji from a TAR GZIP archive at PATH' long_desc <<-LONG_DESC Imports custom emoji from a TAR GZIP archive specified by PATH. @@ -22,6 +23,9 @@ module Mastodon Existing emoji will be skipped unless the --overwrite option is provided, in which case they will be overwritten. + You can specifiy a --category under which the emojis will be + grouped together. + With the --prefix option, a prefix can be added to all generated shortcodes. Likewise, the --suffix option controls the suffix of all shortcodes. @@ -33,6 +37,7 @@ module Mastodon imported = 0 skipped = 0 failed = 0 + category = options[:category] ? CustomEmojiCategory.find_or_create_by(name: options[:category]) : nil Gem::Package::TarReader.new(Zlib::GzipReader.open(path)) do |tar| tar.each do |entry| @@ -50,6 +55,7 @@ module Mastodon custom_emoji.image = StringIO.new(entry.read) custom_emoji.image_file_name = File.basename(entry.full_name) custom_emoji.visible_in_picker = !options[:unlisted] + custom_emoji.category = category if custom_emoji.save imported += 1 diff --git a/spec/fabricators/custom_emoji_category_fabricator.rb b/spec/fabricators/custom_emoji_category_fabricator.rb new file mode 100644 index 000000000..f593b95ed --- /dev/null +++ b/spec/fabricators/custom_emoji_category_fabricator.rb @@ -0,0 +1,3 @@ +Fabricator(:custom_emoji_category) do + name "MyString" +end diff --git a/spec/models/custom_emoji_category_spec.rb b/spec/models/custom_emoji_category_spec.rb new file mode 100644 index 000000000..160033f4d --- /dev/null +++ b/spec/models/custom_emoji_category_spec.rb @@ -0,0 +1,5 @@ +require 'rails_helper' + +RSpec.describe CustomEmojiCategory, type: :model do + pending "add some examples to (or delete) #{__FILE__}" +end diff --git a/yarn.lock b/yarn.lock index 560d84e31..2c402782c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3384,8 +3384,8 @@ elliptic@^6.0.0: minimalistic-crypto-utils "^1.0.0" emoji-mart@Gargron/emoji-mart#build: - version "2.6.2" - resolved "https://codeload.github.com/Gargron/emoji-mart/tar.gz/ff00dc470b5b2d9f145a6d6e977a54de5df2b4c9" + version "2.6.3" + resolved "https://codeload.github.com/Gargron/emoji-mart/tar.gz/934f314fd8322276765066e8a2a6be5bac61b1cf" emoji-regex@^7.0.1, emoji-regex@^7.0.2: version "7.0.3" -- cgit From 20520e6158aaa732c6166ec8e6461d08a081e7c6 Mon Sep 17 00:00:00 2001 From: Thibaut Girka Date: Wed, 17 Jul 2019 21:54:03 +0200 Subject: When cleaning up remote statuses, keep bookmarked ones --- lib/mastodon/statuses_cli.rb | 1 + 1 file changed, 1 insertion(+) (limited to 'lib') diff --git a/lib/mastodon/statuses_cli.rb b/lib/mastodon/statuses_cli.rb index 7f2fbfa85..6271c99df 100644 --- a/lib/mastodon/statuses_cli.rb +++ b/lib/mastodon/statuses_cli.rb @@ -41,6 +41,7 @@ module Mastodon .where('id NOT IN (SELECT status_pins.status_id FROM status_pins WHERE statuses.id = status_id)') # Skip statuses that are pinned on profiles .where('id NOT IN (SELECT mentions.status_id FROM mentions WHERE statuses.id = mentions.status_id AND mentions.account_id IN (SELECT accounts.id FROM accounts WHERE domain IS NULL))') # Skip statuses that mention local accounts .where('id NOT IN (SELECT statuses1.in_reply_to_id FROM statuses AS statuses1 WHERE statuses.id = statuses1.in_reply_to_id)') # Skip statuses favourited by local accounts + .where('id NOT IN (SELECT bookmarks.status_id FROM bookmarks WHERE statuses.id = bookmarks.status_id') # Skip statuses bookmarked by local users .where('id NOT IN (SELECT statuses1.reblog_of_id FROM statuses AS statuses1 WHERE statuses.id = statuses1.reblog_of_id AND statuses1.account_id IN (SELECT accounts.id FROM accounts WHERE accounts.domain IS NULL))') # Skip statuses reblogged by local accounts .where('account_id NOT IN (SELECT follows.target_account_id FROM follows WHERE statuses.account_id = follows.target_account_id)') # Skip accounts followed by local accounts .in_batches -- cgit From e3a551cbd1f894fa779dc55168e56bb5f07b3d72 Mon Sep 17 00:00:00 2001 From: Thomas Citharel Date: Wed, 24 Jul 2019 15:53:35 +0200 Subject: Fix tootctl statuses remove A ending parenthesis was missing in the bookmark special line. --- lib/mastodon/statuses_cli.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/mastodon/statuses_cli.rb b/lib/mastodon/statuses_cli.rb index 6271c99df..eeedc026c 100644 --- a/lib/mastodon/statuses_cli.rb +++ b/lib/mastodon/statuses_cli.rb @@ -41,7 +41,7 @@ module Mastodon .where('id NOT IN (SELECT status_pins.status_id FROM status_pins WHERE statuses.id = status_id)') # Skip statuses that are pinned on profiles .where('id NOT IN (SELECT mentions.status_id FROM mentions WHERE statuses.id = mentions.status_id AND mentions.account_id IN (SELECT accounts.id FROM accounts WHERE domain IS NULL))') # Skip statuses that mention local accounts .where('id NOT IN (SELECT statuses1.in_reply_to_id FROM statuses AS statuses1 WHERE statuses.id = statuses1.in_reply_to_id)') # Skip statuses favourited by local accounts - .where('id NOT IN (SELECT bookmarks.status_id FROM bookmarks WHERE statuses.id = bookmarks.status_id') # Skip statuses bookmarked by local users + .where('id NOT IN (SELECT bookmarks.status_id FROM bookmarks WHERE statuses.id = bookmarks.status_id)') # Skip statuses bookmarked by local users .where('id NOT IN (SELECT statuses1.reblog_of_id FROM statuses AS statuses1 WHERE statuses.id = statuses1.reblog_of_id AND statuses1.account_id IN (SELECT accounts.id FROM accounts WHERE accounts.domain IS NULL))') # Skip statuses reblogged by local accounts .where('account_id NOT IN (SELECT follows.target_account_id FROM follows WHERE statuses.account_id = follows.target_account_id)') # Skip accounts followed by local accounts .in_batches -- cgit From a6b44401156524ef70a5917d17c93886c5e20a0d Mon Sep 17 00:00:00 2001 From: Clar Fon Date: Fri, 26 Jul 2019 01:57:27 -0400 Subject: Remove pre from version, add extra suffix variable (#11407) --- lib/mastodon/version.rb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb index 47eac2432..3db57ceaa 100644 --- a/lib/mastodon/version.rb +++ b/lib/mastodon/version.rb @@ -16,20 +16,20 @@ module Mastodon 2 end - def pre - nil + def flags + '' end - def flags + def suffix '' end def to_a - [major, minor, patch, pre].compact + [major, minor, patch].compact end def to_s - [to_a.join('.'), flags].join + [to_a.join('.'), flags, suffix].join end def repository -- cgit From 0d80f686d81e4a12db1ac54b8f31f69f375dc8dc Mon Sep 17 00:00:00 2001 From: mayaeh Date: Sun, 28 Jul 2019 20:48:19 +0900 Subject: Add `tootctl preview_cards remove` (#11320) * Add `tootctl preview_cards remove` * fix code style * Remove `Scheduler::PreviewCardsCleanupScheduler` file * fix code style again Add exclude case where image_file_name is blank * Added a function to output confirmation if the specified number of days is less than 2 weeks --- .../scheduler/preview_cards_cleanup_scheduler.rb | 22 ----- config/sidekiq.yml | 3 - lib/cli.rb | 4 + lib/mastodon/preview_cards_cli.rb | 94 ++++++++++++++++++++++ 4 files changed, 98 insertions(+), 25 deletions(-) delete mode 100644 app/workers/scheduler/preview_cards_cleanup_scheduler.rb create mode 100644 lib/mastodon/preview_cards_cli.rb (limited to 'lib') diff --git a/app/workers/scheduler/preview_cards_cleanup_scheduler.rb b/app/workers/scheduler/preview_cards_cleanup_scheduler.rb deleted file mode 100644 index 2b38792f0..000000000 --- a/app/workers/scheduler/preview_cards_cleanup_scheduler.rb +++ /dev/null @@ -1,22 +0,0 @@ -# frozen_string_literal: true - -class Scheduler::PreviewCardsCleanupScheduler - include Sidekiq::Worker - - sidekiq_options unique: :until_executed, retry: 0 - - def perform - Maintenance::UncachePreviewWorker.push_bulk(recent_link_preview_cards.pluck(:id)) - Maintenance::UncachePreviewWorker.push_bulk(older_preview_cards.pluck(:id)) - end - - private - - def recent_link_preview_cards - PreviewCard.where(type: :link).where('updated_at < ?', 1.month.ago) - end - - def older_preview_cards - PreviewCard.where('updated_at < ?', 6.months.ago) - end -end diff --git a/config/sidekiq.yml b/config/sidekiq.yml index 7f41b6607..6ebe450b0 100644 --- a/config/sidekiq.yml +++ b/config/sidekiq.yml @@ -24,9 +24,6 @@ ip_cleanup_scheduler: cron: '<%= Random.rand(0..59) %> <%= Random.rand(3..5) %> * * *' class: Scheduler::IpCleanupScheduler - preview_cards_cleanup_scheduler: - cron: '<%= Random.rand(0..59) %> <%= Random.rand(3..5) %> * * *' - class: Scheduler::PreviewCardsCleanupScheduler email_scheduler: cron: '0 10 * * 2' class: Scheduler::EmailScheduler diff --git a/lib/cli.rb b/lib/cli.rb index be276583d..fbdf49fc3 100644 --- a/lib/cli.rb +++ b/lib/cli.rb @@ -9,6 +9,7 @@ require_relative 'mastodon/search_cli' require_relative 'mastodon/settings_cli' require_relative 'mastodon/statuses_cli' require_relative 'mastodon/domains_cli' +require_relative 'mastodon/preview_cards_cli' require_relative 'mastodon/cache_cli' require_relative 'mastodon/version' @@ -42,6 +43,9 @@ module Mastodon desc 'domains SUBCOMMAND ...ARGS', 'Manage account domains' subcommand 'domains', Mastodon::DomainsCLI + desc 'preview_cards SUBCOMMAND ...ARGS', 'Manage preview cards' + subcommand 'preview_cards', Mastodon::PreviewCardsCLI + desc 'cache SUBCOMMAND ...ARGS', 'Manage cache' subcommand 'cache', Mastodon::CacheCLI diff --git a/lib/mastodon/preview_cards_cli.rb b/lib/mastodon/preview_cards_cli.rb new file mode 100644 index 000000000..465fe7d0b --- /dev/null +++ b/lib/mastodon/preview_cards_cli.rb @@ -0,0 +1,94 @@ +# frozen_string_literal: true + +require 'tty-prompt' +require_relative '../../config/boot' +require_relative '../../config/environment' +require_relative 'cli_helper' + +module Mastodon + class PreviewCardsCLI < Thor + include ActionView::Helpers::NumberHelper + + def self.exit_on_failure? + true + end + + option :days, type: :numeric, default: 180 + option :background, type: :boolean, default: false + option :verbose, type: :boolean, default: false + option :dry_run, type: :boolean, default: false + option :link, type: :boolean, default: false + desc 'remove', 'Remove preview cards' + long_desc <<-DESC + Removes locally thumbnails for previews. + + The --days option specifies how old preview cards have to be before + they are removed. It defaults to 180 days. + + With the --background option, instead of deleting the files sequentially, + they will be queued into Sidekiq and the command will exit as soon as + possible. In Sidekiq they will be processed with higher concurrency, but + it may impact other operations of the Mastodon server, and it may overload + the underlying file storage. + + With the --dry-run option, no work will be done. + + With the --verbose option, when preview cards are processed sequentially in the + foreground, the IDs of the preview cards will be printed. + + With the --link option, delete only link-type preview cards. + DESC + def remove + prompt = TTY::Prompt.new + time_ago = options[:days].days.ago + queued = 0 + processed = 0 + size = 0 + dry_run = options[:dry_run] ? '(DRY RUN)' : '' + link = options[:link] ? 'link-type ' : '' + scope = PreviewCard.where.not(image_file_name: nil) + scope = scope.where.not(image_file_name: '') + scope = scope.where(type: :link) if options[:link] + scope = scope.where('updated_at < ?', time_ago) + + if time_ago > 2.weeks.ago + prompt.say "\n" + prompt.say('The preview cards less than the past two weeks will not be re-acquired even when needed.') + prompt.say "\n" + + unless prompt.yes?('Are you sure you want to delete the preview cards?', default: false) + prompt.say "\n" + prompt.warn 'Nothing execute. Bye!' + prompt.say "\n" + exit(1) + end + end + + if options[:background] + scope.select(:id, :image_file_size).reorder(nil).find_in_batches do |preview_cards| + queued += preview_cards.size + size += preview_cards.reduce(0) { |sum, p| sum + (p.image_file_size || 0) } + Maintenance::UncachePreviewWorker.push_bulk(preview_cards.map(&:id)) unless options[:dry_run] + end + + else + scope.select(:id, :image_file_size).reorder(nil).find_in_batches do |preview_cards| + preview_cards.each do |p| + size += p.image_file_size || 0 + Maintenance::UncachePreviewWorker.new.perform(p.id) unless options[:dry_run] + options[:verbose] ? say(p.id) : say('.', :green, false) + processed += 1 + end + end + end + + say + + if options[:background] + say("Scheduled the deletion of #{queued} #{link}preview cards (approx. #{number_to_human_size(size)}) #{dry_run}", :green, true) + else + say("Removed #{processed} #{link}preview cards (approx. #{number_to_human_size(size)}) #{dry_run}", :green, true) + end + end + end +end -- cgit From 15de24a425cb07efd58db5299b043e4110fa138f Mon Sep 17 00:00:00 2001 From: "dependabot-preview[bot]" <27856297+dependabot-preview[bot]@users.noreply.github.com> Date: Sun, 28 Jul 2019 13:48:43 +0200 Subject: Bump json-ld-preloaded from 3.0.2 to 3.0.3 (#11316) * Bump json-ld-preloaded from 3.0.2 to 3.0.3 Bumps [json-ld-preloaded](https://github.com/ruby-rdf/json-ld-preloaded) from 3.0.2 to 3.0.3. - [Release notes](https://github.com/ruby-rdf/json-ld-preloaded/releases) - [Commits](https://github.com/ruby-rdf/json-ld-preloaded/compare/3.0.2...3.0.3) Signed-off-by: dependabot-preview[bot] * use json-ld edge --- Gemfile | 2 +- Gemfile.lock | 23 +++++++++++++++++------ app/helpers/jsonld_helper.rb | 2 +- lib/json_ld/security.rb | 5 +++-- 4 files changed, 22 insertions(+), 10 deletions(-) (limited to 'lib') diff --git a/Gemfile b/Gemfile index 96eb44af7..04ebc4cd8 100644 --- a/Gemfile +++ b/Gemfile @@ -92,7 +92,7 @@ gem 'tzinfo-data', '~> 1.2019' gem 'webpacker', '~> 4.0' gem 'webpush' -gem 'json-ld', '~> 3.0' +gem 'json-ld', git: 'https://github.com/ruby-rdf/json-ld.git', ref: '345b7a5733308af827e8491d284dbafa9128d7a2' gem 'json-ld-preloaded', '~> 3.0' gem 'rdf-normalize', '~> 0.3' diff --git a/Gemfile.lock b/Gemfile.lock index 5adae6f9c..8da261639 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -5,6 +5,19 @@ GIT specs: posix-spawn (0.3.13) +GIT + remote: https://github.com/ruby-rdf/json-ld.git + revision: 345b7a5733308af827e8491d284dbafa9128d7a2 + ref: 345b7a5733308af827e8491d284dbafa9128d7a2 + specs: + json-ld (3.0.2) + htmlentities (~> 4.3) + json-canonicalization (~> 0.1) + link_header (~> 0.0, >= 0.0.8) + multi_json (~> 1.13) + rack (>= 1.6, < 3.0) + rdf (~> 3.0, >= 3.0.8) + GIT remote: https://github.com/tmm1/http_parser.rb revision: 54b17ba8c7d8d20a16dfc65d1775241833219cf2 @@ -297,10 +310,8 @@ GEM jaro_winkler (1.5.3) jmespath (1.4.0) json (2.2.0) - json-ld (3.0.2) - multi_json (~> 1.12) - rdf (>= 2.2.8, < 4.0) - json-ld-preloaded (3.0.2) + json-canonicalization (0.1.0) + json-ld-preloaded (3.0.3) json-ld (~> 3.0) multi_json (~> 1.12) rdf (~> 3.0) @@ -479,7 +490,7 @@ GEM thor (>= 0.19.0, < 2.0) rainbow (3.0.0) rake (12.3.2) - rdf (3.0.9) + rdf (3.0.12) hamster (~> 3.0) link_header (~> 0.0, >= 0.0.8) rdf-normalize (0.3.3) @@ -700,7 +711,7 @@ DEPENDENCIES i18n-tasks (~> 0.9) idn-ruby iso-639 - json-ld (~> 3.0) + json-ld! json-ld-preloaded (~> 3.0) kaminari (~> 1.1) letter_opener (~> 1.7) diff --git a/app/helpers/jsonld_helper.rb b/app/helpers/jsonld_helper.rb index 83a5b2462..1c473efa3 100644 --- a/app/helpers/jsonld_helper.rb +++ b/app/helpers/jsonld_helper.rb @@ -130,7 +130,7 @@ module JsonLdHelper end end - doc = JSON::LD::API::RemoteDocument.new(url, json) + doc = JSON::LD::API::RemoteDocument.new(json, documentUrl: url) block_given? ? yield(doc) : doc end diff --git a/lib/json_ld/security.rb b/lib/json_ld/security.rb index 1230206f0..a6fbce95f 100644 --- a/lib/json_ld/security.rb +++ b/lib/json_ld/security.rb @@ -1,9 +1,9 @@ # -*- encoding: utf-8 -*- # frozen_string_literal: true -# This file generated automatically from https://w3id.org/security/v1 +# This file generated automatically from http://w3id.org/security/v1 require 'json/ld' class JSON::LD::Context - add_preloaded("https://w3id.org/security/v1") do + add_preloaded("http://w3id.org/security/v1") do new(processingMode: "json-ld-1.0", term_definitions: { "CryptographicKey" => TermDefinition.new("CryptographicKey", id: "https://w3id.org/security#Key", simple: true), "EcdsaKoblitzSignature2016" => TermDefinition.new("EcdsaKoblitzSignature2016", id: "https://w3id.org/security#EcdsaKoblitzSignature2016", simple: true), @@ -47,4 +47,5 @@ class JSON::LD::Context "xsd" => TermDefinition.new("xsd", id: "http://www.w3.org/2001/XMLSchema#", simple: true, prefix: true) }) end + alias_preloaded("https://w3id.org/security/v1", "http://w3id.org/security/v1") end -- cgit From 24552b5160a5090e7d6056fb69a209aa48fe4fce Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 30 Jul 2019 11:10:46 +0200 Subject: Add whitelist mode (#11291) --- app/controllers/about_controller.rb | 5 +++ app/controllers/activitypub/base_controller.rb | 2 ++ app/controllers/activitypub/inboxes_controller.rb | 2 +- app/controllers/admin/domain_allows_controller.rb | 40 ++++++++++++++++++++++ app/controllers/admin/instances_controller.rb | 28 +++++++++++++-- app/controllers/api/base_controller.rb | 9 +++++ app/controllers/api/v1/accounts_controller.rb | 2 ++ app/controllers/api/v1/apps_controller.rb | 2 ++ .../api/v1/instances/activity_controller.rb | 3 +- .../api/v1/instances/peers_controller.rb | 3 +- app/controllers/api/v1/instances_controller.rb | 1 + app/controllers/application_controller.rb | 4 ++- app/controllers/concerns/account_owned_concern.rb | 1 + app/controllers/directories_controller.rb | 5 +-- app/controllers/home_controller.rb | 2 +- app/controllers/media_controller.rb | 1 + app/controllers/media_proxy_controller.rb | 2 ++ app/controllers/public_timelines_controller.rb | 5 +-- app/controllers/remote_interaction_controller.rb | 1 + app/controllers/tags_controller.rb | 1 + app/helpers/domain_control_helper.rb | 10 +++++- app/models/domain_allow.rb | 33 ++++++++++++++++++ app/models/instance.rb | 3 +- app/models/instance_filter.rb | 4 +++ app/policies/domain_allow_policy.rb | 11 ++++++ app/services/concerns/payloadable.rb | 2 +- app/services/unallow_domain_service.rb | 11 ++++++ app/views/admin/domain_allows/new.html.haml | 14 ++++++++ app/views/admin/instances/index.html.haml | 35 ++++++++++++------- app/views/admin/instances/show.html.haml | 4 ++- app/views/admin/settings/edit.html.haml | 28 ++++++++------- app/views/auth/registrations/new.html.haml | 2 +- app/views/layouts/public.html.haml | 9 +++-- config/initializers/2_whitelist_mode.rb | 5 +++ config/locales/en.yml | 7 ++++ config/locales/simple_form.en.yml | 2 ++ config/navigation.rb | 2 +- config/routes.rb | 1 + db/migrate/20190705002136_create_domain_allows.rb | 9 +++++ db/schema.rb | 9 ++++- lib/mastodon/domains_cli.rb | 22 ++++++++++-- spec/fabricators/domain_allow_fabricator.rb | 3 ++ spec/models/domain_allow_spec.rb | 5 +++ streaming/index.js | 5 +-- 44 files changed, 302 insertions(+), 53 deletions(-) create mode 100644 app/controllers/admin/domain_allows_controller.rb create mode 100644 app/models/domain_allow.rb create mode 100644 app/policies/domain_allow_policy.rb create mode 100644 app/services/unallow_domain_service.rb create mode 100644 app/views/admin/domain_allows/new.html.haml create mode 100644 config/initializers/2_whitelist_mode.rb create mode 100644 db/migrate/20190705002136_create_domain_allows.rb create mode 100644 spec/fabricators/domain_allow_fabricator.rb create mode 100644 spec/models/domain_allow_spec.rb (limited to 'lib') diff --git a/app/controllers/about_controller.rb b/app/controllers/about_controller.rb index 31cf17710..d276e8fe5 100644 --- a/app/controllers/about_controller.rb +++ b/app/controllers/about_controller.rb @@ -3,6 +3,7 @@ class AboutController < ApplicationController layout 'public' + before_action :require_open_federation!, only: [:show, :more] before_action :set_body_classes, only: :show before_action :set_instance_presenter before_action :set_expires_in @@ -19,6 +20,10 @@ class AboutController < ApplicationController private + def require_open_federation! + not_found if whitelist_mode? + end + def new_user User.new.tap do |user| user.build_account diff --git a/app/controllers/activitypub/base_controller.rb b/app/controllers/activitypub/base_controller.rb index a3b5c4dfa..0c2591e97 100644 --- a/app/controllers/activitypub/base_controller.rb +++ b/app/controllers/activitypub/base_controller.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class ActivityPub::BaseController < Api::BaseController + skip_before_action :require_authenticated_user! + private def set_cache_headers diff --git a/app/controllers/activitypub/inboxes_controller.rb b/app/controllers/activitypub/inboxes_controller.rb index 7cfd9a25e..bcfc1e6d4 100644 --- a/app/controllers/activitypub/inboxes_controller.rb +++ b/app/controllers/activitypub/inboxes_controller.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -class ActivityPub::InboxesController < Api::BaseController +class ActivityPub::InboxesController < ActivityPub::BaseController include SignatureVerification include JsonLdHelper include AccountOwnedConcern diff --git a/app/controllers/admin/domain_allows_controller.rb b/app/controllers/admin/domain_allows_controller.rb new file mode 100644 index 000000000..31be1978b --- /dev/null +++ b/app/controllers/admin/domain_allows_controller.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +class Admin::DomainAllowsController < Admin::BaseController + before_action :set_domain_allow, only: [:destroy] + + def new + authorize :domain_allow, :create? + + @domain_allow = DomainAllow.new(domain: params[:_domain]) + end + + def create + authorize :domain_allow, :create? + + @domain_allow = DomainAllow.new(resource_params) + + if @domain_allow.save + log_action :create, @domain_allow + redirect_to admin_instances_path, notice: I18n.t('admin.domain_allows.created_msg') + else + render :new + end + end + + def destroy + authorize @domain_allow, :destroy? + UnallowDomainService.new.call(@domain_allow) + redirect_to admin_instances_path, notice: I18n.t('admin.domain_allows.destroyed_msg') + end + + private + + def set_domain_allow + @domain_allow = DomainAllow.find(params[:id]) + end + + def resource_params + params.require(:domain_allow).permit(:domain) + end +end diff --git a/app/controllers/admin/instances_controller.rb b/app/controllers/admin/instances_controller.rb index 7888e844f..d4f201807 100644 --- a/app/controllers/admin/instances_controller.rb +++ b/app/controllers/admin/instances_controller.rb @@ -2,6 +2,10 @@ module Admin class InstancesController < BaseController + before_action :set_domain_block, only: :show + before_action :set_domain_allow, only: :show + before_action :set_instance, only: :show + def index authorize :instance, :index? @@ -11,20 +15,38 @@ module Admin def show authorize :instance, :show? - @instance = Instance.new(Account.by_domain_accounts.find_by(domain: params[:id]) || DomainBlock.find_by!(domain: params[:id])) @following_count = Follow.where(account: Account.where(domain: params[:id])).count @followers_count = Follow.where(target_account: Account.where(domain: params[:id])).count @reports_count = Report.where(target_account: Account.where(domain: params[:id])).count @blocks_count = Block.where(target_account: Account.where(domain: params[:id])).count @available = DeliveryFailureTracker.available?(Account.select(:shared_inbox_url).where(domain: params[:id]).first&.shared_inbox_url) @media_storage = MediaAttachment.where(account: Account.where(domain: params[:id])).sum(:file_file_size) - @domain_block = DomainBlock.rule_for(params[:id]) end private + def set_domain_block + @domain_block = DomainBlock.rule_for(params[:id]) + end + + def set_domain_allow + @domain_allow = DomainAllow.rule_for(params[:id]) + end + + def set_instance + resource = Account.by_domain_accounts.find_by(domain: params[:id]) + resource ||= @domain_block + resource ||= @domain_allow + + if resource + @instance = Instance.new(resource) + else + not_found + end + end + def filtered_instances - InstanceFilter.new(filter_params).results + InstanceFilter.new(whitelist_mode? ? { allowed: true } : filter_params).results end def paginated_instances diff --git a/app/controllers/api/base_controller.rb b/app/controllers/api/base_controller.rb index 6f33a1ea9..109e38ffa 100644 --- a/app/controllers/api/base_controller.rb +++ b/app/controllers/api/base_controller.rb @@ -9,6 +9,7 @@ class Api::BaseController < ApplicationController skip_before_action :store_current_location skip_before_action :require_functional! + before_action :require_authenticated_user!, if: :disallow_unauthenticated_api_access? before_action :set_cache_headers protect_from_forgery with: :null_session @@ -69,6 +70,10 @@ class Api::BaseController < ApplicationController nil end + def require_authenticated_user! + render json: { error: 'This API requires an authenticated user' }, status: 401 unless current_user + end + def require_user! if !current_user render json: { error: 'This method requires an authenticated user' }, status: 422 @@ -94,4 +99,8 @@ class Api::BaseController < ApplicationController def set_cache_headers response.headers['Cache-Control'] = 'no-cache, no-store, max-age=0, must-revalidate' end + + def disallow_unauthenticated_api_access? + authorized_fetch_mode? + end end diff --git a/app/controllers/api/v1/accounts_controller.rb b/app/controllers/api/v1/accounts_controller.rb index b0c62778e..b306e8e8c 100644 --- a/app/controllers/api/v1/accounts_controller.rb +++ b/app/controllers/api/v1/accounts_controller.rb @@ -12,6 +12,8 @@ class Api::V1::AccountsController < Api::BaseController before_action :check_account_suspension, only: [:show] before_action :check_enabled_registrations, only: [:create] + skip_before_action :require_authenticated_user!, only: :create + respond_to :json def show diff --git a/app/controllers/api/v1/apps_controller.rb b/app/controllers/api/v1/apps_controller.rb index e9f7a7291..97177547a 100644 --- a/app/controllers/api/v1/apps_controller.rb +++ b/app/controllers/api/v1/apps_controller.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true class Api::V1::AppsController < Api::BaseController + skip_before_action :require_authenticated_user! + def create @app = Doorkeeper::Application.create!(application_options) render json: @app, serializer: REST::ApplicationSerializer diff --git a/app/controllers/api/v1/instances/activity_controller.rb b/app/controllers/api/v1/instances/activity_controller.rb index d0080c5c2..4fb5a69d8 100644 --- a/app/controllers/api/v1/instances/activity_controller.rb +++ b/app/controllers/api/v1/instances/activity_controller.rb @@ -2,6 +2,7 @@ class Api::V1::Instances::ActivityController < Api::BaseController before_action :require_enabled_api! + skip_before_action :set_cache_headers respond_to :json @@ -33,6 +34,6 @@ class Api::V1::Instances::ActivityController < Api::BaseController end def require_enabled_api! - head 404 unless Setting.activity_api_enabled + head 404 unless Setting.activity_api_enabled && !whitelist_mode? end end diff --git a/app/controllers/api/v1/instances/peers_controller.rb b/app/controllers/api/v1/instances/peers_controller.rb index 450e6502f..75c3cb4ba 100644 --- a/app/controllers/api/v1/instances/peers_controller.rb +++ b/app/controllers/api/v1/instances/peers_controller.rb @@ -2,6 +2,7 @@ class Api::V1::Instances::PeersController < Api::BaseController before_action :require_enabled_api! + skip_before_action :set_cache_headers respond_to :json @@ -14,6 +15,6 @@ class Api::V1::Instances::PeersController < Api::BaseController private def require_enabled_api! - head 404 unless Setting.peers_api_enabled + head 404 unless Setting.peers_api_enabled && !whitelist_mode? end end diff --git a/app/controllers/api/v1/instances_controller.rb b/app/controllers/api/v1/instances_controller.rb index 93e4f0003..8d8231423 100644 --- a/app/controllers/api/v1/instances_controller.rb +++ b/app/controllers/api/v1/instances_controller.rb @@ -2,6 +2,7 @@ class Api::V1::InstancesController < Api::BaseController respond_to :json + skip_before_action :set_cache_headers def show diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 41ce1a0ca..0d3913ee0 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -11,12 +11,14 @@ class ApplicationController < ActionController::Base include UserTrackingConcern include SessionTrackingConcern include CacheConcern + include DomainControlHelper helper_method :current_account helper_method :current_session helper_method :current_theme helper_method :single_user_mode? helper_method :use_seamless_external_login? + helper_method :whitelist_mode? rescue_from ActionController::RoutingError, with: :not_found rescue_from ActiveRecord::RecordNotFound, with: :not_found @@ -38,7 +40,7 @@ class ApplicationController < ActionController::Base end def authorized_fetch_mode? - ENV['AUTHORIZED_FETCH'] == 'true' + ENV['AUTHORIZED_FETCH'] == 'true' || Rails.configuration.x.whitelist_mode end def public_fetch_mode? diff --git a/app/controllers/concerns/account_owned_concern.rb b/app/controllers/concerns/account_owned_concern.rb index 99c240fe9..460f71f65 100644 --- a/app/controllers/concerns/account_owned_concern.rb +++ b/app/controllers/concerns/account_owned_concern.rb @@ -4,6 +4,7 @@ module AccountOwnedConcern extend ActiveSupport::Concern included do + before_action :authenticate_user!, if: -> { whitelist_mode? && request.format != :json } before_action :set_account, if: :account_required? before_action :check_account_approval, if: :account_required? before_action :check_account_suspension, if: :account_required? diff --git a/app/controllers/directories_controller.rb b/app/controllers/directories_controller.rb index 594907674..d2ef76f06 100644 --- a/app/controllers/directories_controller.rb +++ b/app/controllers/directories_controller.rb @@ -3,7 +3,8 @@ class DirectoriesController < ApplicationController layout 'public' - before_action :check_enabled + before_action :authenticate_user!, if: :whitelist_mode? + before_action :require_enabled! before_action :set_instance_presenter before_action :set_tag, only: :show before_action :set_tags @@ -19,7 +20,7 @@ class DirectoriesController < ApplicationController private - def check_enabled + def require_enabled! return not_found unless Setting.profile_directory end diff --git a/app/controllers/home_controller.rb b/app/controllers/home_controller.rb index 42493cd78..22d507e77 100644 --- a/app/controllers/home_controller.rb +++ b/app/controllers/home_controller.rb @@ -55,7 +55,7 @@ class HomeController < ApplicationController end def default_redirect_path - if request.path.start_with?('/web') + if request.path.start_with?('/web') || whitelist_mode? new_user_session_path elsif single_user_mode? short_account_path(Account.local.without_suspended.where('id > 0').first) diff --git a/app/controllers/media_controller.rb b/app/controllers/media_controller.rb index b3b7519a1..1f693de32 100644 --- a/app/controllers/media_controller.rb +++ b/app/controllers/media_controller.rb @@ -5,6 +5,7 @@ class MediaController < ApplicationController skip_before_action :store_current_location + before_action :authenticate_user!, if: :whitelist_mode? before_action :set_media_attachment before_action :verify_permitted_status! before_action :check_playable, only: :player diff --git a/app/controllers/media_proxy_controller.rb b/app/controllers/media_proxy_controller.rb index 8fc18dd06..8da6c6fe0 100644 --- a/app/controllers/media_proxy_controller.rb +++ b/app/controllers/media_proxy_controller.rb @@ -5,6 +5,8 @@ class MediaProxyController < ApplicationController skip_before_action :store_current_location + before_action :authenticate_user!, if: :whitelist_mode? + def show RedisLock.acquire(lock_options) do |lock| if lock.acquired? diff --git a/app/controllers/public_timelines_controller.rb b/app/controllers/public_timelines_controller.rb index 23506b990..324bdc508 100644 --- a/app/controllers/public_timelines_controller.rb +++ b/app/controllers/public_timelines_controller.rb @@ -3,7 +3,8 @@ class PublicTimelinesController < ApplicationController layout 'public' - before_action :check_enabled + before_action :authenticate_user!, if: :whitelist_mode? + before_action :require_enabled! before_action :set_body_classes before_action :set_instance_presenter @@ -16,7 +17,7 @@ class PublicTimelinesController < ApplicationController private - def check_enabled + def require_enabled! not_found unless Setting.timeline_preview end diff --git a/app/controllers/remote_interaction_controller.rb b/app/controllers/remote_interaction_controller.rb index cc6993c52..fa742fb0a 100644 --- a/app/controllers/remote_interaction_controller.rb +++ b/app/controllers/remote_interaction_controller.rb @@ -5,6 +5,7 @@ class RemoteInteractionController < ApplicationController layout 'modal' + before_action :authenticate_user!, if: :whitelist_mode? before_action :set_interaction_type before_action :set_status before_action :set_body_classes diff --git a/app/controllers/tags_controller.rb b/app/controllers/tags_controller.rb index d08e5a61a..3cd2d9e20 100644 --- a/app/controllers/tags_controller.rb +++ b/app/controllers/tags_controller.rb @@ -8,6 +8,7 @@ class TagsController < ApplicationController layout 'public' before_action :require_signature!, if: -> { request.format == :json && authorized_fetch_mode? } + before_action :authenticate_user!, if: :whitelist_mode? before_action :set_tag before_action :set_body_classes before_action :set_instance_presenter diff --git a/app/helpers/domain_control_helper.rb b/app/helpers/domain_control_helper.rb index efd328f81..067b2c2cd 100644 --- a/app/helpers/domain_control_helper.rb +++ b/app/helpers/domain_control_helper.rb @@ -12,6 +12,14 @@ module DomainControlHelper end end - DomainBlock.blocked?(domain) + if whitelist_mode? + !DomainAllow.allowed?(domain) + else + DomainBlock.blocked?(domain) + end + end + + def whitelist_mode? + Rails.configuration.x.whitelist_mode end end diff --git a/app/models/domain_allow.rb b/app/models/domain_allow.rb new file mode 100644 index 000000000..85018b636 --- /dev/null +++ b/app/models/domain_allow.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +# == Schema Information +# +# Table name: domain_allows +# +# id :bigint(8) not null, primary key +# domain :string default(""), not null +# created_at :datetime not null +# updated_at :datetime not null +# + +class DomainAllow < ApplicationRecord + include DomainNormalizable + + validates :domain, presence: true, uniqueness: true + + scope :matches_domain, ->(value) { where(arel_table[:domain].matches("%#{value}%")) } + + class << self + def allowed?(domain) + !rule_for(domain).nil? + end + + def rule_for(domain) + return if domain.blank? + + uri = Addressable::URI.new.tap { |u| u.host = domain.gsub(/[\/]/, '') } + + find_by(domain: uri.normalized_host) + end + end +end diff --git a/app/models/instance.rb b/app/models/instance.rb index 797a191e0..3c740f8a2 100644 --- a/app/models/instance.rb +++ b/app/models/instance.rb @@ -7,8 +7,9 @@ class Instance def initialize(resource) @domain = resource.domain - @accounts_count = resource.is_a?(DomainBlock) ? nil : resource.accounts_count + @accounts_count = resource.respond_to?(:accounts_count) ? resource.accounts_count : nil @domain_block = resource.is_a?(DomainBlock) ? resource : DomainBlock.rule_for(domain) + @domain_allow = resource.is_a?(DomainAllow) ? resource : DomainAllow.rule_for(domain) end def countable? diff --git a/app/models/instance_filter.rb b/app/models/instance_filter.rb index 848fff53e..8bfab826d 100644 --- a/app/models/instance_filter.rb +++ b/app/models/instance_filter.rb @@ -12,6 +12,10 @@ class InstanceFilter scope = DomainBlock scope = scope.matches_domain(params[:by_domain]) if params[:by_domain].present? scope.order(id: :desc) + elsif params[:allowed].present? + scope = DomainAllow + scope = scope.matches_domain(params[:by_domain]) if params[:by_domain].present? + scope.order(id: :desc) else scope = Account.remote scope = scope.matches_domain(params[:by_domain]) if params[:by_domain].present? diff --git a/app/policies/domain_allow_policy.rb b/app/policies/domain_allow_policy.rb new file mode 100644 index 000000000..5030453bb --- /dev/null +++ b/app/policies/domain_allow_policy.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class DomainAllowPolicy < ApplicationPolicy + def create? + admin? + end + + def destroy? + admin? + end +end diff --git a/app/services/concerns/payloadable.rb b/app/services/concerns/payloadable.rb index 953740faa..7f9f21c4b 100644 --- a/app/services/concerns/payloadable.rb +++ b/app/services/concerns/payloadable.rb @@ -14,6 +14,6 @@ module Payloadable end def signing_enabled? - ENV['AUTHORIZED_FETCH'] != 'true' + ENV['AUTHORIZED_FETCH'] != 'true' && !Rails.configuration.x.whitelist_mode end end diff --git a/app/services/unallow_domain_service.rb b/app/services/unallow_domain_service.rb new file mode 100644 index 000000000..d4387c1a1 --- /dev/null +++ b/app/services/unallow_domain_service.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class UnallowDomainService < BaseService + def call(domain_allow) + Account.where(domain: domain_allow.domain).find_each do |account| + SuspendAccountService.new.call(account, destroy: true) + end + + domain_allow.destroy + end +end diff --git a/app/views/admin/domain_allows/new.html.haml b/app/views/admin/domain_allows/new.html.haml new file mode 100644 index 000000000..52599857a --- /dev/null +++ b/app/views/admin/domain_allows/new.html.haml @@ -0,0 +1,14 @@ +- content_for :header_tags do + = javascript_pack_tag 'admin', integrity: true, async: true, crossorigin: 'anonymous' + +- content_for :page_title do + = t('admin.domain_allows.add_new') + += simple_form_for @domain_allow, url: admin_domain_allows_path do |f| + = render 'shared/error_messages', object: @domain_allow + + .fields-group + = f.input :domain, wrapper: :with_label, label: t('admin.domain_blocks.domain'), required: true + + .actions + = f.button :button, t('admin.domain_allows.add_new'), type: :submit diff --git a/app/views/admin/instances/index.html.haml b/app/views/admin/instances/index.html.haml index 61e578409..982dc5035 100644 --- a/app/views/admin/instances/index.html.haml +++ b/app/views/admin/instances/index.html.haml @@ -6,24 +6,30 @@ %strong= t('admin.instances.moderation.title') %ul %li= filter_link_to t('admin.instances.moderation.all'), limited: nil - %li= filter_link_to t('admin.instances.moderation.limited'), limited: '1' + + - unless whitelist_mode? + %li= filter_link_to t('admin.instances.moderation.limited'), limited: '1' %div{ style: 'flex: 1 1 auto; text-align: right' } - = link_to t('admin.domain_blocks.add_new'), new_admin_domain_block_path, class: 'button' + - if whitelist_mode? + = link_to t('admin.domain_allows.add_new'), new_admin_domain_allow_path, class: 'button' + - else + = link_to t('admin.domain_blocks.add_new'), new_admin_domain_block_path, class: 'button' -= form_tag admin_instances_url, method: 'GET', class: 'simple_form' do - .fields-group - - Admin::FilterHelper::INSTANCES_FILTERS.each do |key| - - if params[key].present? - = hidden_field_tag key, params[key] +- unless whitelist_mode? + = form_tag admin_instances_url, method: 'GET', class: 'simple_form' do + .fields-group + - Admin::FilterHelper::INSTANCES_FILTERS.each do |key| + - if params[key].present? + = hidden_field_tag key, params[key] - - %i(by_domain).each do |key| - .input.string.optional - = text_field_tag key, params[key], class: 'string optional', placeholder: I18n.t("admin.instances.#{key}") + - %i(by_domain).each do |key| + .input.string.optional + = text_field_tag key, params[key], class: 'string optional', placeholder: I18n.t("admin.instances.#{key}") - .actions - %button= t('admin.accounts.search') - = link_to t('admin.accounts.reset'), admin_instances_path, class: 'button negative' + .actions + %button= t('admin.accounts.search') + = link_to t('admin.accounts.reset'), admin_instances_path, class: 'button negative' %hr.spacer/ @@ -47,8 +53,11 @@ - unless first_item • = t('admin.domain_blocks.rejecting_reports') + - elsif whitelist_mode? + = t('admin.accounts.whitelisted') - else = t('admin.accounts.no_limits_imposed') - if instance.countable? .trends__item__current{ title: t('admin.instances.known_accounts', count: instance.accounts_count) }= number_to_human instance.accounts_count, strip_insignificant_zeros: true + = paginate paginated_instances diff --git a/app/views/admin/instances/show.html.haml b/app/views/admin/instances/show.html.haml index c7992a490..fbb49ba02 100644 --- a/app/views/admin/instances/show.html.haml +++ b/app/views/admin/instances/show.html.haml @@ -38,7 +38,9 @@ = link_to t('admin.accounts.title'), admin_accounts_path(remote: '1', by_domain: @instance.domain), class: 'button' %div{ style: 'float: right' } - - if @domain_block + - if @domain_allow + = link_to t('admin.domain_allows.undo'), admin_domain_allow_path(@domain_allow), class: 'button button--destructive', data: { confirm: t('admin.accounts.are_you_sure'), method: :delete } + - elsif @domain_block = link_to t('admin.domain_blocks.undo'), admin_domain_block_path(@domain_block), class: 'button' - else = link_to t('admin.domain_blocks.add_new'), new_admin_domain_block_path(_domain: @instance.domain), class: 'button' diff --git a/app/views/admin/settings/edit.html.haml b/app/views/admin/settings/edit.html.haml index b3bf3849c..1e2ed3f77 100644 --- a/app/views/admin/settings/edit.html.haml +++ b/app/views/admin/settings/edit.html.haml @@ -42,11 +42,12 @@ %hr.spacer/ - .fields-group - = f.input :timeline_preview, as: :boolean, wrapper: :with_label, label: t('admin.settings.timeline_preview.title'), hint: t('admin.settings.timeline_preview.desc_html') + - unless whitelist_mode? + .fields-group + = f.input :timeline_preview, as: :boolean, wrapper: :with_label, label: t('admin.settings.timeline_preview.title'), hint: t('admin.settings.timeline_preview.desc_html') - .fields-group - = f.input :show_known_fediverse_at_about_page, as: :boolean, wrapper: :with_label, label: t('admin.settings.show_known_fediverse_at_about_page.title'), hint: t('admin.settings.show_known_fediverse_at_about_page.desc_html') + .fields-group + = f.input :show_known_fediverse_at_about_page, as: :boolean, wrapper: :with_label, label: t('admin.settings.show_known_fediverse_at_about_page.title'), hint: t('admin.settings.show_known_fediverse_at_about_page.desc_html') .fields-group = f.input :show_staff_badge, as: :boolean, wrapper: :with_label, label: t('admin.settings.show_staff_badge.title'), hint: t('admin.settings.show_staff_badge.desc_html') @@ -54,17 +55,18 @@ .fields-group = f.input :open_deletion, as: :boolean, wrapper: :with_label, label: t('admin.settings.registrations.deletion.title'), hint: t('admin.settings.registrations.deletion.desc_html') - .fields-group - = f.input :activity_api_enabled, as: :boolean, wrapper: :with_label, label: t('admin.settings.activity_api_enabled.title'), hint: t('admin.settings.activity_api_enabled.desc_html') + - unless whitelist_mode? + .fields-group + = f.input :activity_api_enabled, as: :boolean, wrapper: :with_label, label: t('admin.settings.activity_api_enabled.title'), hint: t('admin.settings.activity_api_enabled.desc_html') - .fields-group - = f.input :peers_api_enabled, as: :boolean, wrapper: :with_label, label: t('admin.settings.peers_api_enabled.title'), hint: t('admin.settings.peers_api_enabled.desc_html') + .fields-group + = f.input :peers_api_enabled, as: :boolean, wrapper: :with_label, label: t('admin.settings.peers_api_enabled.title'), hint: t('admin.settings.peers_api_enabled.desc_html') - .fields-group - = f.input :preview_sensitive_media, as: :boolean, wrapper: :with_label, label: t('admin.settings.preview_sensitive_media.title'), hint: t('admin.settings.preview_sensitive_media.desc_html') + .fields-group + = f.input :preview_sensitive_media, as: :boolean, wrapper: :with_label, label: t('admin.settings.preview_sensitive_media.title'), hint: t('admin.settings.preview_sensitive_media.desc_html') - .fields-group - = f.input :profile_directory, as: :boolean, wrapper: :with_label, label: t('admin.settings.profile_directory.title'), hint: t('admin.settings.profile_directory.desc_html') + .fields-group + = f.input :profile_directory, as: :boolean, wrapper: :with_label, label: t('admin.settings.profile_directory.title'), hint: t('admin.settings.profile_directory.desc_html') .fields-group = f.input :spam_check_enabled, as: :boolean, wrapper: :with_label, label: t('admin.settings.spam_check_enabled.title'), hint: t('admin.settings.spam_check_enabled.desc_html') @@ -76,7 +78,7 @@ .fields-group = f.input :closed_registrations_message, as: :text, wrapper: :with_block_label, label: t('admin.settings.registrations.closed_message.title'), hint: t('admin.settings.registrations.closed_message.desc_html'), input_html: { rows: 8 } - = f.input :site_extended_description, wrapper: :with_block_label, as: :text, label: t('admin.settings.site_description_extended.title'), hint: t('admin.settings.site_description_extended.desc_html'), input_html: { rows: 8 } + = f.input :site_extended_description, wrapper: :with_block_label, as: :text, label: t('admin.settings.site_description_extended.title'), hint: t('admin.settings.site_description_extended.desc_html'), input_html: { rows: 8 } unless whitelist_mode? = f.input :site_terms, wrapper: :with_block_label, as: :text, label: t('admin.settings.site_terms.title'), hint: t('admin.settings.site_terms.desc_html'), input_html: { rows: 8 } = f.input :custom_css, wrapper: :with_block_label, as: :text, input_html: { rows: 8 }, label: t('admin.settings.custom_css.title'), hint: t('admin.settings.custom_css.desc_html') diff --git a/app/views/auth/registrations/new.html.haml b/app/views/auth/registrations/new.html.haml index b4a7cced5..83384d737 100644 --- a/app/views/auth/registrations/new.html.haml +++ b/app/views/auth/registrations/new.html.haml @@ -33,7 +33,7 @@ = f.input :invite_code, as: :hidden .fields-group - = f.input :agreement, as: :boolean, wrapper: :with_label, label: t('auth.checkbox_agreement_html', rules_path: about_more_path, terms_path: terms_path) + = f.input :agreement, as: :boolean, wrapper: :with_label, label: whitelist_mode? ? t('auth.checkbox_agreement_without_rules_html', terms_path: terms_path) : t('auth.checkbox_agreement_html', rules_path: about_more_path, terms_path: terms_path) .actions = f.button :button, @invite.present? ? t('auth.register') : sign_up_message, type: :submit diff --git a/app/views/layouts/public.html.haml b/app/views/layouts/public.html.haml index 2929ac599..69738a2f7 100644 --- a/app/views/layouts/public.html.haml +++ b/app/views/layouts/public.html.haml @@ -10,10 +10,13 @@ = link_to root_url, class: 'brand' do = svg_logo_full - = link_to t('directories.directory'), explore_path, class: 'nav-link optional' if Setting.profile_directory - = link_to t('about.about_this'), about_more_path, class: 'nav-link optional' - = link_to t('about.apps'), 'https://joinmastodon.org/apps', class: 'nav-link optional' + - unless whitelist_mode? + = link_to t('directories.directory'), explore_path, class: 'nav-link optional' if Setting.profile_directory + = link_to t('about.about_this'), about_more_path, class: 'nav-link optional' + = link_to t('about.apps'), 'https://joinmastodon.org/apps', class: 'nav-link optional' + .nav-center + .nav-right - if user_signed_in? = link_to t('settings.back'), root_url, class: 'nav-link nav-button webapp-btn' diff --git a/config/initializers/2_whitelist_mode.rb b/config/initializers/2_whitelist_mode.rb new file mode 100644 index 000000000..a17ad07a2 --- /dev/null +++ b/config/initializers/2_whitelist_mode.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +Rails.application.configure do + config.x.whitelist_mode = ENV['WHITELIST_MODE'] == 'true' +end diff --git a/config/locales/en.yml b/config/locales/en.yml index 9e1be87be..6c1a34300 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -186,6 +186,7 @@ en: username: Username warn: Warn web: Web + whitelisted: Whitelisted action_logs: actions: assigned_to_self_report: "%{name} assigned report %{target} to themselves" @@ -269,6 +270,11 @@ en: week_interactions: interactions this week week_users_active: active this week week_users_new: users this week + domain_allows: + add_new: Whitelist domain + created_msg: Domain has been successfully whitelisted + destroyed_msg: Domain has been removed from the whitelist + undo: Remove from whitelist domain_blocks: add_new: Add new domain block created_msg: Domain block is now being processed @@ -524,6 +530,7 @@ en: apply_for_account: Request an invite change_password: Password checkbox_agreement_html: I agree to the server rules and terms of service + checkbox_agreement_without_rules_html: I agree to the terms of service delete_account: Delete account delete_account_html: If you wish to delete your account, you can proceed here. You will be asked for confirmation. didnt_get_confirmation: Didn't receive confirmation instructions? diff --git a/config/locales/simple_form.en.yml b/config/locales/simple_form.en.yml index 12a7ec2b3..10b30e627 100644 --- a/config/locales/simple_form.en.yml +++ b/config/locales/simple_form.en.yml @@ -38,6 +38,8 @@ en: setting_use_pending_items: Hide timeline updates behind a click instead of automatically scrolling the feed username: Your username will be unique on %{domain} whole_word: When the keyword or phrase is alphanumeric only, it will only be applied if it matches the whole word + domain_allow: + domain: This domain will be able to fetch data from this server and incoming data from it will be processed and stored featured_tag: name: 'You might want to use one of these:' imports: diff --git a/config/navigation.rb b/config/navigation.rb index 5ab2e4399..9b46da603 100644 --- a/config/navigation.rb +++ b/config/navigation.rb @@ -39,7 +39,7 @@ SimpleNavigation::Configuration.run do |navigation| s.item :accounts, safe_join([fa_icon('users fw'), t('admin.accounts.title')]), admin_accounts_url, highlights_on: %r{/admin/accounts|/admin/pending_accounts} s.item :invites, safe_join([fa_icon('user-plus fw'), t('admin.invites.title')]), admin_invites_path s.item :tags, safe_join([fa_icon('tag fw'), t('admin.tags.title')]), admin_tags_path - s.item :instances, safe_join([fa_icon('cloud fw'), t('admin.instances.title')]), admin_instances_url(limited: '1'), highlights_on: %r{/admin/instances|/admin/domain_blocks}, if: -> { current_user.admin? } + s.item :instances, safe_join([fa_icon('cloud fw'), t('admin.instances.title')]), admin_instances_url(limited: whitelist_mode? ? nil : '1'), highlights_on: %r{/admin/instances|/admin/domain_blocks|/admin/domain_allows}, if: -> { current_user.admin? } s.item :email_domain_blocks, safe_join([fa_icon('envelope fw'), t('admin.email_domain_blocks.title')]), admin_email_domain_blocks_url, highlights_on: %r{/admin/email_domain_blocks}, if: -> { current_user.admin? } end diff --git a/config/routes.rb b/config/routes.rb index b6c215888..04424bbbd 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -154,6 +154,7 @@ Rails.application.routes.draw do namespace :admin do get '/dashboard', to: 'dashboard#index' + resources :domain_allows, only: [:new, :create, :show, :destroy] resources :domain_blocks, only: [:new, :create, :show, :destroy] resources :email_domain_blocks, only: [:index, :new, :create, :destroy] resources :action_logs, only: [:index] diff --git a/db/migrate/20190705002136_create_domain_allows.rb b/db/migrate/20190705002136_create_domain_allows.rb new file mode 100644 index 000000000..83b0728d9 --- /dev/null +++ b/db/migrate/20190705002136_create_domain_allows.rb @@ -0,0 +1,9 @@ +class CreateDomainAllows < ActiveRecord::Migration[5.2] + def change + create_table :domain_allows do |t| + t.string :domain, default: '', null: false, index: { unique: true } + + t.timestamps + end + end +end diff --git a/db/schema.rb b/db/schema.rb index 1847305c7..2d83d8b76 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2019_07_26_175042) do +ActiveRecord::Schema.define(version: 2019_07_28_084117) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -245,6 +245,13 @@ ActiveRecord::Schema.define(version: 2019_07_26_175042) do t.index ["account_id"], name: "index_custom_filters_on_account_id" end + create_table "domain_allows", force: :cascade do |t| + t.string "domain", default: "", null: false + t.datetime "created_at", null: false + t.datetime "updated_at", null: false + t.index ["domain"], name: "index_domain_allows_on_domain", unique: true + end + create_table "domain_blocks", force: :cascade do |t| t.string "domain", default: "", null: false t.datetime "created_at", null: false diff --git a/lib/mastodon/domains_cli.rb b/lib/mastodon/domains_cli.rb index b081581fe..f30062363 100644 --- a/lib/mastodon/domains_cli.rb +++ b/lib/mastodon/domains_cli.rb @@ -12,17 +12,33 @@ module Mastodon end option :dry_run, type: :boolean - desc 'purge DOMAIN', 'Remove accounts from a DOMAIN without a trace' + option :whitelist_mode, type: :boolean + desc 'purge [DOMAIN]', 'Remove accounts from a DOMAIN without a trace' long_desc <<-LONG_DESC Remove all accounts from a given DOMAIN without leaving behind any records. Unlike a suspension, if the DOMAIN still exists in the wild, it means the accounts could return if they are resolved again. + + When the --whitelist-mode option is given, instead of purging accounts + from a single domain, all accounts from domains that are not whitelisted + are removed from the database. LONG_DESC - def purge(domain) + def purge(domain = nil) removed = 0 dry_run = options[:dry_run] ? ' (DRY RUN)' : '' - Account.where(domain: domain).find_each do |account| + scope = begin + if options[:whitelist_mode] + Account.remote.where.not(domain: DomainAllow.pluck(:domain)) + elsif domain.present? + Account.remote.where(domain: domain) + else + say('No domain given', :red) + exit(1) + end + end + + scope.find_each do |account| SuspendAccountService.new.call(account, destroy: true) unless options[:dry_run] removed += 1 say('.', :green, false) diff --git a/spec/fabricators/domain_allow_fabricator.rb b/spec/fabricators/domain_allow_fabricator.rb new file mode 100644 index 000000000..6226b1e20 --- /dev/null +++ b/spec/fabricators/domain_allow_fabricator.rb @@ -0,0 +1,3 @@ +Fabricator(:domain_allow) do + domain "MyString" +end diff --git a/spec/models/domain_allow_spec.rb b/spec/models/domain_allow_spec.rb new file mode 100644 index 000000000..e65435127 --- /dev/null +++ b/spec/models/domain_allow_spec.rb @@ -0,0 +1,5 @@ +require 'rails_helper' + +RSpec.describe DomainAllow, type: :model do + pending "add some examples to (or delete) #{__FILE__}" +end diff --git a/streaming/index.js b/streaming/index.js index 0529804b1..304e7e046 100644 --- a/streaming/index.js +++ b/streaming/index.js @@ -12,6 +12,7 @@ const uuid = require('uuid'); const fs = require('fs'); const env = process.env.NODE_ENV || 'development'; +const alwaysRequireAuth = process.env.WHITELIST_MODE === 'true' || process.env.AUTHORIZED_FETCH === 'true'; dotenv.config({ path: env === 'production' ? '.env.production' : '.env', @@ -271,7 +272,7 @@ const startWorker = (workerId) => { const wsVerifyClient = (info, cb) => { const location = url.parse(info.req.url, true); - const authRequired = !PUBLIC_STREAMS.some(stream => stream === location.query.stream); + const authRequired = alwaysRequireAuth || !PUBLIC_STREAMS.some(stream => stream === location.query.stream); const allowedScopes = []; if (authRequired) { @@ -306,7 +307,7 @@ const startWorker = (workerId) => { return; } - const authRequired = !PUBLIC_ENDPOINTS.some(endpoint => endpoint === req.path); + const authRequired = alwaysRequireAuth || !PUBLIC_ENDPOINTS.some(endpoint => endpoint === req.path); const allowedScopes = []; if (authRequired) { -- cgit From f96f45ef12341648cb8d79d9b70f8c5636f98f64 Mon Sep 17 00:00:00 2001 From: Darius Kazemi Date: Sat, 3 Aug 2019 10:11:09 -0700 Subject: Add option to exclude suspended domains/subdomains from tootctl domains crawl (#11454) * Add "--exclude-suspended" to tootctl domains crawl This new option ignores any instances suspended server-wide as well as their associated subdomains. This queries all domain blocks up front, then runs a regexp on each domain. This improves performance over what may be the obvious implementation, which is to ask `DomainBlocks.blocked?(domain)` for each domain -- this hits the DB many times, slowing things down considerably. * cleaning up code style * Compiling regex * Removing ternary operator --- lib/mastodon/domains_cli.rb | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/lib/mastodon/domains_cli.rb b/lib/mastodon/domains_cli.rb index f30062363..17cafd1bc 100644 --- a/lib/mastodon/domains_cli.rb +++ b/lib/mastodon/domains_cli.rb @@ -58,6 +58,7 @@ module Mastodon option :concurrency, type: :numeric, default: 50, aliases: [:c] option :silent, type: :boolean, default: false, aliases: [:s] option :format, type: :string, default: 'summary', aliases: [:f] + option :exclude_suspended, type: :boolean, default: false, aliases: [:x] desc 'crawl [START]', 'Crawl all known peers, optionally beginning at START' long_desc <<-LONG_DESC Crawl the fediverse by using the Mastodon REST API endpoints that expose @@ -74,18 +75,25 @@ module Mastodon default (`summary`), a summary of the statistics is returned. The other options are `domains`, which returns a newline-delimited list of all discovered peers, and `json`, which dumps all the aggregated data raw. + + The --exclude-suspended (-x) option means that domains that are suspended + instance-wide do not appear in the output and are not included in summaries. + This also excludes subdomains of any of those domains. LONG_DESC def crawl(start = nil) - stats = Concurrent::Hash.new - processed = Concurrent::AtomicFixnum.new(0) - failed = Concurrent::AtomicFixnum.new(0) - start_at = Time.now.to_f - seed = start ? [start] : Account.remote.domains + stats = Concurrent::Hash.new + processed = Concurrent::AtomicFixnum.new(0) + failed = Concurrent::AtomicFixnum.new(0) + start_at = Time.now.to_f + seed = start ? [start] : Account.remote.domains + blocked_domains = Regexp.new('\\.?' + DomainBlock.where(severity: 1).pluck(:domain).join('|') + '$') pool = Concurrent::ThreadPoolExecutor.new(min_threads: 0, max_threads: options[:concurrency], idletime: 10, auto_terminate: true, max_queue: 0) work_unit = ->(domain) do next if stats.key?(domain) + next if options[:exclude_suspended] && domain.match(blocked_domains) + stats[domain] = nil processed.increment -- cgit From 3a77090d015203a1ae20376ed69ca699eed3976d Mon Sep 17 00:00:00 2001 From: Stanislas Date: Sat, 17 Aug 2019 22:59:40 +0200 Subject: Limit "tootctl accounts follow" to local accounts (#11592) To (somewhat) limit mass remote follow. Fix #11360 --- lib/mastodon/accounts_cli.rb | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/lib/mastodon/accounts_cli.rb b/lib/mastodon/accounts_cli.rb index 7d0215313..d1854acc0 100644 --- a/lib/mastodon/accounts_cli.rb +++ b/lib/mastodon/accounts_cli.rb @@ -314,11 +314,15 @@ module Mastodon desc 'follow ACCT', 'Make all local accounts follow account specified by ACCT' long_desc <<-LONG_DESC - Make all local accounts follow an account specified by ACCT. ACCT can be - a simple username, in case of a local user. It can also be in the format - username@domain, in case of a remote user. + Make all local accounts follow another local account specified by ACCT. + ACCT should be the username only. LONG_DESC def follow(acct) + if acct.include? '@' + say('Target account name should not contain a target instance, since it has to be a local account.', :red) + exit(1) + end + target_account = ResolveAccountService.new.call(acct) processed = 0 failed = 0 -- cgit From 96702e7f6727d9c688b2c6e2527c4a5bfefff886 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sun, 18 Aug 2019 14:55:03 +0200 Subject: Add `tootctl cache recount` command (#11597) --- app/models/concerns/account_counters.rb | 3 ++- app/models/status.rb | 7 +++-- lib/mastodon/cache_cli.rb | 45 +++++++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 3 deletions(-) (limited to 'lib') diff --git a/app/models/concerns/account_counters.rb b/app/models/concerns/account_counters.rb index 3581df8dd..6e25e1905 100644 --- a/app/models/concerns/account_counters.rb +++ b/app/models/concerns/account_counters.rb @@ -26,7 +26,8 @@ module AccountCounters private def save_account_stat - return unless account_stat&.changed? + return unless association(:account_stat).loaded? && account_stat&.changed? + account_stat.save end end diff --git a/app/models/status.rb b/app/models/status.rb index 23682c84b..0538c4e9e 100644 --- a/app/models/status.rb +++ b/app/models/status.rb @@ -388,13 +388,16 @@ class Status < ApplicationRecord end end + def status_stat + super || build_status_stat + end + private def update_status_stat!(attrs) return if marked_for_destruction? || destroyed? - record = status_stat || build_status_stat - record.update(attrs) + status_stat.update(attrs) end def store_uri diff --git a/lib/mastodon/cache_cli.rb b/lib/mastodon/cache_cli.rb index e9b6667b3..5b0eea91b 100644 --- a/lib/mastodon/cache_cli.rb +++ b/lib/mastodon/cache_cli.rb @@ -15,5 +15,50 @@ module Mastodon Rails.cache.clear say('OK', :green) end + + desc 'recount TYPE', 'Update hard-cached counters' + long_desc <<~LONG_DESC + Update hard-cached counters of TYPE by counting referenced + records from scratch. TYPE can be "accounts" or "statuses". + + It may take a very long time to finish, depending on the + size of the database. + LONG_DESC + def recount(type) + processed = 0 + + case type + when 'accounts' + Account.local.includes(:account_stat).find_each do |account| + account_stat = account.account_stat + account_stat.following_count = account.active_relationships.count + account_stat.followers_count = account.passive_relationships.count + account_stat.statuses_count = account.statuses.where.not(visibility: :direct).count + + account_stat.save if account_stat.changed? + + processed += 1 + say('.', :green, false) + end + when 'statuses' + Status.includes(:status_stat).find_each do |status| + status_stat = status.status_stat + status_stat.replies_count = status.replies.where.not(visibility: :direct).count + status_stat.reblogs_count = status.reblogs.count + status_stat.favourites_count = status.favourites.count + + status_stat.save if status_stat.changed? + + processed += 1 + say('.', :green, false) + end + else + say("Unknown type: #{type}", :red) + exit(1) + end + + say + say("OK, recounted #{processed} records", :green) + end end end -- cgit From 86748148256b504c0411119628435b1445959309 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 10 Sep 2019 13:48:48 +0200 Subject: Change tootctl to use inline parallelization instead of Sidekiq (#11776) - Remove --background option - Add --concurrency(=5) option - Add progress bars --- Gemfile | 1 + Gemfile.lock | 1 + app/models/media_attachment.rb | 1 + app/models/preview_card.rb | 2 + app/workers/maintenance/destroy_media_worker.rb | 14 -- .../maintenance/redownload_account_media_worker.rb | 16 -- app/workers/maintenance/uncache_media_worker.rb | 18 -- app/workers/maintenance/uncache_preview_worker.rb | 18 -- lib/mastodon/accounts_cli.rb | 192 +++++++++------------ lib/mastodon/cache_cli.rb | 16 +- lib/mastodon/cli_helper.rb | 49 ++++++ lib/mastodon/domains_cli.rb | 27 ++- lib/mastodon/feeds_cli.rb | 42 +---- lib/mastodon/media_cli.rb | 57 ++---- lib/mastodon/preview_cards_cli.rb | 83 +++------ 15 files changed, 200 insertions(+), 337 deletions(-) delete mode 100644 app/workers/maintenance/destroy_media_worker.rb delete mode 100644 app/workers/maintenance/redownload_account_media_worker.rb delete mode 100644 app/workers/maintenance/uncache_media_worker.rb delete mode 100644 app/workers/maintenance/uncache_preview_worker.rb (limited to 'lib') diff --git a/Gemfile b/Gemfile index 73edb2a6a..af0e8e2fc 100644 --- a/Gemfile +++ b/Gemfile @@ -77,6 +77,7 @@ gem 'rails-settings-cached', '~> 0.6' gem 'redis', '~> 4.1', require: ['redis', 'redis/connection/hiredis'] gem 'mario-redis-lock', '~> 1.2', require: 'redis_lock' gem 'rqrcode', '~> 0.10' +gem 'ruby-progressbar', '~> 1.10' gem 'sanitize', '~> 5.1' gem 'sidekiq', '~> 5.2' gem 'sidekiq-scheduler', '~> 3.0' diff --git a/Gemfile.lock b/Gemfile.lock index 125cca4ba..6e931c611 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -769,6 +769,7 @@ DEPENDENCIES rspec-sidekiq (~> 3.0) rubocop (~> 0.74) rubocop-rails (~> 2.3) + ruby-progressbar (~> 1.10) sanitize (~> 5.1) sidekiq (~> 5.2) sidekiq-bulk (~> 0.2.0) diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb index 44f8e6be6..b58025015 100644 --- a/app/models/media_attachment.rb +++ b/app/models/media_attachment.rb @@ -129,6 +129,7 @@ class MediaAttachment < ApplicationRecord scope :unattached, -> { where(status_id: nil, scheduled_status_id: nil) } scope :local, -> { where(remote_url: '') } scope :remote, -> { where.not(remote_url: '') } + scope :cached, -> { remote.where.not(file_file_name: nil) } default_scope { order(id: :asc) } diff --git a/app/models/preview_card.rb b/app/models/preview_card.rb index a792b352b..9d6c1938a 100644 --- a/app/models/preview_card.rb +++ b/app/models/preview_card.rb @@ -43,6 +43,8 @@ class PreviewCard < ApplicationRecord validates_attachment_size :image, less_than: LIMIT remotable_attachment :image, LIMIT + scope :cached, -> { where.not(image_file_name: [nil, '']) } + before_save :extract_dimensions, if: :link? def save_with_optional_image! diff --git a/app/workers/maintenance/destroy_media_worker.rb b/app/workers/maintenance/destroy_media_worker.rb deleted file mode 100644 index cde33d6d7..000000000 --- a/app/workers/maintenance/destroy_media_worker.rb +++ /dev/null @@ -1,14 +0,0 @@ -# frozen_string_literal: true - -class Maintenance::DestroyMediaWorker - include Sidekiq::Worker - - sidekiq_options queue: 'pull' - - def perform(media_attachment_id) - media = media_attachment_id.is_a?(MediaAttachment) ? media_attachment_id : MediaAttachment.find(media_attachment_id) - media.destroy - rescue ActiveRecord::RecordNotFound - true - end -end diff --git a/app/workers/maintenance/redownload_account_media_worker.rb b/app/workers/maintenance/redownload_account_media_worker.rb deleted file mode 100644 index 6afbe6e19..000000000 --- a/app/workers/maintenance/redownload_account_media_worker.rb +++ /dev/null @@ -1,16 +0,0 @@ -# frozen_string_literal: true - -class Maintenance::RedownloadAccountMediaWorker - include Sidekiq::Worker - - sidekiq_options queue: 'pull', retry: false - - def perform(account_id) - account = account_id.is_a?(Account) ? account_id : Account.find(account_id) - account.reset_avatar! - account.reset_header! - account.save - rescue ActiveRecord::RecordNotFound - true - end -end diff --git a/app/workers/maintenance/uncache_media_worker.rb b/app/workers/maintenance/uncache_media_worker.rb deleted file mode 100644 index 4bc62ef75..000000000 --- a/app/workers/maintenance/uncache_media_worker.rb +++ /dev/null @@ -1,18 +0,0 @@ -# frozen_string_literal: true - -class Maintenance::UncacheMediaWorker - include Sidekiq::Worker - - sidekiq_options queue: 'pull' - - def perform(media_attachment_id) - media = media_attachment_id.is_a?(MediaAttachment) ? media_attachment_id : MediaAttachment.find(media_attachment_id) - - return if media.file.blank? - - media.file.destroy - media.save - rescue ActiveRecord::RecordNotFound - true - end -end diff --git a/app/workers/maintenance/uncache_preview_worker.rb b/app/workers/maintenance/uncache_preview_worker.rb deleted file mode 100644 index 810ffd8cc..000000000 --- a/app/workers/maintenance/uncache_preview_worker.rb +++ /dev/null @@ -1,18 +0,0 @@ -# frozen_string_literal: true - -class Maintenance::UncachePreviewWorker - include Sidekiq::Worker - - sidekiq_options queue: 'pull' - - def perform(preview_card_id) - preview_card = PreviewCard.find(preview_card_id) - - return if preview_card.image.blank? - - preview_card.image.destroy - preview_card.save - rescue ActiveRecord::RecordNotFound - true - end -end diff --git a/lib/mastodon/accounts_cli.rb b/lib/mastodon/accounts_cli.rb index d1854acc0..b16bf2e38 100644 --- a/lib/mastodon/accounts_cli.rb +++ b/lib/mastodon/accounts_cli.rb @@ -7,6 +7,8 @@ require_relative 'cli_helper' module Mastodon class AccountsCLI < Thor + include CLIHelper + def self.exit_on_failure? true end @@ -26,18 +28,20 @@ module Mastodon if options[:all] processed = 0 delay = 0 + scope = Account.local.without_suspended + progress = create_progress_bar(scope.count) - Account.local.without_suspended.find_in_batches do |accounts| + scope.find_in_batches do |accounts| accounts.each do |account| rotate_keys_for_account(account, delay) + progress.increment processed += 1 - say('.', :green, false) end delay += 5.minutes end - say + progress.finish say("OK, rotated keys for #{processed} accounts", :green) elsif username.present? rotate_keys_for_account(Account.find_local(username)) @@ -206,6 +210,8 @@ module Mastodon say('OK', :green) end + option :concurrency, type: :numeric, default: 5, aliases: [:c] + option :verbose, type: :boolean, aliases: [:v] option :dry_run, type: :boolean desc 'cull', 'Remove remote accounts that no longer exist' long_desc <<-LONG_DESC @@ -215,63 +221,45 @@ module Mastodon Accounts that have had confirmed activity within the last week are excluded from the checks. - - Domains that are unreachable are not checked. - - With the --dry-run option, no deletes will actually be carried - out. LONG_DESC def cull skip_threshold = 7.days.ago - culled = 0 - dry_run_culled = [] - skip_domains = Set.new dry_run = options[:dry_run] ? ' (DRY RUN)' : '' + skip_domains = Concurrent::Set.new - Account.remote.where(protocol: :activitypub).partitioned.find_each do |account| - next if account.updated_at >= skip_threshold || (account.last_webfingered_at.present? && account.last_webfingered_at >= skip_threshold) + processed, culled = parallelize_with_progress(Account.remote.where(protocol: :activitypub).partitioned) do |account| + next if account.updated_at >= skip_threshold || (account.last_webfingered_at.present? && account.last_webfingered_at >= skip_threshold) || skip_domains.include?(account.domain) code = 0 - unless skip_domains.include?(account.domain) - begin - code = Request.new(:head, account.uri).perform(&:code) - rescue HTTP::ConnectionError - skip_domains << account.domain - rescue StandardError - next - end + + begin + code = Request.new(:head, account.uri).perform(&:code) + rescue HTTP::ConnectionError + skip_domains << account.domain end if [404, 410].include?(code) - if options[:dry_run] - dry_run_culled << account.acct - else - SuspendAccountService.new.call(account, destroy: true) - end - culled += 1 - say('+', :green, false) + SuspendAccountService.new.call(account, destroy: true) unless options[:dry_run] + 1 else - account.touch # Touch account even during dry run to avoid getting the account into the window again - say('.', nil, false) + # Touch account even during dry run to avoid getting the account into the window again + account.touch end end - say - say("Removed #{culled} accounts. #{skip_domains.size} servers skipped#{dry_run}", skip_domains.empty? ? :green : :yellow) + say("Visited #{processed} accounts, removed #{culled}#{dry_run}", :green) unless skip_domains.empty? - say('The following servers were not available during the check:', :yellow) + say('The following domains were not available during the check:', :yellow) skip_domains.each { |domain| say(' ' + domain) } end - - unless dry_run_culled.empty? - say('The following accounts would have been deleted:', :green) - dry_run_culled.each { |account| say(' ' + account) } - end end option :all, type: :boolean option :domain + option :concurrency, type: :numeric, default: 5, aliases: [:c] + option :verbose, type: :boolean, aliases: [:v] + option :dry_run, type: :boolean desc 'refresh [USERNAME]', 'Fetch remote user data and files' long_desc <<-LONG_DESC Fetch remote user data and files for one or multiple accounts. @@ -280,21 +268,23 @@ module Mastodon Through the --domain option, this can be narrowed down to a specific domain only. Otherwise, a single remote account must be specified with USERNAME. - - All processing is done in the background through Sidekiq. LONG_DESC def refresh(username = nil) + dry_run = options[:dry_run] ? ' (DRY RUN)' : '' + if options[:domain] || options[:all] - queued = 0 scope = Account.remote scope = scope.where(domain: options[:domain]) if options[:domain] - scope.select(:id).reorder(nil).find_in_batches do |accounts| - Maintenance::RedownloadAccountMediaWorker.push_bulk(accounts.map(&:id)) - queued += accounts.size + processed, = parallelize_with_progress(scope) do |account| + next if options[:dry_run] + + account.reset_avatar! + account.reset_header! + account.save end - say("Scheduled refreshment of #{queued} accounts", :green, true) + say("Refreshed #{processed} accounts#{dry_run}", :green, true) elsif username.present? username, domain = username.split('@') account = Account.find_remote(username, domain) @@ -304,76 +294,53 @@ module Mastodon exit(1) end - Maintenance::RedownloadAccountMediaWorker.perform_async(account.id) - say('OK', :green) + unless options[:dry_run] + account.reset_avatar! + account.reset_header! + account.save + end + + say("OK#{dry_run}", :green) else say('No account(s) given', :red) exit(1) end end - desc 'follow ACCT', 'Make all local accounts follow account specified by ACCT' - long_desc <<-LONG_DESC - Make all local accounts follow another local account specified by ACCT. - ACCT should be the username only. - LONG_DESC - def follow(acct) - if acct.include? '@' - say('Target account name should not contain a target instance, since it has to be a local account.', :red) - exit(1) - end - - target_account = ResolveAccountService.new.call(acct) - processed = 0 - failed = 0 + option :concurrency, type: :numeric, default: 5, aliases: [:c] + option :verbose, type: :boolean, aliases: [:v] + desc 'follow USERNAME', 'Make all local accounts follow account specified by USERNAME' + def follow(username) + target_account = Account.find_local(username) if target_account.nil? - say("Target account (#{acct}) could not be resolved", :red) + say('No such account', :red) exit(1) end - Account.local.without_suspended.find_each do |account| - begin - FollowService.new.call(account, target_account) - processed += 1 - say('.', :green, false) - rescue StandardError - failed += 1 - say('.', :red, false) - end + processed, = parallelize_with_progress(Account.local.without_suspended) do |account| + FollowService.new.call(account, target_account) end - say("OK, followed target from #{processed} accounts, skipped #{failed}", :green) + say("OK, followed target from #{processed} accounts", :green) end + option :concurrency, type: :numeric, default: 5, aliases: [:c] + option :verbose, type: :boolean, aliases: [:v] desc 'unfollow ACCT', 'Make all local accounts unfollow account specified by ACCT' - long_desc <<-LONG_DESC - Make all local accounts unfollow an account specified by ACCT. ACCT can be - a simple username, in case of a local user. It can also be in the format - username@domain, in case of a remote user. - LONG_DESC def unfollow(acct) target_account = Account.find_remote(*acct.split('@')) - processed = 0 - failed = 0 if target_account.nil? - say("Target account (#{acct}) was not found", :red) + say('No such account', :red) exit(1) end - target_account.followers.local.find_each do |account| - begin - UnfollowService.new.call(account, target_account) - processed += 1 - say('.', :green, false) - rescue StandardError - failed += 1 - say('.', :red, false) - end + parallelize_with_progress(target_account.followers.local) do |account| + UnfollowService.new.call(account, target_account) end - say("OK, unfollowed target from #{processed} accounts, skipped #{failed}", :green) + say("OK, unfollowed target from #{processed} accounts", :green) end option :follows, type: :boolean, default: false @@ -396,51 +363,50 @@ module Mastodon account = Account.find_local(username) if account.nil? - say('No user with such username', :red) + say('No such account', :red) exit(1) end - if options[:follows] - processed = 0 - failed = 0 + total = 0 + total += Account.where(id: ::Follow.where(account: account).select(:target_account_id)).count if options[:follows] + total += Account.where(id: ::Follow.where(target_account: account).select(:account_id)).count if options[:followers] + progress = create_progress_bar(total) + processed = 0 - say("Unfollowing #{account.username}'s followees, this might take a while...") + if options[:follows] + scope = Account.where(id: ::Follow.where(account: account).select(:target_account_id)) - Account.where(id: ::Follow.where(account: account).select(:target_account_id)).find_each do |target_account| + scope.find_each do |target_account| begin UnfollowService.new.call(account, target_account) + rescue => e + progress.log pastel.red("Error processing #{target_account.id}: #{e}") + ensure + progress.increment processed += 1 - say('.', :green, false) - rescue StandardError - failed += 1 - say('.', :red, false) end end BootstrapTimelineWorker.perform_async(account.id) - - say("OK, unfollowed #{processed} followees, skipped #{failed}", :green) end if options[:followers] - processed = 0 - failed = 0 - - say("Removing #{account.username}'s followers, this might take a while...") + scope = Account.where(id: ::Follow.where(target_account: account).select(:account_id)) - Account.where(id: ::Follow.where(target_account: account).select(:account_id)).find_each do |target_account| + scope.find_each do |target_account| begin UnfollowService.new.call(target_account, account) + rescue => e + progress.log pastel.red("Error processing #{target_account.id}: #{e}") + ensure + progress.increment processed += 1 - say('.', :green, false) - rescue StandardError - failed += 1 - say('.', :red, false) end end - - say("OK, removed #{processed} followers, skipped #{failed}", :green) end + + progress.finish + say("Processed #{processed} relationships", :green, true) end option :number, type: :numeric, aliases: [:n] diff --git a/lib/mastodon/cache_cli.rb b/lib/mastodon/cache_cli.rb index 5b0eea91b..803404c34 100644 --- a/lib/mastodon/cache_cli.rb +++ b/lib/mastodon/cache_cli.rb @@ -6,6 +6,8 @@ require_relative 'cli_helper' module Mastodon class CacheCLI < Thor + include CLIHelper + def self.exit_on_failure? true end @@ -16,6 +18,8 @@ module Mastodon say('OK', :green) end + option :concurrency, type: :numeric, default: 5, aliases: [:c] + option :verbose, type: :boolean, aliases: [:v] desc 'recount TYPE', 'Update hard-cached counters' long_desc <<~LONG_DESC Update hard-cached counters of TYPE by counting referenced @@ -25,32 +29,24 @@ module Mastodon size of the database. LONG_DESC def recount(type) - processed = 0 - case type when 'accounts' - Account.local.includes(:account_stat).find_each do |account| + processed, = parallelize_with_progress(Account.local.includes(:account_stat)) do |account| account_stat = account.account_stat account_stat.following_count = account.active_relationships.count account_stat.followers_count = account.passive_relationships.count account_stat.statuses_count = account.statuses.where.not(visibility: :direct).count account_stat.save if account_stat.changed? - - processed += 1 - say('.', :green, false) end when 'statuses' - Status.includes(:status_stat).find_each do |status| + processed, = parallelize_with_progress(Status.includes(:status_stat)) do |status| status_stat = status.status_stat status_stat.replies_count = status.replies.where.not(visibility: :direct).count status_stat.reblogs_count = status.reblogs.count status_stat.favourites_count = status.favourites.count status_stat.save if status_stat.changed? - - processed += 1 - say('.', :green, false) end else say("Unknown type: #{type}", :red) diff --git a/lib/mastodon/cli_helper.rb b/lib/mastodon/cli_helper.rb index 2f807d08c..da7348349 100644 --- a/lib/mastodon/cli_helper.rb +++ b/lib/mastodon/cli_helper.rb @@ -7,3 +7,52 @@ ActiveRecord::Base.logger = dev_null ActiveJob::Base.logger = dev_null HttpLog.configuration.logger = dev_null Paperclip.options[:log] = false + +module Mastodon + module CLIHelper + def create_progress_bar(total = nil) + ProgressBar.create(total: total, format: '%c/%u |%b%i| %e') + end + + def parallelize_with_progress(scope) + ActiveRecord::Base.configurations[Rails.env]['pool'] = options[:concurrency] + + progress = create_progress_bar(scope.count) + pool = Concurrent::FixedThreadPool.new(options[:concurrency]) + total = Concurrent::AtomicFixnum.new(0) + aggregate = Concurrent::AtomicFixnum.new(0) + + scope.reorder(nil).find_in_batches do |items| + futures = [] + + items.each do |item| + futures << Concurrent::Future.execute(executor: pool) do + ActiveRecord::Base.connection_pool.with_connection do + begin + progress.log("Processing #{item.id}") if options[:verbose] + + result = yield(item) + aggregate.increment(result) if result.is_a?(Integer) + rescue => e + progress.log pastel.red("Error processing #{item.id}: #{e}") + ensure + progress.increment + end + end + end + end + + total.increment(items.size) + futures.map(&:value) + end + + progress.finish + + [total.value, aggregate.value] + end + + def pastel + @pastel ||= Pastel.new + end + end +end diff --git a/lib/mastodon/domains_cli.rb b/lib/mastodon/domains_cli.rb index 17cafd1bc..c612c2d72 100644 --- a/lib/mastodon/domains_cli.rb +++ b/lib/mastodon/domains_cli.rb @@ -7,10 +7,14 @@ require_relative 'cli_helper' module Mastodon class DomainsCLI < Thor + include CLIHelper + def self.exit_on_failure? true end + option :concurrency, type: :numeric, default: 5, aliases: [:c] + option :verbose, type: :boolean, aliases: [:v] option :dry_run, type: :boolean option :whitelist_mode, type: :boolean desc 'purge [DOMAIN]', 'Remove accounts from a DOMAIN without a trace' @@ -24,7 +28,6 @@ module Mastodon are removed from the database. LONG_DESC def purge(domain = nil) - removed = 0 dry_run = options[:dry_run] ? ' (DRY RUN)' : '' scope = begin @@ -38,25 +41,22 @@ module Mastodon end end - scope.find_each do |account| + processed, = parallelize_with_progress(scope) do |account| SuspendAccountService.new.call(account, destroy: true) unless options[:dry_run] - removed += 1 - say('.', :green, false) end DomainBlock.where(domain: domain).destroy_all unless options[:dry_run] - say - say("Removed #{removed} accounts#{dry_run}", :green) + say("Removed #{processed} accounts#{dry_run}", :green) custom_emojis = CustomEmoji.where(domain: domain) custom_emojis_count = custom_emojis.count custom_emojis.destroy_all unless options[:dry_run] + say("Removed #{custom_emojis_count} custom emojis", :green) end option :concurrency, type: :numeric, default: 50, aliases: [:c] - option :silent, type: :boolean, default: false, aliases: [:s] option :format, type: :string, default: 'summary', aliases: [:f] option :exclude_suspended, type: :boolean, default: false, aliases: [:x] desc 'crawl [START]', 'Crawl all known peers, optionally beginning at START' @@ -69,8 +69,6 @@ module Mastodon The --concurrency (-c) option controls the number of threads performing HTTP requests at the same time. More threads means the crawl may complete faster. - The --silent (-s) option controls progress output. - The --format (-f) option controls how the data is displayed at the end. By default (`summary`), a summary of the statistics is returned. The other options are `domains`, which returns a newline-delimited list of all discovered peers, @@ -87,6 +85,7 @@ module Mastodon start_at = Time.now.to_f seed = start ? [start] : Account.remote.domains blocked_domains = Regexp.new('\\.?' + DomainBlock.where(severity: 1).pluck(:domain).join('|') + '$') + progress = create_progress_bar pool = Concurrent::ThreadPoolExecutor.new(min_threads: 0, max_threads: options[:concurrency], idletime: 10, auto_terminate: true, max_queue: 0) @@ -95,7 +94,6 @@ module Mastodon next if options[:exclude_suspended] && domain.match(blocked_domains) stats[domain] = nil - processed.increment begin Request.new(:get, "https://#{domain}/api/v1/instance").perform do |res| @@ -115,11 +113,11 @@ module Mastodon next unless res.code == 200 stats[domain]['activity'] = Oj.load(res.to_s) end - - say('.', :green, false) unless options[:silent] rescue StandardError failed.increment - say('.', :red, false) unless options[:silent] + ensure + processed.increment + progress.increment unless progress.finished? end end @@ -133,10 +131,9 @@ module Mastodon pool.shutdown pool.wait_for_termination(20) ensure + progress.finish pool.shutdown - say unless options[:silent] - case options[:format] when 'summary' stats_to_summary(stats, processed, failed, start_at) diff --git a/lib/mastodon/feeds_cli.rb b/lib/mastodon/feeds_cli.rb index fe11c3df4..ea7c90dff 100644 --- a/lib/mastodon/feeds_cli.rb +++ b/lib/mastodon/feeds_cli.rb @@ -6,55 +6,33 @@ require_relative 'cli_helper' module Mastodon class FeedsCLI < Thor + include CLIHelper + def self.exit_on_failure? true end option :all, type: :boolean, default: false - option :background, type: :boolean, default: false + option :concurrency, type: :numeric, default: 5, aliases: [:c] + option :verbose, type: :boolean, aliases: [:v] option :dry_run, type: :boolean, default: false - option :verbose, type: :boolean, default: false desc 'build [USERNAME]', 'Build home and list feeds for one or all users' long_desc <<-LONG_DESC Build home and list feeds that are stored in Redis from the database. With the --all option, all active users will be processed. Otherwise, a single user specified by USERNAME. - - With the --background option, regeneration will be queued into Sidekiq, - and the command will exit as soon as possible. - - With the --dry-run option, no work will be done. - - With the --verbose option, when accounts are processed sequentially in the - foreground, the IDs of the accounts will be printed. LONG_DESC def build(username = nil) dry_run = options[:dry_run] ? '(DRY RUN)' : '' if options[:all] || username.nil? - processed = 0 - queued = 0 - User.active.select(:id, :account_id).reorder(nil).find_in_batches do |users| - if options[:background] - RegenerationWorker.push_bulk(users.map(&:account_id)) unless options[:dry_run] - queued += users.size - else - users.each do |user| - RegenerationWorker.new.perform(user.account_id) unless options[:dry_run] - options[:verbose] ? say(user.account_id) : say('.', :green, false) - processed += 1 - end - end + processed, = parallelize_with_progress(Account.joins(:user).merge(User.active)) do |account| + PrecomputeFeedService.new.call(account) unless options[:dry_run] end - if options[:background] - say("Scheduled feed regeneration for #{queued} accounts #{dry_run}", :green, true) - else - say - say("Regenerated feeds for #{processed} accounts #{dry_run}", :green, true) - end + say("Regenerated feeds for #{processed} accounts #{dry_run}", :green, true) elsif username.present? account = Account.find_local(username) @@ -63,11 +41,7 @@ module Mastodon exit(1) end - if options[:background] - RegenerationWorker.perform_async(account.id) unless options[:dry_run] - else - RegenerationWorker.new.perform(account.id) unless options[:dry_run] - end + PrecomputeFeedService.new.call(account) unless options[:dry_run] say("OK #{dry_run}", :green, true) else diff --git a/lib/mastodon/media_cli.rb b/lib/mastodon/media_cli.rb index 6152d5a09..0659b6b65 100644 --- a/lib/mastodon/media_cli.rb +++ b/lib/mastodon/media_cli.rb @@ -7,14 +7,15 @@ require_relative 'cli_helper' module Mastodon class MediaCLI < Thor include ActionView::Helpers::NumberHelper + include CLIHelper def self.exit_on_failure? true end - option :days, type: :numeric, default: 7 - option :background, type: :boolean, default: false - option :verbose, type: :boolean, default: false + option :days, type: :numeric, default: 7, aliases: [:d] + option :concurrency, type: :numeric, default: 5, aliases: [:c] + option :verbose, type: :boolean, default: false, aliases: [:v] option :dry_run, type: :boolean, default: false desc 'remove', 'Remove remote media files' long_desc <<-DESC @@ -22,49 +23,25 @@ module Mastodon The --days option specifies how old media attachments have to be before they are removed. It defaults to 7 days. - - With the --background option, instead of deleting the files sequentially, - they will be queued into Sidekiq and the command will exit as soon as - possible. In Sidekiq they will be processed with higher concurrency, but - it may impact other operations of the Mastodon server, and it may overload - the underlying file storage. - - With the --dry-run option, no work will be done. - - With the --verbose option, when media attachments are processed sequentially in the - foreground, the IDs of the media attachments will be printed. DESC def remove - time_ago = options[:days].days.ago - queued = 0 - processed = 0 - size = 0 - dry_run = options[:dry_run] ? '(DRY RUN)' : '' + time_ago = options[:days].days.ago + dry_run = options[:dry_run] ? '(DRY RUN)' : '' - if options[:background] - MediaAttachment.where.not(remote_url: '').where.not(file_file_name: nil).where('created_at < ?', time_ago).select(:id, :file_file_size).reorder(nil).find_in_batches do |media_attachments| - queued += media_attachments.size - size += media_attachments.reduce(0) { |sum, m| sum + (m.file_file_size || 0) } - Maintenance::UncacheMediaWorker.push_bulk(media_attachments.map(&:id)) unless options[:dry_run] - end - else - MediaAttachment.where.not(remote_url: '').where.not(file_file_name: nil).where('created_at < ?', time_ago).reorder(nil).find_in_batches do |media_attachments| - media_attachments.each do |m| - size += m.file_file_size || 0 - Maintenance::UncacheMediaWorker.new.perform(m) unless options[:dry_run] - options[:verbose] ? say(m.id) : say('.', :green, false) - processed += 1 - end - end - end + processed, aggregate = parallelize_with_progress(MediaAttachment.cached.where.not(remote_url: '').where('created_at < ?', time_ago)) do |media_attachment| + next if media_attachment.file.blank? + + size = media_attachment.file_file_size - say + unless options[:dry_run] + media_attachment.file.destroy + media_attachment.save + end - if options[:background] - say("Scheduled the deletion of #{queued} media attachments (approx. #{number_to_human_size(size)}) #{dry_run}", :green, true) - else - say("Removed #{processed} media attachments (approx. #{number_to_human_size(size)}) #{dry_run}", :green, true) + size end + + say("Removed #{processed} media attachments (approx. #{number_to_human_size(aggregate)}) #{dry_run}", :green, true) end end end diff --git a/lib/mastodon/preview_cards_cli.rb b/lib/mastodon/preview_cards_cli.rb index 465fe7d0b..cf4407250 100644 --- a/lib/mastodon/preview_cards_cli.rb +++ b/lib/mastodon/preview_cards_cli.rb @@ -8,87 +8,52 @@ require_relative 'cli_helper' module Mastodon class PreviewCardsCLI < Thor include ActionView::Helpers::NumberHelper + include CLIHelper def self.exit_on_failure? true end option :days, type: :numeric, default: 180 - option :background, type: :boolean, default: false - option :verbose, type: :boolean, default: false + option :concurrency, type: :numeric, default: 5, aliases: [:c] + option :verbose, type: :boolean, aliases: [:v] option :dry_run, type: :boolean, default: false option :link, type: :boolean, default: false desc 'remove', 'Remove preview cards' long_desc <<-DESC - Removes locally thumbnails for previews. + Removes local thumbnails for preview cards. The --days option specifies how old preview cards have to be before - they are removed. It defaults to 180 days. + they are removed. It defaults to 180 days. Since preview cards will + not be re-fetched unless the link is re-posted after 2 weeks from + last time, it is not recommended to delete preview cards within the + last 14 days. - With the --background option, instead of deleting the files sequentially, - they will be queued into Sidekiq and the command will exit as soon as - possible. In Sidekiq they will be processed with higher concurrency, but - it may impact other operations of the Mastodon server, and it may overload - the underlying file storage. - - With the --dry-run option, no work will be done. - - With the --verbose option, when preview cards are processed sequentially in the - foreground, the IDs of the preview cards will be printed. - - With the --link option, delete only link-type preview cards. + With the --link option, only link-type preview cards will be deleted, + leaving video and photo cards untouched. DESC def remove - prompt = TTY::Prompt.new - time_ago = options[:days].days.ago - queued = 0 - processed = 0 - size = 0 - dry_run = options[:dry_run] ? '(DRY RUN)' : '' - link = options[:link] ? 'link-type ' : '' - scope = PreviewCard.where.not(image_file_name: nil) - scope = scope.where.not(image_file_name: '') - scope = scope.where(type: :link) if options[:link] - scope = scope.where('updated_at < ?', time_ago) + time_ago = options[:days].days.ago + dry_run = options[:dry_run] ? ' (DRY RUN)' : '' + link = options[:link] ? 'link-type ' : '' + scope = PreviewCard.cached + scope = scope.where(type: :link) if options[:link] + scope = scope.where('updated_at < ?', time_ago) - if time_ago > 2.weeks.ago - prompt.say "\n" - prompt.say('The preview cards less than the past two weeks will not be re-acquired even when needed.') - prompt.say "\n" + processed, aggregate = parallelize_with_progress(scope) do |preview_card| + next if preview_card.image.blank? - unless prompt.yes?('Are you sure you want to delete the preview cards?', default: false) - prompt.say "\n" - prompt.warn 'Nothing execute. Bye!' - prompt.say "\n" - exit(1) - end - end + size = preview_card.image_file_size - if options[:background] - scope.select(:id, :image_file_size).reorder(nil).find_in_batches do |preview_cards| - queued += preview_cards.size - size += preview_cards.reduce(0) { |sum, p| sum + (p.image_file_size || 0) } - Maintenance::UncachePreviewWorker.push_bulk(preview_cards.map(&:id)) unless options[:dry_run] + unless options[:dry_run] + preview_card.image.destroy + preview_card.save end - else - scope.select(:id, :image_file_size).reorder(nil).find_in_batches do |preview_cards| - preview_cards.each do |p| - size += p.image_file_size || 0 - Maintenance::UncachePreviewWorker.new.perform(p.id) unless options[:dry_run] - options[:verbose] ? say(p.id) : say('.', :green, false) - processed += 1 - end - end + size end - say - - if options[:background] - say("Scheduled the deletion of #{queued} #{link}preview cards (approx. #{number_to_human_size(size)}) #{dry_run}", :green, true) - else - say("Removed #{processed} #{link}preview cards (approx. #{number_to_human_size(size)}) #{dry_run}", :green, true) - end + say("Removed #{processed} #{link}preview cards (approx. #{number_to_human_size(aggregate)})#{dry_run}", :green, true) end end end -- cgit From 031ca25014e0ba88d3dcc3086947b41449a672e2 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 10 Sep 2019 15:29:12 +0200 Subject: Add retry for failed media downloads and `tootctl media refresh` (#11775) --- app/lib/activitypub/activity/create.rb | 21 +++++++------ app/models/concerns/remotable.rb | 12 ++++---- app/models/media_attachment.rb | 2 +- app/workers/redownload_media_worker.rb | 19 ++++++++++++ lib/mastodon/media_cli.rb | 54 ++++++++++++++++++++++++++++++++++ 5 files changed, 92 insertions(+), 16 deletions(-) create mode 100644 app/workers/redownload_media_worker.rb (limited to 'lib') diff --git a/app/lib/activitypub/activity/create.rb b/app/lib/activitypub/activity/create.rb index 000b77df5..dea7fd43c 100644 --- a/app/lib/activitypub/activity/create.rb +++ b/app/lib/activitypub/activity/create.rb @@ -189,22 +189,25 @@ class ActivityPub::Activity::Create < ActivityPub::Activity media_attachments = [] as_array(@object['attachment']).each do |attachment| - next if attachment['url'].blank? + next if attachment['url'].blank? || media_attachments.size >= 4 - href = Addressable::URI.parse(attachment['url']).normalize.to_s - media_attachment = MediaAttachment.create(account: @account, remote_url: href, description: attachment['name'].presence, focus: attachment['focalPoint'], blurhash: supported_blurhash?(attachment['blurhash']) ? attachment['blurhash'] : nil) - media_attachments << media_attachment + begin + href = Addressable::URI.parse(attachment['url']).normalize.to_s + media_attachment = MediaAttachment.create(account: @account, remote_url: href, description: attachment['name'].presence, focus: attachment['focalPoint'], blurhash: supported_blurhash?(attachment['blurhash']) ? attachment['blurhash'] : nil) + media_attachments << media_attachment - next if unsupported_media_type?(attachment['mediaType']) || skip_download? + next if unsupported_media_type?(attachment['mediaType']) || skip_download? - media_attachment.file_remote_url = href - media_attachment.save + media_attachment.file_remote_url = href + media_attachment.save + rescue Mastodon::UnexpectedResponseError, HTTP::TimeoutError, HTTP::ConnectionError, OpenSSL::SSL::SSLError + RedownloadMediaWorker.perform_in(rand(30..600).seconds, media_attachment.id) + end end media_attachments rescue Addressable::URI::InvalidURIError => e - Rails.logger.debug e - + Rails.logger.debug "Invalid URL in attachment: #{e}" media_attachments end diff --git a/app/models/concerns/remotable.rb b/app/models/concerns/remotable.rb index 9372a963b..082302619 100644 --- a/app/models/concerns/remotable.rb +++ b/app/models/concerns/remotable.rb @@ -4,7 +4,7 @@ module Remotable extend ActiveSupport::Concern class_methods do - def remotable_attachment(attachment_name, limit) + def remotable_attachment(attachment_name, limit, suppress_errors: true) attribute_name = "#{attachment_name}_remote_url".to_sym method_name = "#{attribute_name}=".to_sym alt_method_name = "reset_#{attachment_name}!".to_sym @@ -22,7 +22,7 @@ module Remotable begin Request.new(:get, url).perform do |response| - next if response.code != 200 + raise Mastodon::UnexpectedResponseError, response unless (200...300).cover?(response.code) content_type = parse_content_type(response.headers.get('content-type').last) extname = detect_extname_from_content_type(content_type) @@ -41,11 +41,11 @@ module Remotable self[attribute_name] = url if has_attribute?(attribute_name) end - rescue HTTP::TimeoutError, HTTP::ConnectionError, OpenSSL::SSL::SSLError, Paperclip::Errors::NotIdentifiedByImageMagickError, Addressable::URI::InvalidURIError, Mastodon::HostValidationError, Mastodon::LengthValidationError => e + rescue Mastodon::UnexpectedResponseError, HTTP::TimeoutError, HTTP::ConnectionError, OpenSSL::SSL::SSLError => e + Rails.logger.debug "Error fetching remote #{attachment_name}: #{e}" + raise e unless suppress_errors + rescue Paperclip::Errors::NotIdentifiedByImageMagickError, Addressable::URI::InvalidURIError, Mastodon::HostValidationError, Mastodon::LengthValidationError, Paperclip::Error, Mastodon::DimensionsValidationError => e Rails.logger.debug "Error fetching remote #{attachment_name}: #{e}" - nil - rescue Paperclip::Error, Mastodon::DimensionsValidationError => e - Rails.logger.debug "Error processing remote #{attachment_name}: #{e}" nil end end diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb index b58025015..a2b73f150 100644 --- a/app/models/media_attachment.rb +++ b/app/models/media_attachment.rb @@ -118,7 +118,7 @@ class MediaAttachment < ApplicationRecord validates_attachment_content_type :file, content_type: IMAGE_MIME_TYPES + VIDEO_MIME_TYPES + AUDIO_MIME_TYPES validates_attachment_size :file, less_than: IMAGE_LIMIT, unless: :larger_media_format? validates_attachment_size :file, less_than: VIDEO_LIMIT, if: :larger_media_format? - remotable_attachment :file, VIDEO_LIMIT + remotable_attachment :file, VIDEO_LIMIT, suppress_errors: false include Attachmentable diff --git a/app/workers/redownload_media_worker.rb b/app/workers/redownload_media_worker.rb new file mode 100644 index 000000000..98e995918 --- /dev/null +++ b/app/workers/redownload_media_worker.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class RedownloadMediaWorker + include Sidekiq::Worker + include ExponentialBackoff + + sidekiq_options queue: 'pull', retry: 3 + + def perform(id) + media_attachment = MediaAttachment.find(id) + + return if media_attachment.remote_url.blank? + + media_attachment.reset_file! + media_attachment.save + rescue ActiveRecord::RecordNotFound + true + end +end diff --git a/lib/mastodon/media_cli.rb b/lib/mastodon/media_cli.rb index 0659b6b65..ec2f36c30 100644 --- a/lib/mastodon/media_cli.rb +++ b/lib/mastodon/media_cli.rb @@ -43,5 +43,59 @@ module Mastodon say("Removed #{processed} media attachments (approx. #{number_to_human_size(aggregate)}) #{dry_run}", :green, true) end + + option :account, type: :string + option :domain, type: :string + option :status, type: :numeric + option :concurrency, type: :numeric, default: 5, aliases: [:c] + option :verbose, type: :boolean, default: false, aliases: [:v] + option :dry_run, type: :boolean, default: false + desc 'refresh', 'Fetch remote media files' + long_desc <<-DESC + Re-downloads media attachments from other servers. You must specify the + source of media attachments with one of the following options: + + Use the --status option to download attachments from a specific status, + using the status local numeric ID. + + Use the --account option to download attachments from a specific account, + using username@domain handle of the account. + + Use the --domain option to download attachments from a specific domain. + DESC + def refresh + dry_run = options[:dry_run] ? ' (DRY RUN)' : '' + + if options[:status] + scope = MediaAttachment.where(status_id: options[:status]) + elsif options[:account] + username, domain = username.split('@') + account = Account.find_remote(username, domain) + + if account.nil? + say('No such account', :red) + exit(1) + end + + scope = MediaAttachment.where(account_id: account.id) + elsif options[:domain] + scope = MediaAttachment.joins(:account).merge(Account.by_domain_and_subdomains(options[:domain])) + else + exit(1) + end + + processed, aggregate = parallelize_with_progress(scope) do |media_attachment| + next if media_attachment.remote_url.blank? + + unless options[:dry_run] + media_attachment.reset_file! + media_attachment.save + end + + media_attachment.file_file_size + end + + say("Downloaded #{processed} media attachments (approx. #{number_to_human_size(aggregate)})#{dry_run}", :green, true) + end end end -- cgit From c5d37f18cb3f4d6212fb8f3e1c4e1e027f677ec5 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Wed, 11 Sep 2019 16:32:44 +0200 Subject: Change deletes to preserve soft-deleted statuses in unresolved reports (#11805) Change all account actions except "none" to resolve all unresolved reports Refactor `SuspendAccountService` to be more readable --- app/controllers/admin/accounts_controller.rb | 2 +- app/controllers/admin/report_notes_controller.rb | 9 ++-- .../api/v1/admin/accounts_controller.rb | 2 +- app/lib/activitypub/activity/delete.rb | 3 +- app/models/account.rb | 1 + app/models/admin/account_action.rb | 24 +++++++-- app/models/form/account_batch.rb | 2 +- app/models/form/status_batch.rb | 2 +- app/models/report.rb | 1 + app/models/status.rb | 4 ++ app/models/user.rb | 4 ++ app/services/block_domain_service.rb | 2 +- app/services/remove_status_service.rb | 7 +-- app/services/suspend_account_service.rb | 62 ++++++++++++++++------ app/services/unallow_domain_service.rb | 2 +- app/workers/admin/suspension_worker.rb | 2 +- lib/mastodon/accounts_cli.rb | 4 +- lib/mastodon/domains_cli.rb | 2 +- .../admin/reported_statuses_controller_spec.rb | 2 +- spec/controllers/admin/statuses_controller_spec.rb | 2 +- spec/models/form/status_batch_spec.rb | 4 +- 21 files changed, 98 insertions(+), 45 deletions(-) (limited to 'lib') diff --git a/app/controllers/admin/accounts_controller.rb b/app/controllers/admin/accounts_controller.rb index 2fa1dfe5f..68b6352f8 100644 --- a/app/controllers/admin/accounts_controller.rb +++ b/app/controllers/admin/accounts_controller.rb @@ -41,7 +41,7 @@ module Admin def reject authorize @account.user, :reject? - SuspendAccountService.new.call(@account, including_user: true, destroy: true, skip_distribution: true) + SuspendAccountService.new.call(@account, reserve_email: false, reserve_username: false) redirect_to admin_pending_accounts_path end diff --git a/app/controllers/admin/report_notes_controller.rb b/app/controllers/admin/report_notes_controller.rb index bcb3f2026..b816c5b5d 100644 --- a/app/controllers/admin/report_notes_controller.rb +++ b/app/controllers/admin/report_notes_controller.rb @@ -5,10 +5,10 @@ module Admin before_action :set_report_note, only: [:destroy] def create - authorize ReportNote, :create? + authorize :report_note, :create? @report_note = current_account.report_notes.new(resource_params) - @report = @report_note.report + @report = @report_note.report if @report_note.save if params[:create_and_resolve] @@ -26,9 +26,8 @@ module Admin redirect_to admin_report_path(@report), notice: I18n.t('admin.report_notes.created_msg') else - @report_notes = @report.notes.latest - @report_history = @report.history - @form = Form::StatusBatch.new + @report_notes = (@report.notes.latest + @report.history + @report.target_account.targeted_account_warnings.latest.custom).sort_by(&:created_at) + @form = Form::StatusBatch.new render template: 'admin/reports/show' end diff --git a/app/controllers/api/v1/admin/accounts_controller.rb b/app/controllers/api/v1/admin/accounts_controller.rb index c306180ca..c35ea5ab2 100644 --- a/app/controllers/api/v1/admin/accounts_controller.rb +++ b/app/controllers/api/v1/admin/accounts_controller.rb @@ -58,7 +58,7 @@ class Api::V1::Admin::AccountsController < Api::BaseController def reject authorize @account.user, :reject? - SuspendAccountService.new.call(@account, including_user: true, destroy: true, skip_distribution: true) + SuspendAccountService.new.call(@account, reserve_email: false, reserve_username: false) render json: @account, serializer: REST::Admin::AccountSerializer end diff --git a/app/lib/activitypub/activity/delete.rb b/app/lib/activitypub/activity/delete.rb index 345060462..dc9ff580c 100644 --- a/app/lib/activitypub/activity/delete.rb +++ b/app/lib/activitypub/activity/delete.rb @@ -13,8 +13,7 @@ class ActivityPub::Activity::Delete < ActivityPub::Activity def delete_person lock_or_return("delete_in_progress:#{@account.id}") do - SuspendAccountService.new.call(@account) - @account.destroy! + SuspendAccountService.new.call(@account, reserve_username: false) end end diff --git a/app/models/account.rb b/app/models/account.rb index 8c9388b95..55fe53fae 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -115,6 +115,7 @@ class Account < ApplicationRecord :approved?, :pending?, :disabled?, + :unconfirmed_or_pending?, :role, :admin?, :moderator?, diff --git a/app/models/admin/account_action.rb b/app/models/admin/account_action.rb index c7da8b52c..b30a82369 100644 --- a/app/models/admin/account_action.rb +++ b/app/models/admin/account_action.rb @@ -83,19 +83,23 @@ class Admin::AccountAction # A log entry is only interesting if the warning contains # custom text from someone. Otherwise it's just noise. + log_action(:create, warning) if warning.text.present? end def process_reports! - return if report_id.blank? + # If we're doing "mark as resolved" on a single report, + # then we want to keep other reports open in case they + # contain new actionable information. + # + # Otherwise, we will mark all unresolved reports about + # the account as resolved. - authorize(report, :update?) + reports.each { |report| authorize(report, :update?) } - if type == 'none' + reports.each do |report| log_action(:resolve, report) report.resolve!(current_account) - else - Report.where(target_account: target_account).unresolved.update_all(action_taken: true, action_taken_by_account_id: current_account.id) end end @@ -141,6 +145,16 @@ class Admin::AccountAction @report.status_ids if @report && include_statuses end + def reports + @reports ||= begin + if type == 'none' && with_report? + [report] + else + Report.where(target_account: target_account).unresolved + end + end + end + def warning_preset @warning_preset ||= AccountWarningPreset.find(warning_preset_id) if warning_preset_id.present? end diff --git a/app/models/form/account_batch.rb b/app/models/form/account_batch.rb index f1b7a4566..0b285fde9 100644 --- a/app/models/form/account_batch.rb +++ b/app/models/form/account_batch.rb @@ -69,6 +69,6 @@ class Form::AccountBatch records = accounts.includes(:user) records.each { |account| authorize(account.user, :reject?) } - .each { |account| SuspendAccountService.new.call(account, including_user: true, destroy: true, skip_distribution: true) } + .each { |account| SuspendAccountService.new.call(account, reserve_email: false, reserve_username: false) } end end diff --git a/app/models/form/status_batch.rb b/app/models/form/status_batch.rb index e09cc2594..c4943a7ea 100644 --- a/app/models/form/status_batch.rb +++ b/app/models/form/status_batch.rb @@ -35,7 +35,7 @@ class Form::StatusBatch def delete_statuses Status.where(id: status_ids).reorder(nil).find_each do |status| status.discard - RemovalWorker.perform_async(status.id, redraft: false) + RemovalWorker.perform_async(status.id, immediate: true) Tombstone.find_or_create_by(uri: status.uri, account: status.account, by_moderator: true) log_action :destroy, status end diff --git a/app/models/report.rb b/app/models/report.rb index 1e707ff1c..fb2e040ee 100644 --- a/app/models/report.rb +++ b/app/models/report.rb @@ -59,6 +59,7 @@ class Report < ApplicationRecord end def resolve!(acting_account) + RemovalWorker.push_bulk(Status.with_discarded.discarded.where(id: status_ids).pluck(:id)) { |status_id| [status_id, { immediate: true }] } update!(action_taken: true, action_taken_by_account_id: acting_account.id) end diff --git a/app/models/status.rb b/app/models/status.rb index 9cfaddcec..471bb03b4 100644 --- a/app/models/status.rb +++ b/app/models/status.rb @@ -214,6 +214,10 @@ class Status < ApplicationRecord !sensitive? && with_media? end + def reported? + @reported ||= Report.where(target_account: account).unresolved.where('? = ANY(status_ids)', id).exists? + end + def emojis return @emojis if defined?(@emojis) diff --git a/app/models/user.rb b/app/models/user.rb index 95f1d8fc5..78b82a68f 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -171,6 +171,10 @@ class User < ApplicationRecord confirmed? && approved? && !disabled? && !account.suspended? end + def unconfirmed_or_pending? + !(confirmed? && approved?) + end + def inactive_message !approved? ? :pending : super end diff --git a/app/services/block_domain_service.rb b/app/services/block_domain_service.rb index 0ec6be503..ae461abf2 100644 --- a/app/services/block_domain_service.rb +++ b/app/services/block_domain_service.rb @@ -53,7 +53,7 @@ class BlockDomainService < BaseService def suspend_accounts! blocked_domain_accounts.without_suspended.reorder(nil).find_each do |account| - SuspendAccountService.new.call(account, suspended_at: @domain_block.created_at) + SuspendAccountService.new.call(account, reserve_username: true, suspended_at: @domain_block.created_at) end end diff --git a/app/services/remove_status_service.rb b/app/services/remove_status_service.rb index 685c1d4bf..f9352ed3d 100644 --- a/app/services/remove_status_service.rb +++ b/app/services/remove_status_service.rb @@ -8,7 +8,8 @@ class RemoveStatusService < BaseService # @param [Status] status # @param [Hash] options # @option [Boolean] :redraft - # @options [Boolean] :original_removed + # @option [Boolean] :immediate + # @option [Boolean] :original_removed def call(status, **options) @payload = Oj.dump(event: :delete, payload: status.id.to_s) @status = status @@ -31,7 +32,7 @@ class RemoveStatusService < BaseService remove_from_spam_check remove_media - @status.destroy! + @status.destroy! if @options[:immediate] || !@status.reported? else raise Mastodon::RaceConditionError end @@ -150,7 +151,7 @@ class RemoveStatusService < BaseService end def remove_media - return if @options[:redraft] + return if @options[:redraft] || (!@options[:immediate] && @status.reported?) @status.media_attachments.destroy_all end diff --git a/app/services/suspend_account_service.rb b/app/services/suspend_account_service.rb index 85da7e921..ecc893931 100644 --- a/app/services/suspend_account_service.rb +++ b/app/services/suspend_account_service.rb @@ -15,7 +15,6 @@ class SuspendAccountService < BaseService favourites follow_requests list_accounts - media_attachments mute_relationships muted_by_relationships notifications @@ -32,14 +31,26 @@ class SuspendAccountService < BaseService targeted_reports ).freeze - # Suspend an account and remove as much of its data as possible + # Suspend or remove an account and remove as much of its data + # as possible. If it's a local account and it has not been confirmed + # or never been approved, then side effects are skipped and both + # the user and account records are removed fully. Otherwise, + # it is controlled by options. # @param [Account] # @param [Hash] options - # @option [Boolean] :including_user Remove the user record as well - # @option [Boolean] :destroy Remove the account record instead of suspending + # @option [Boolean] :reserve_email Keep user record. Only applicable for local accounts + # @option [Boolean] :reserve_username Keep account record + # @option [Boolean] :skip_side_effects Side effects are ActivityPub and streaming API payloads + # @option [Time] :suspended_at Only applicable when :reserve_username is true def call(account, **options) @account = account - @options = options + @options = { reserve_username: true, reserve_email: true }.merge(options) + + if @account.local? && @account.user_unconfirmed_or_pending? + @options[:reserve_email] = false + @options[:reserve_username] = false + @options[:skip_side_effects] = true + end reject_follows! purge_user! @@ -60,27 +71,39 @@ class SuspendAccountService < BaseService def purge_user! return if !@account.local? || @account.user.nil? - if @options[:including_user] - @options[:destroy] = true if !@account.user_confirmed? || @account.user_pending? - @account.user.destroy - else + if @options[:reserve_email] @account.user.disable! @account.user.invites.where(uses: 0).destroy_all + else + @account.user.destroy end end def purge_content! - distribute_delete_actor! if @account.local? && !@options[:skip_distribution] + distribute_delete_actor! if @account.local? && !@options[:skip_side_effects] @account.statuses.reorder(nil).find_in_batches do |statuses| - BatchedRemoveStatusService.new.call(statuses, skip_side_effects: @options[:destroy]) + statuses.reject! { |status| reported_status_ids.include?(status.id) } if @options[:reserve_username] + BatchedRemoveStatusService.new.call(statuses, skip_side_effects: @options[:skip_side_effects]) + end + + @account.media_attachments.reorder(nil).find_each do |media_attachment| + next if @options[:reserve_username] && reported_status_ids.include?(media_attachment.status_id) + + media_attachment.destroy + end + + @account.polls.reorder(nil).find_each do |poll| + next if @options[:reserve_username] && reported_status_ids.include?(poll.status_id) + + poll.destroy end associations_for_destruction.each do |association_name| destroy_all(@account.public_send(association_name)) end - @account.destroy if @options[:destroy] + @account.destroy unless @options[:reserve_username] end def purge_profile! @@ -88,11 +111,13 @@ class SuspendAccountService < BaseService # there is no point wasting time updating # its values first - return if @options[:destroy] + return unless @options[:reserve_username] @account.silenced_at = nil @account.suspended_at = @options[:suspended_at] || Time.now.utc @account.locked = false + @account.memorial = false + @account.discoverable = false @account.display_name = '' @account.note = '' @account.fields = [] @@ -100,6 +125,7 @@ class SuspendAccountService < BaseService @account.followers_count = 0 @account.following_count = 0 @account.moved_to_account = nil + @account.trust_level = :untrusted @account.avatar.destroy @account.header.destroy @account.save! @@ -135,11 +161,15 @@ class SuspendAccountService < BaseService Account.inboxes - delivery_inboxes end + def reported_status_ids + @reported_status_ids ||= Report.where(target_account: @account).unresolved.pluck(:status_ids).flatten.uniq + end + def associations_for_destruction - if @options[:destroy] - ASSOCIATIONS_ON_SUSPEND + ASSOCIATIONS_ON_DESTROY - else + if @options[:reserve_username] ASSOCIATIONS_ON_SUSPEND + else + ASSOCIATIONS_ON_SUSPEND + ASSOCIATIONS_ON_DESTROY end end end diff --git a/app/services/unallow_domain_service.rb b/app/services/unallow_domain_service.rb index d4387c1a1..bd1ad328d 100644 --- a/app/services/unallow_domain_service.rb +++ b/app/services/unallow_domain_service.rb @@ -3,7 +3,7 @@ class UnallowDomainService < BaseService def call(domain_allow) Account.where(domain: domain_allow.domain).find_each do |account| - SuspendAccountService.new.call(account, destroy: true) + SuspendAccountService.new.call(account, reserve_username: false) end domain_allow.destroy diff --git a/app/workers/admin/suspension_worker.rb b/app/workers/admin/suspension_worker.rb index ae8b24d8c..83c815efd 100644 --- a/app/workers/admin/suspension_worker.rb +++ b/app/workers/admin/suspension_worker.rb @@ -6,6 +6,6 @@ class Admin::SuspensionWorker sidekiq_options queue: 'pull' def perform(account_id, remove_user = false) - SuspendAccountService.new.call(Account.find(account_id), including_user: remove_user) + SuspendAccountService.new.call(Account.find(account_id), reserve_username: true, reserve_email: !remove_user) end end diff --git a/lib/mastodon/accounts_cli.rb b/lib/mastodon/accounts_cli.rb index b16bf2e38..a09a6ab04 100644 --- a/lib/mastodon/accounts_cli.rb +++ b/lib/mastodon/accounts_cli.rb @@ -185,7 +185,7 @@ module Mastodon end say("Deleting user with #{account.statuses_count} statuses, this might take a while...") - SuspendAccountService.new.call(account, including_user: true) + SuspendAccountService.new.call(account, reserve_email: false) say('OK', :green) end @@ -239,7 +239,7 @@ module Mastodon end if [404, 410].include?(code) - SuspendAccountService.new.call(account, destroy: true) unless options[:dry_run] + SuspendAccountService.new.call(account, reserve_username: false) unless options[:dry_run] 1 else # Touch account even during dry run to avoid getting the account into the window again diff --git a/lib/mastodon/domains_cli.rb b/lib/mastodon/domains_cli.rb index c612c2d72..8e52de1c3 100644 --- a/lib/mastodon/domains_cli.rb +++ b/lib/mastodon/domains_cli.rb @@ -42,7 +42,7 @@ module Mastodon end processed, = parallelize_with_progress(scope) do |account| - SuspendAccountService.new.call(account, destroy: true) unless options[:dry_run] + SuspendAccountService.new.call(account, reserve_username: false, skip_side_effects: true) unless options[:dry_run] end DomainBlock.where(domain: domain).destroy_all unless options[:dry_run] diff --git a/spec/controllers/admin/reported_statuses_controller_spec.rb b/spec/controllers/admin/reported_statuses_controller_spec.rb index bd146b795..2a1598123 100644 --- a/spec/controllers/admin/reported_statuses_controller_spec.rb +++ b/spec/controllers/admin/reported_statuses_controller_spec.rb @@ -47,7 +47,7 @@ describe Admin::ReportedStatusesController do it 'removes a status' do allow(RemovalWorker).to receive(:perform_async) subject.call - expect(RemovalWorker).to have_received(:perform_async).with(status_ids.first, redraft: false) + expect(RemovalWorker).to have_received(:perform_async).with(status_ids.first, immediate: true) end end diff --git a/spec/controllers/admin/statuses_controller_spec.rb b/spec/controllers/admin/statuses_controller_spec.rb index 6b06343ef..d9690d83f 100644 --- a/spec/controllers/admin/statuses_controller_spec.rb +++ b/spec/controllers/admin/statuses_controller_spec.rb @@ -65,7 +65,7 @@ describe Admin::StatusesController do it 'removes a status' do allow(RemovalWorker).to receive(:perform_async) subject.call - expect(RemovalWorker).to have_received(:perform_async).with(status_ids.first, redraft: false) + expect(RemovalWorker).to have_received(:perform_async).with(status_ids.first, immediate: true) end end diff --git a/spec/models/form/status_batch_spec.rb b/spec/models/form/status_batch_spec.rb index f9c58c90f..68d84a737 100644 --- a/spec/models/form/status_batch_spec.rb +++ b/spec/models/form/status_batch_spec.rb @@ -41,12 +41,12 @@ describe Form::StatusBatch do it 'call RemovalWorker' do form.save - expect(RemovalWorker).to have_received(:perform_async).with(status.id, redraft: false) + expect(RemovalWorker).to have_received(:perform_async).with(status.id, immediate: true) end it 'do not call RemovalWorker' do form.save - expect(RemovalWorker).not_to have_received(:perform_async).with(another_status.id, redraft: false) + expect(RemovalWorker).not_to have_received(:perform_async).with(another_status.id, immediate: true) end end end -- cgit From 9361981388ed372b0502aec7d74fee9cacb4cf86 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Thu, 12 Sep 2019 04:58:33 +0200 Subject: Add missing locale file for ga and add rake task to check for it (#11813) * Add missing locale file for ga and add rake task to check for it * Update lib/tasks/repo.rake Co-Authored-By: Yamagishi Kazutoshi * Fix check-i18n build --- .circleci/config.yml | 2 ++ app/javascript/mastodon/locales/ga.json | 1 + lib/tasks/repo.rake | 15 +++++++++++++++ 3 files changed, 18 insertions(+) create mode 100644 app/javascript/mastodon/locales/ga.json (limited to 'lib') diff --git a/.circleci/config.yml b/.circleci/config.yml index 8ab08ebae..ff8eb4859 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -173,9 +173,11 @@ jobs: <<: *defaults steps: - *attach_workspace + - *install_system_dependencies - run: bundle exec i18n-tasks check-normalized - run: bundle exec i18n-tasks unused -l en - run: bundle exec i18n-tasks check-consistent-interpolations + - run: bundle exec rake repo:check_locales_files workflows: version: 2 diff --git a/app/javascript/mastodon/locales/ga.json b/app/javascript/mastodon/locales/ga.json new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/app/javascript/mastodon/locales/ga.json @@ -0,0 +1 @@ +{} diff --git a/lib/tasks/repo.rake b/lib/tasks/repo.rake index 8ceec3085..d1de17b7c 100644 --- a/lib/tasks/repo.rake +++ b/lib/tasks/repo.rake @@ -76,4 +76,19 @@ namespace :repo do tmp.unlink end end + + task check_locales_files: :environment do + pastel = Pastel.new + + missing_yaml_files = I18n.available_locales.reject { |locale| File.exist?(Rails.root.join('config', 'locales', "#{locale}.yml")) } + missing_json_files = I18n.available_locales.reject { |locale| File.exist?(Rails.root.join('app', 'javascript', 'mastodon', 'locales', "#{locale}.json")) } + + if missing_json_files.empty? && missing_yaml_files.empty? + puts pastel.green('OK') + else + puts pastel.red("Missing YAML files: #{pastel.bold(missing_yaml_files.join(', '))}") unless missing_yaml_files.empty? + puts pastel.red("Missing JSON files: #{pastel.bold(missing_json_files.join(', '))}") unless missing_json_files.empty? + exit(1) + end + end end -- cgit From 3c8372fa81c77bb470be150f1fb56136317e4cfe Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Fri, 20 Sep 2019 22:59:29 +0200 Subject: Bump version to 2.9.3 (#11899) --- CHANGELOG.md | 59 +++++++++++++++++++++++++++++++++++++++++++++++++ lib/mastodon/version.rb | 2 +- 2 files changed, 60 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/CHANGELOG.md b/CHANGELOG.md index 539fec531..a17fbf8f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,65 @@ Changelog All notable changes to this project will be documented in this file. +## [2.9.3] - 2019-08-10 +### Added + +- Add GIF and WebP support for custom emojis ([Gargron](https://github.com/tootsuite/mastodon/pull/11519)) +- Add logout link to dropdown menu in web UI ([koyuawsmbrtn](https://github.com/tootsuite/mastodon/pull/11353)) +- Add indication that text search is unavailable in web UI ([ThibG](https://github.com/tootsuite/mastodon/pull/11112), [ThibG](https://github.com/tootsuite/mastodon/pull/11202)) +- Add `suffix` to `Mastodon::Version` to help forks ([clarfon](https://github.com/tootsuite/mastodon/pull/11407)) +- Add on-hover animation to animated custom emoji in web UI ([ThibG](https://github.com/tootsuite/mastodon/pull/11348), [ThibG](https://github.com/tootsuite/mastodon/pull/11404), [ThibG](https://github.com/tootsuite/mastodon/pull/11522)) +- Add custom emoji support in profile metadata labels ([ThibG](https://github.com/tootsuite/mastodon/pull/11350)) + +### Changed + +- Change default interface of web and streaming from 0.0.0.0 to 127.0.0.1 ([Gargron](https://github.com/tootsuite/mastodon/pull/11302), [zunda](https://github.com/tootsuite/mastodon/pull/11378), [Gargron](https://github.com/tootsuite/mastodon/pull/11351), [zunda](https://github.com/tootsuite/mastodon/pull/11326)) +- Change the retry limit of web push notifications ([highemerly](https://github.com/tootsuite/mastodon/pull/11292)) +- Change ActivityPub deliveries to not retry HTTP 501 errors ([Gargron](https://github.com/tootsuite/mastodon/pull/11233)) +- Change language detection to include hashtags as words ([Gargron](https://github.com/tootsuite/mastodon/pull/11341)) +- Change terms and privacy policy pages to always be accessible ([Gargron](https://github.com/tootsuite/mastodon/pull/11334)) +- Change robots tag to include `noarchive` when user opts out of indexing ([Kjwon15](https://github.com/tootsuite/mastodon/pull/11421)) + +### Fixed + +- Fix account domain block not clearing out notifications ([Gargron](https://github.com/tootsuite/mastodon/pull/11393)) +- Fix incorrect locale sometimes being detected for browser ([Gargron](https://github.com/tootsuite/mastodon/pull/8657)) +- Fix crash when saving invalid domain name ([Gargron](https://github.com/tootsuite/mastodon/pull/11528)) +- Fix pinned statuses REST API returning pagination headers ([Gargron](https://github.com/tootsuite/mastodon/pull/11526)) +- Fix "cancel follow request" button having unreadable text in web UI ([Gargron](https://github.com/tootsuite/mastodon/pull/11521)) +- Fix image uploads being blank when canvas read access is blocked ([ThibG](https://github.com/tootsuite/mastodon/pull/11499)) +- Fix avatars not being animated on hover when not logged in ([ThibG](https://github.com/tootsuite/mastodon/pull/11349)) +- Fix overzealous sanitization of HTML lists ([ThibG](https://github.com/tootsuite/mastodon/pull/11354)) +- Fix block crashing when a follow request exists ([ThibG](https://github.com/tootsuite/mastodon/pull/11288)) +- Fix backup service crashing when an attachment is missing ([ThibG](https://github.com/tootsuite/mastodon/pull/11241)) +- Fix account moderation action always sending e-mail notification ([Gargron](https://github.com/tootsuite/mastodon/pull/11242)) +- Fix swiping columns on mobile sometimes failing in web UI ([ThibG](https://github.com/tootsuite/mastodon/pull/11200)) +- Fix wrong actor URI being serialized into poll updates ([ThibG](https://github.com/tootsuite/mastodon/pull/11194)) +- Fix statsd UDP sockets not being cleaned up in Sidekiq ([Gargron](https://github.com/tootsuite/mastodon/pull/11230)) +- Fix expiration date of filters being set to "never" when editing them ([ThibG](https://github.com/tootsuite/mastodon/pull/11204)) +- Fix support for MP4 files that are actually M4V files ([Gargron](https://github.com/tootsuite/mastodon/pull/11210)) +- Fix `alerts` not being typecast correctly in push subscription in REST API ([Gargron](https://github.com/tootsuite/mastodon/pull/11343)) +- Fix some notices staying on unrelated pages ([ThibG](https://github.com/tootsuite/mastodon/pull/11364)) +- Fix unboosting sometimes preventing a boost from reappearing on feed ([ThibG](https://github.com/tootsuite/mastodon/pull/11405), [Gargron](https://github.com/tootsuite/mastodon/pull/11450)) +- Fix only one middle dot being recognized in hashtags ([Gargron](https://github.com/tootsuite/mastodon/pull/11345), [ThibG](https://github.com/tootsuite/mastodon/pull/11363)) +- Fix unnecessary SQL query performed on unauthenticated requests ([Gargron](https://github.com/tootsuite/mastodon/pull/11179)) +- Fix incorrect timestamp displayed on featured tags ([Kjwon15](https://github.com/tootsuite/mastodon/pull/11477)) +- Fix privacy dropdown active state when dropdown is placed on top of it ([ThibG](https://github.com/tootsuite/mastodon/pull/11495)) +- Fix filters not being applied to poll options ([ThibG](https://github.com/tootsuite/mastodon/pull/11174)) +- Fix keyboard navigation on various dropdowns ([ThibG](https://github.com/tootsuite/mastodon/pull/11511), [ThibG](https://github.com/tootsuite/mastodon/pull/11492), [ThibG](https://github.com/tootsuite/mastodon/pull/11491)) +- Fix keyboard navigation in modals ([ThibG](https://github.com/tootsuite/mastodon/pull/11493)) +- Fix image conversation being non-deterministic due to timestamps ([Gargron](https://github.com/tootsuite/mastodon/pull/11408)) +- Fix web UI performance ([ThibG](https://github.com/tootsuite/mastodon/pull/11211), [ThibG](https://github.com/tootsuite/mastodon/pull/11234)) +- Fix scrolling to compose form when not necessary in web UI ([ThibG](https://github.com/tootsuite/mastodon/pull/11246), [ThibG](https://github.com/tootsuite/mastodon/pull/11182)) +- Fix save button being enabled when list title is empty in web UI ([ThibG](https://github.com/tootsuite/mastodon/pull/11475)) +- Fix poll expiration not being pre-filled on delete & redraft in web UI ([ThibG](https://github.com/tootsuite/mastodon/pull/11203)) +- Fix content warning sometimes being set when not requested in web UI ([ThibG](https://github.com/tootsuite/mastodon/pull/11206)) + +### Security + +- Fix invites not being disabled upon account suspension ([ThibG](https://github.com/tootsuite/mastodon/pull/11412)) +- Fix blocked domains still being able to fill database with account records ([Gargron](https://github.com/tootsuite/mastodon/pull/11219)) + ## [2.9.2] - 2019-06-22 ### Added diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb index 3db57ceaa..99d709c98 100644 --- a/lib/mastodon/version.rb +++ b/lib/mastodon/version.rb @@ -13,7 +13,7 @@ module Mastodon end def patch - 2 + 3 end def flags -- cgit From e87bcaa10761b73d22a6b826e8aa7eebf316bad3 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sat, 21 Sep 2019 20:02:14 +0200 Subject: Bump version to 3.0.0rc1 (#11900) --- CHANGELOG.md | 4 ++++ lib/mastodon/version.rb | 8 ++++---- 2 files changed, 8 insertions(+), 4 deletions(-) (limited to 'lib') diff --git a/CHANGELOG.md b/CHANGELOG.md index a17fbf8f0..d8f7c77d3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,10 @@ Changelog All notable changes to this project will be documented in this file. +## Unreleased + +TODO + ## [2.9.3] - 2019-08-10 ### Added diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb index 99d709c98..bd49f0a17 100644 --- a/lib/mastodon/version.rb +++ b/lib/mastodon/version.rb @@ -5,19 +5,19 @@ module Mastodon module_function def major - 2 + 3 end def minor - 9 + 0 end def patch - 3 + 0 end def flags - '' + 'rc1' end def suffix -- cgit From 172eaeba3fd217228dead279712aecce8c3ac080 Mon Sep 17 00:00:00 2001 From: Yamagishi Kazutoshi Date: Mon, 23 Sep 2019 22:37:45 +0900 Subject: Add config of multipart threshold for S3 (#11924) --- .env.production.sample | 14 ++++++++++++++ config/initializers/paperclip.rb | 1 + lib/tasks/mastodon.rake | 30 +++++++++++++++++++++++++++++- 3 files changed, 44 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/.env.production.sample b/.env.production.sample index b322aee1d..e4ea861e7 100644 --- a/.env.production.sample +++ b/.env.production.sample @@ -115,6 +115,20 @@ SMTP_FROM_ADDRESS=notifications@example.com # S3_ENDPOINT= # S3_SIGNATURE_VERSION= +# Google Cloud Storage (optional) +# Use S3 compatible API. Since GCS does not support Multipart Upload, +# increase the value of S3_MULTIPART_THRESHOLD to disable Multipart Upload. +# The attachment host must allow cross origin request - see the description +# above. +# S3_ENABLED=true +# AWS_ACCESS_KEY_ID= +# AWS_SECRET_ACCESS_KEY= +# S3_REGION= +# S3_PROTOCOL=https +# S3_HOSTNAME=storage.googleapis.com +# S3_ENDPOINT=https://storage.googleapis.com +# S3_MULTIPART_THRESHOLD=52428801 # 50.megabytes + # Swift (optional) # The attachment host must allow cross origin request - see the description # above. diff --git a/config/initializers/paperclip.rb b/config/initializers/paperclip.rb index ce4185e02..cfc95330c 100644 --- a/config/initializers/paperclip.rb +++ b/config/initializers/paperclip.rb @@ -25,6 +25,7 @@ if ENV['S3_ENABLED'] == 'true' s3_protocol: s3_protocol, s3_host_name: s3_hostname, s3_headers: { + 'X-Amz-Multipart-Threshold' => ENV.fetch('S3_MULTIPART_THRESHOLD') { 15.megabytes }, 'Cache-Control' => 'public, max-age=315576000, immutable', }, s3_permissions: ENV.fetch('S3_PERMISSION') { 'public-read' }, diff --git a/lib/tasks/mastodon.rake b/lib/tasks/mastodon.rake index ee9657b0e..2e92e8ded 100644 --- a/lib/tasks/mastodon.rake +++ b/lib/tasks/mastodon.rake @@ -135,7 +135,7 @@ namespace :mastodon do prompt.say "\n" if prompt.yes?('Do you want to store uploaded files on the cloud?', default: false) - case prompt.select('Provider', ['Amazon S3', 'Wasabi', 'Minio']) + case prompt.select('Provider', ['Amazon S3', 'Wasabi', 'Minio', 'Google Cloud Storage']) when 'Amazon S3' env['S3_ENABLED'] = 'true' env['S3_PROTOCOL'] = 'https' @@ -217,6 +217,34 @@ namespace :mastodon do q.required true q.modify :strip end + when 'Google Cloud Storage' + env['S3_ENABLED'] = 'true' + env['S3_PROTOCOL'] = 'https' + env['S3_HOSTNAME'] = 'storage.googleapis.com' + env['S3_ENDPOINT'] = 'https://storage.googleapis.com' + env['S3_MULTIPART_THRESHOLD'] = 50.megabytes + + env['S3_BUCKET'] = prompt.ask('GCS bucket name:') do |q| + q.required true + q.default "files.#{env['LOCAL_DOMAIN']}" + q.modify :strip + end + + env['S3_REGION'] = prompt.ask('GCS region:') do |q| + q.required true + q.default 'us-west1' + q.modify :strip + end + + env['AWS_ACCESS_KEY_ID'] = prompt.ask('GCS access key:') do |q| + q.required true + q.modify :strip + end + + env['AWS_SECRET_ACCESS_KEY'] = prompt.ask('GCS secret key:') do |q| + q.required true + q.modify :strip + end end if prompt.yes?('Do you want to access the uploaded files from your own domain?') -- cgit From a1f04c1e3497e9dff5970038461d9f454f2650df Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 24 Sep 2019 04:35:36 +0200 Subject: Fix authentication before 2FA challenge (#11943) Regression from #11831 --- app/controllers/auth/sessions_controller.rb | 61 +++++++++++++++------------ app/models/concerns/ldap_authenticable.rb | 44 +++++++++++++++---- config/application.rb | 3 +- config/initializers/devise.rb | 11 ++--- lib/devise/ldap_authenticatable.rb | 55 ------------------------ lib/devise/two_factor_ldap_authenticatable.rb | 32 ++++++++++++++ lib/devise/two_factor_pam_authenticatable.rb | 31 ++++++++++++++ 7 files changed, 139 insertions(+), 98 deletions(-) delete mode 100644 lib/devise/ldap_authenticatable.rb create mode 100644 lib/devise/two_factor_ldap_authenticatable.rb create mode 100644 lib/devise/two_factor_pam_authenticatable.rb (limited to 'lib') diff --git a/app/controllers/auth/sessions_controller.rb b/app/controllers/auth/sessions_controller.rb index b3113bbef..f48b17c79 100644 --- a/app/controllers/auth/sessions_controller.rb +++ b/app/controllers/auth/sessions_controller.rb @@ -8,6 +8,8 @@ class Auth::SessionsController < Devise::SessionsController skip_before_action :require_no_authentication, only: [:create] skip_before_action :require_functional! + prepend_before_action :authenticate_with_two_factor, if: :two_factor_enabled?, only: [:create] + before_action :set_instance_presenter, only: [:new] before_action :set_body_classes @@ -20,22 +22,9 @@ class Auth::SessionsController < Devise::SessionsController end def create - self.resource = begin - if user_params[:email].blank? && session[:otp_user_id].present? - User.find(session[:otp_user_id]) - else - warden.authenticate!(auth_options) - end - end - - if resource.otp_required_for_login? - if user_params[:otp_attempt].present? && session[:otp_user_id].present? - authenticate_with_two_factor_via_otp(resource) - else - prompt_for_two_factor(resource) - end - else - authenticate_and_respond(resource) + super do |resource| + remember_me(resource) + flash.delete(:notice) end end @@ -49,6 +38,16 @@ class Auth::SessionsController < Devise::SessionsController protected + def find_user + if session[:otp_user_id] + User.find(session[:otp_user_id]) + else + user = User.authenticate_with_ldap(user_params) if Devise.ldap_authentication + user ||= User.authenticate_with_pam(user_params) if Devise.pam_authentication + user ||= User.find_for_authentication(email: user_params[:email]) + end + end + def user_params params.require(:user).permit(:email, :password, :otp_attempt) end @@ -71,6 +70,10 @@ class Auth::SessionsController < Devise::SessionsController super end + def two_factor_enabled? + find_user&.otp_required_for_login? + end + def valid_otp_attempt?(user) user.validate_and_consume_otp!(user_params[:otp_attempt]) || user.invalidate_otp_backup_code!(user_params[:otp_attempt]) @@ -78,10 +81,24 @@ class Auth::SessionsController < Devise::SessionsController false end + def authenticate_with_two_factor + user = self.resource = find_user + + if user_params[:otp_attempt].present? && session[:otp_user_id] + authenticate_with_two_factor_via_otp(user) + elsif user.present? && (user.encrypted_password.blank? || user.valid_password?(user_params[:password])) + # If encrypted_password is blank, we got the user from LDAP or PAM, + # so credentials are already valid + + prompt_for_two_factor(user) + end + end + def authenticate_with_two_factor_via_otp(user) if valid_otp_attempt?(user) session.delete(:otp_user_id) - authenticate_and_respond(user) + remember_me(user) + sign_in(user) else flash.now[:alert] = I18n.t('users.invalid_otp_token') prompt_for_two_factor(user) @@ -90,16 +107,10 @@ class Auth::SessionsController < Devise::SessionsController def prompt_for_two_factor(user) session[:otp_user_id] = user.id + @body_classes = 'lighter' render :two_factor end - def authenticate_and_respond(user) - sign_in(user) - remember_me(user) - - respond_with user, location: after_sign_in_path_for(user) - end - private def set_instance_presenter @@ -112,11 +123,9 @@ class Auth::SessionsController < Devise::SessionsController def home_paths(resource) paths = [about_path] - if single_user_mode? && resource.is_a?(User) paths << short_account_path(username: resource.account) end - paths end diff --git a/app/models/concerns/ldap_authenticable.rb b/app/models/concerns/ldap_authenticable.rb index 84ff84c4b..117993947 100644 --- a/app/models/concerns/ldap_authenticable.rb +++ b/app/models/concerns/ldap_authenticable.rb @@ -3,24 +3,50 @@ module LdapAuthenticable extend ActiveSupport::Concern - def ldap_setup(_attributes) - self.confirmed_at = Time.now.utc - self.admin = false - self.external = true + class_methods do + def authenticate_with_ldap(params = {}) + ldap = Net::LDAP.new(ldap_options) + filter = format(Devise.ldap_search_filter, uid: Devise.ldap_uid, email: params[:email]) - save! - end + if (user_info = ldap.bind_as(base: Devise.ldap_base, filter: filter, password: params[:password])) + ldap_get_user(user_info.first) + end + end - class_methods do def ldap_get_user(attributes = {}) resource = joins(:account).find_by(accounts: { username: attributes[Devise.ldap_uid.to_sym].first }) if resource.blank? - resource = new(email: attributes[:mail].first, agreement: true, account_attributes: { username: attributes[Devise.ldap_uid.to_sym].first }) - resource.ldap_setup(attributes) + resource = new(email: attributes[:mail].first, agreement: true, account_attributes: { username: attributes[Devise.ldap_uid.to_sym].first }, admin: false, external: true, confirmed_at: Time.now.utc) + resource.save! end resource end + + def ldap_options + opts = { + host: Devise.ldap_host, + port: Devise.ldap_port, + base: Devise.ldap_base, + + auth: { + method: :simple, + username: Devise.ldap_bind_dn, + password: Devise.ldap_password, + }, + + connect_timeout: 10, + } + + if [:simple_tls, :start_tls].include?(Devise.ldap_method) + opts[:encryption] = { + method: Devise.ldap_method, + tls_options: OpenSSL::SSL::SSLContext::DEFAULT_PARAMS.tap { |options| options[:verify_mode] = OpenSSL::SSL::VERIFY_NONE if Devise.ldap_tls_no_verify }, + } + end + + opts + end end end diff --git a/config/application.rb b/config/application.rb index 5fd37120d..3ced81b8f 100644 --- a/config/application.rb +++ b/config/application.rb @@ -13,7 +13,8 @@ require_relative '../lib/paperclip/video_transcoder' require_relative '../lib/paperclip/type_corrector' require_relative '../lib/mastodon/snowflake' require_relative '../lib/mastodon/version' -require_relative '../lib/devise/ldap_authenticatable' +require_relative '../lib/devise/two_factor_ldap_authenticatable' +require_relative '../lib/devise/two_factor_pam_authenticatable' Dotenv::Railtie.load diff --git a/config/initializers/devise.rb b/config/initializers/devise.rb index 311583820..fd9a5a8b9 100644 --- a/config/initializers/devise.rb +++ b/config/initializers/devise.rb @@ -71,13 +71,10 @@ end Devise.setup do |config| config.warden do |manager| - manager.default_strategies(scope: :user).unshift :database_authenticatable - manager.default_strategies(scope: :user).unshift :ldap_authenticatable if Devise.ldap_authentication - manager.default_strategies(scope: :user).unshift :pam_authenticatable if Devise.pam_authentication - - # We handle 2FA in our own sessions controller so this gets in the way - manager.default_strategies(scope: :user).delete :two_factor_backupable - manager.default_strategies(scope: :user).delete :two_factor_authenticatable + manager.default_strategies(scope: :user).unshift :two_factor_ldap_authenticatable if Devise.ldap_authentication + manager.default_strategies(scope: :user).unshift :two_factor_pam_authenticatable if Devise.pam_authentication + manager.default_strategies(scope: :user).unshift :two_factor_authenticatable + manager.default_strategies(scope: :user).unshift :two_factor_backupable end # The secret key used by Devise. Devise uses this key to generate diff --git a/lib/devise/ldap_authenticatable.rb b/lib/devise/ldap_authenticatable.rb deleted file mode 100644 index 6903d468d..000000000 --- a/lib/devise/ldap_authenticatable.rb +++ /dev/null @@ -1,55 +0,0 @@ -# frozen_string_literal: true - -require 'net/ldap' -require 'devise/strategies/authenticatable' - -module Devise - module Strategies - class LdapAuthenticatable < Authenticatable - def authenticate! - if params[:user] - ldap = Net::LDAP.new( - host: Devise.ldap_host, - port: Devise.ldap_port, - base: Devise.ldap_base, - encryption: { - method: Devise.ldap_method, - tls_options: tls_options, - }, - auth: { - method: :simple, - username: Devise.ldap_bind_dn, - password: Devise.ldap_password, - }, - connect_timeout: 10 - ) - - filter = format(Devise.ldap_search_filter, uid: Devise.ldap_uid, email: email) - - if (user_info = ldap.bind_as(base: Devise.ldap_base, filter: filter, password: password)) - user = User.ldap_get_user(user_info.first) - success!(user) - else - return fail(:invalid) - end - end - end - - def email - params[:user][:email] - end - - def password - params[:user][:password] - end - - def tls_options - OpenSSL::SSL::SSLContext::DEFAULT_PARAMS.tap do |options| - options[:verify_mode] = OpenSSL::SSL::VERIFY_NONE if Devise.ldap_tls_no_verify - end - end - end - end -end - -Warden::Strategies.add(:ldap_authenticatable, Devise::Strategies::LdapAuthenticatable) diff --git a/lib/devise/two_factor_ldap_authenticatable.rb b/lib/devise/two_factor_ldap_authenticatable.rb new file mode 100644 index 000000000..065aa2de8 --- /dev/null +++ b/lib/devise/two_factor_ldap_authenticatable.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +require 'net/ldap' +require 'devise/strategies/base' + +module Devise + module Strategies + class TwoFactorLdapAuthenticatable < Base + def valid? + valid_params? && mapping.to.respond_to?(:authenticate_with_ldap) + end + + def authenticate! + resource = mapping.to.authenticate_with_ldap(params[scope]) + + if resource && !resource.otp_required_for_login? + success!(resource) + else + fail(:invalid) + end + end + + protected + + def valid_params? + params[scope] && params[scope][:password].present? + end + end + end +end + +Warden::Strategies.add(:two_factor_ldap_authenticatable, Devise::Strategies::TwoFactorLdapAuthenticatable) diff --git a/lib/devise/two_factor_pam_authenticatable.rb b/lib/devise/two_factor_pam_authenticatable.rb new file mode 100644 index 000000000..5ce723b33 --- /dev/null +++ b/lib/devise/two_factor_pam_authenticatable.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +require 'devise/strategies/base' + +module Devise + module Strategies + class TwoFactorPamAuthenticatable < Base + def valid? + valid_params? && mapping.to.respond_to?(:authenticate_with_pam) + end + + def authenticate! + resource = mapping.to.authenticate_with_pam(params[scope]) + + if resource && !resource.otp_required_for_login? + success!(resource) + else + fail(:invalid) + end + end + + protected + + def valid_params? + params[scope] && params[scope][:password].present? + end + end + end +end + +Warden::Strategies.add(:two_factor_pam_authenticatable, Devise::Strategies::TwoFactorPamAuthenticatable) -- cgit From 13b06d4b3b705deb90d063f4903737b5609dfbc7 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sun, 29 Sep 2019 18:50:16 +0200 Subject: Bump version to 3.0.0rc2 (#11999) --- CHANGELOG.md | 4 ++++ lib/mastodon/version.rb | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/CHANGELOG.md b/CHANGELOG.md index a170c3ecd..4e9ccdc8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,6 +50,7 @@ All notable changes to this project will be documented in this file. - **Add search syntax for operators and phrases** ([Gargron](https://github.com/tootsuite/mastodon/pull/11411)) - **Add REST API for managing featured hashtags** ([noellabo](https://github.com/tootsuite/mastodon/pull/11778)) - **Add REST API for managing timeline read markers** ([Gargron](https://github.com/tootsuite/mastodon/pull/11762)) +- Add `exclude_unreviewed` param to `GET /api/v2/search` REST API ([Gargron](https://github.com/tootsuite/mastodon/pull/11977)) - **Add ActivityPub secure mode** ([Gargron](https://github.com/tootsuite/mastodon/pull/11269), [ThibG](https://github.com/tootsuite/mastodon/pull/11332), [ThibG](https://github.com/tootsuite/mastodon/pull/11295)) - Add HTTP signatures to all outgoing ActivityPub GET requests ([Gargron](https://github.com/tootsuite/mastodon/pull/11284), [ThibG](https://github.com/tootsuite/mastodon/pull/11300)) - Add support for ActivityPub Audio activities ([ThibG](https://github.com/tootsuite/mastodon/pull/11189)) @@ -98,6 +99,7 @@ All notable changes to this project will be documented in this file. - Change Dockerfile ([Shleeble](https://github.com/tootsuite/mastodon/pull/11710), [ykzts](https://github.com/tootsuite/mastodon/pull/11768), [Shleeble](https://github.com/tootsuite/mastodon/pull/11707)) - Change supported Node versions to include v12 ([abcang](https://github.com/tootsuite/mastodon/pull/11706)) - Change Portuguese language from `pt` to `pt-PT` ([Gargron](https://github.com/tootsuite/mastodon/pull/11820)) +- Change domain block silence to always require approval on follow ([ThibG](https://github.com/tootsuite/mastodon/pull/11975)) ### Removed @@ -172,6 +174,8 @@ All notable changes to this project will be documented in this file. - Fix URLs counting towards RTL detection ([ahangarha](https://github.com/tootsuite/mastodon/pull/11759)) - Fix unnecessary status re-rendering in web UI ([ThibG](https://github.com/tootsuite/mastodon/pull/11211)) - Fix http_parser.rb gem not being compiled when no network available ([petabyteboy](https://github.com/tootsuite/mastodon/pull/11444)) +- Fix muted text color not applying to all text ([trwnh](https://github.com/tootsuite/mastodon/pull/11996)) +- Fix follower/following lists resetting on back-navigation in web UI ([Gargron](https://github.com/tootsuite/mastodon/pull/11986)) ## [2.9.3] - 2019-08-10 ### Added diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb index bd49f0a17..9c5686ed2 100644 --- a/lib/mastodon/version.rb +++ b/lib/mastodon/version.rb @@ -17,7 +17,7 @@ module Mastodon end def flags - 'rc1' + 'rc2' end def suffix -- cgit From 5c42f47617d311219d06e082e4daa41e671903c8 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 1 Oct 2019 01:19:11 +0200 Subject: Fix records not being indexed sometimes (#12024) It's possible that after commit callbacks were not firing when exceptions occurred in the process. Also, the default Sidekiq strategy does not push indexing jobs immediately, which is not necessary and could be part of the issue too. --- app/models/account.rb | 2 +- app/models/account_stat.rb | 2 +- app/models/application_record.rb | 6 ++++++ app/models/favourite.rb | 2 +- app/models/status.rb | 2 +- app/models/tag.rb | 2 +- config/application.rb | 1 + config/initializers/chewy.rb | 5 +++-- lib/chewy/strategy/custom_sidekiq.rb | 30 ++++++++++++++++++++++++++++++ spec/rails_helper.rb | 2 +- 10 files changed, 46 insertions(+), 8 deletions(-) create mode 100644 lib/chewy/strategy/custom_sidekiq.rb (limited to 'lib') diff --git a/app/models/account.rb b/app/models/account.rb index 55fe53fae..01d45e36c 100644 --- a/app/models/account.rb +++ b/app/models/account.rb @@ -129,7 +129,7 @@ class Account < ApplicationRecord delegate :chosen_languages, to: :user, prefix: false, allow_nil: true - update_index('accounts#account', :self) if Chewy.enabled? + update_index('accounts#account', :self) def local? domain.nil? diff --git a/app/models/account_stat.rb b/app/models/account_stat.rb index 6d1097cec..1351f7d8a 100644 --- a/app/models/account_stat.rb +++ b/app/models/account_stat.rb @@ -16,7 +16,7 @@ class AccountStat < ApplicationRecord belongs_to :account, inverse_of: :account_stat - update_index('accounts#account', :account) if Chewy.enabled? + update_index('accounts#account', :account) def increment_count!(key) update(attributes_for_increment(key)) diff --git a/app/models/application_record.rb b/app/models/application_record.rb index c1b873da6..5d7d3a096 100644 --- a/app/models/application_record.rb +++ b/app/models/application_record.rb @@ -5,6 +5,12 @@ class ApplicationRecord < ActiveRecord::Base include Remotable + class << self + def update_index(_type_name, *_args, &_block) + super if Chewy.enabled? + end + end + def boolean_with_default(key, default_value) value = attributes[key] diff --git a/app/models/favourite.rb b/app/models/favourite.rb index 17f8c9fa6..bf0ec4449 100644 --- a/app/models/favourite.rb +++ b/app/models/favourite.rb @@ -13,7 +13,7 @@ class Favourite < ApplicationRecord include Paginable - update_index('statuses#status', :status) if Chewy.enabled? + update_index('statuses#status', :status) belongs_to :account, inverse_of: :favourites belongs_to :status, inverse_of: :favourites diff --git a/app/models/status.rb b/app/models/status.rb index 5e7474577..078a64566 100644 --- a/app/models/status.rb +++ b/app/models/status.rb @@ -39,7 +39,7 @@ class Status < ApplicationRecord # will be based on current time instead of `created_at` attr_accessor :override_timestamps - update_index('statuses#status', :proper) if Chewy.enabled? + update_index('statuses#status', :proper) enum visibility: [:public, :unlisted, :private, :direct, :limited], _suffix: :visibility diff --git a/app/models/tag.rb b/app/models/tag.rb index 9aca3983f..82786daa8 100644 --- a/app/models/tag.rb +++ b/app/models/tag.rb @@ -49,7 +49,7 @@ class Tag < ApplicationRecord after_save :save_account_tag_stat - update_index('tags#tag', :self) if Chewy.enabled? + update_index('tags#tag', :self) def account_tag_stat super || build_account_tag_stat diff --git a/config/application.rb b/config/application.rb index 3ced81b8f..60f73f8bb 100644 --- a/config/application.rb +++ b/config/application.rb @@ -15,6 +15,7 @@ require_relative '../lib/mastodon/snowflake' require_relative '../lib/mastodon/version' require_relative '../lib/devise/two_factor_ldap_authenticatable' require_relative '../lib/devise/two_factor_pam_authenticatable' +require_relative '../lib/chewy/strategy/custom_sidekiq' Dotenv::Railtie.load diff --git a/config/initializers/chewy.rb b/config/initializers/chewy.rb index d5347f2bf..9ff0dccc1 100644 --- a/config/initializers/chewy.rb +++ b/config/initializers/chewy.rb @@ -12,8 +12,9 @@ Chewy.settings = { sidekiq: { queue: 'pull' }, } -Chewy.root_strategy = enabled ? :sidekiq : :bypass -Chewy.request_strategy = enabled ? :sidekiq : :bypass +Chewy.root_strategy = :custom_sidekiq +Chewy.request_strategy = :custom_sidekiq +Chewy.use_after_commit_callbacks = false module Chewy class << self diff --git a/lib/chewy/strategy/custom_sidekiq.rb b/lib/chewy/strategy/custom_sidekiq.rb new file mode 100644 index 000000000..3e54326ba --- /dev/null +++ b/lib/chewy/strategy/custom_sidekiq.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Chewy + class Strategy + class CustomSidekiq < Base + class Worker + include ::Sidekiq::Worker + + sidekiq_options queue: 'pull' + + def perform(type, ids, options = {}) + options[:refresh] = !Chewy.disable_refresh_async if Chewy.disable_refresh_async + type.constantize.import!(ids, options) + end + end + + def update(type, objects, _options = {}) + return unless Chewy.enabled? + + ids = type.root.id ? Array.wrap(objects) : type.adapter.identify(objects) + + return if ids.empty? + + Worker.perform_async(type.name, ids) + end + + def leave; end + end + end +end diff --git a/spec/rails_helper.rb b/spec/rails_helper.rb index 3a5e7491e..6fbceca53 100644 --- a/spec/rails_helper.rb +++ b/spec/rails_helper.rb @@ -12,7 +12,7 @@ require 'capybara/rspec' Dir[Rails.root.join('spec/support/**/*.rb')].each { |f| require f } ActiveRecord::Migration.maintain_test_schema! -WebMock.disable_net_connect! +WebMock.disable_net_connect!(allow: Chewy.settings[:host]) Redis.current = Redis::Namespace.new("mastodon_test#{ENV['TEST_ENV_NUMBER']}", redis: Redis.current) Sidekiq::Testing.inline! Sidekiq::Logging.logger = nil -- cgit From c7adf8082726ed3059de1c09bcc7b1cde5ca2909 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Wed, 2 Oct 2019 04:49:53 +0200 Subject: Change `tootctl domains purge` to accept multiple domains at once (#12046) --- lib/mastodon/domains_cli.rb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'lib') diff --git a/lib/mastodon/domains_cli.rb b/lib/mastodon/domains_cli.rb index 8e52de1c3..b5435bb5e 100644 --- a/lib/mastodon/domains_cli.rb +++ b/lib/mastodon/domains_cli.rb @@ -17,7 +17,7 @@ module Mastodon option :verbose, type: :boolean, aliases: [:v] option :dry_run, type: :boolean option :whitelist_mode, type: :boolean - desc 'purge [DOMAIN]', 'Remove accounts from a DOMAIN without a trace' + desc 'purge [DOMAIN...]', 'Remove accounts from a DOMAIN without a trace' long_desc <<-LONG_DESC Remove all accounts from a given DOMAIN without leaving behind any records. Unlike a suspension, if the DOMAIN still exists in the wild, @@ -27,16 +27,16 @@ module Mastodon from a single domain, all accounts from domains that are not whitelisted are removed from the database. LONG_DESC - def purge(domain = nil) + def purge(*domains) dry_run = options[:dry_run] ? ' (DRY RUN)' : '' scope = begin if options[:whitelist_mode] Account.remote.where.not(domain: DomainAllow.pluck(:domain)) - elsif domain.present? - Account.remote.where(domain: domain) + elsif !domains.empty? + Account.remote.where(domain: domains) else - say('No domain given', :red) + say('No domain(s) given', :red) exit(1) end end @@ -45,11 +45,11 @@ module Mastodon SuspendAccountService.new.call(account, reserve_username: false, skip_side_effects: true) unless options[:dry_run] end - DomainBlock.where(domain: domain).destroy_all unless options[:dry_run] + DomainBlock.where(domain: domains).destroy_all unless options[:dry_run] say("Removed #{processed} accounts#{dry_run}", :green) - custom_emojis = CustomEmoji.where(domain: domain) + custom_emojis = CustomEmoji.where(domain: domains) custom_emojis_count = custom_emojis.count custom_emojis.destroy_all unless options[:dry_run] -- cgit From 0ce0baa9b5799be61fed7fc509837ca0ba971402 Mon Sep 17 00:00:00 2001 From: Takeshi Umeda Date: Thu, 3 Oct 2019 04:50:43 +0900 Subject: Add parallelization to `tootctl search deploy` (#12051) * Add parallel gem * Modify parallel option in tootctl search deploy * Add paralell option to tootctl search deploy * Change 1 to false * Clean up * Rename --parallel to --processes --- Gemfile | 1 + Gemfile.lock | 1 + lib/mastodon/search_cli.rb | 23 +++++++++++++++++++++-- 3 files changed, 23 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/Gemfile b/Gemfile index 7ed1a4e6b..f6ee3b52e 100644 --- a/Gemfile +++ b/Gemfile @@ -67,6 +67,7 @@ gem 'oj', '~> 3.9' gem 'ostatus2', '~> 2.0' gem 'ox', '~> 2.11' gem 'parslet' +gem 'parallel', '~> 1.17' gem 'posix-spawn', git: 'https://github.com/rtomayko/posix-spawn', ref: '58465d2e213991f8afb13b984854a49fcdcc980c' gem 'pundit', '~> 2.1' gem 'premailer-rails' diff --git a/Gemfile.lock b/Gemfile.lock index d708b34c1..b94647a20 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -748,6 +748,7 @@ DEPENDENCIES ox (~> 2.11) paperclip (~> 6.0) paperclip-av-transcoder (~> 0.6) + parallel (~> 1.17) parallel_tests (~> 2.29) parslet pg (~> 1.1) diff --git a/lib/mastodon/search_cli.rb b/lib/mastodon/search_cli.rb index 42ad93f1e..8bd5f9543 100644 --- a/lib/mastodon/search_cli.rb +++ b/lib/mastodon/search_cli.rb @@ -6,6 +6,7 @@ require_relative 'cli_helper' module Mastodon class SearchCLI < Thor + option :processes, default: 2, aliases: [:p] desc 'deploy', 'Create or update an ElasticSearch index and populate it' long_desc <<~LONG_DESC If ElasticSearch is empty, this command will create the necessary indices @@ -13,10 +14,28 @@ module Mastodon This command will also upgrade indices if the underlying schema has been changed since the last run. + + With the --processes option, parallelize execution of the command. The + default is 2. If "auto" is specified, the number is automatically + derived from available CPUs. LONG_DESC def deploy - processed = Chewy::RakeHelper.upgrade - Chewy::RakeHelper.sync(except: processed) + processed = Chewy::RakeHelper.upgrade(parallel: processes) + Chewy::RakeHelper.sync(except: processed, parallel: processes) + end + + private + + def processes + return true if options[:processes] == 'auto' + + num = options[:processes].to_i + + if num < 2 + nil + else + num + end end end end -- cgit From ca22a22d7f4db867ad0045a7978e3d8dcd251a69 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Thu, 3 Oct 2019 01:09:12 +0200 Subject: Fix performance of GIF re-encoding (#12057) * Change animated GIF detection to not shell out to ImageMagick Signed-off-by: Eugen Rochko * Change video encoding parameters to limit to 10800 video frames Signed-off-by: Eugen Rochko * Limit GIF image size further Signed-off-by: Eugen Rochko * Always strip metadata from video files * Fix code style issues --- app/models/concerns/attachmentable.rb | 4 +- app/models/media_attachment.rb | 33 ++++++++--- lib/paperclip/gif_transcoder.rb | 101 +++++++++++++++++++++++++++++++++- lib/paperclip/video_transcoder.rb | 2 + 4 files changed, 128 insertions(+), 12 deletions(-) (limited to 'lib') diff --git a/app/models/concerns/attachmentable.rb b/app/models/concerns/attachmentable.rb index 246c2c27c..3bbc6453c 100644 --- a/app/models/concerns/attachmentable.rb +++ b/app/models/concerns/attachmentable.rb @@ -6,6 +6,7 @@ module Attachmentable extend ActiveSupport::Concern MAX_MATRIX_LIMIT = 16_777_216 # 4096x4096px or approx. 16MB + GIF_MATRIX_LIMIT = 921_600 # 1280x720px included do before_post_process :set_file_extensions @@ -42,8 +43,9 @@ module Attachmentable next if attachment.blank? || !/image.*/.match?(attachment.content_type) || attachment.queued_for_write[:original].blank? width, height = FastImage.size(attachment.queued_for_write[:original].path) + matrix_limit = attachment.content_type == 'image/gif' ? GIF_MATRIX_LIMIT : MAX_MATRIX_LIMIT - raise Mastodon::DimensionsValidationError, "#{width}x#{height} images are not supported, must be below #{MAX_MATRIX_LIMIT} sqpx" if width.present? && height.present? && (width * height >= MAX_MATRIX_LIMIT) + raise Mastodon::DimensionsValidationError, "#{width}x#{height} images are not supported" if width.present? && height.present? && (width * height > matrix_limit) end end diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb index c4932f2ef..630dab55a 100644 --- a/app/models/media_attachment.rb +++ b/app/models/media_attachment.rb @@ -65,6 +65,17 @@ class MediaAttachment < ApplicationRecord file_geometry_parser: FastGeometryParser, blurhash: BLURHASH_OPTIONS, }, + + original: { + keep_same_format: true, + convert_options: { + output: { + 'map_metadata' => '-1', + 'c:v' => 'copy', + 'c:a' => 'copy', + }, + }, + }, }.freeze AUDIO_STYLES = { @@ -86,14 +97,15 @@ class MediaAttachment < ApplicationRecord output: { 'loglevel' => 'fatal', 'movflags' => 'faststart', - 'pix_fmt' => 'yuv420p', - 'vf' => 'scale=\'trunc(iw/2)*2:trunc(ih/2)*2\'', - 'vsync' => 'cfr', - 'c:v' => 'h264', - 'b:v' => '500K', - 'maxrate' => '1300K', - 'bufsize' => '1300K', - 'crf' => 18, + 'pix_fmt' => 'yuv420p', + 'vf' => 'scale=\'trunc(iw/2)*2:trunc(ih/2)*2\'', + 'vsync' => 'cfr', + 'c:v' => 'h264', + 'maxrate' => '1300K', + 'bufsize' => '1300K', + 'frames:v' => 60 * 60 * 3, + 'crf' => 18, + 'map_metadata' => '-1', }, }, }.freeze @@ -103,7 +115,7 @@ class MediaAttachment < ApplicationRecord original: VIDEO_FORMAT, }.freeze - IMAGE_LIMIT = 8.megabytes + IMAGE_LIMIT = 10.megabytes VIDEO_LIMIT = 40.megabytes belongs_to :account, inverse_of: :media_attachments, optional: true @@ -244,7 +256,9 @@ class MediaAttachment < ApplicationRecord def set_meta meta = populate_meta + return if meta == {} + file.instance_write :meta, meta end @@ -287,6 +301,7 @@ class MediaAttachment < ApplicationRecord def reset_parent_cache return if status_id.nil? + Rails.cache.delete("statuses/#{status_id}") end end diff --git a/lib/paperclip/gif_transcoder.rb b/lib/paperclip/gif_transcoder.rb index cbab6fd99..64f12f963 100644 --- a/lib/paperclip/gif_transcoder.rb +++ b/lib/paperclip/gif_transcoder.rb @@ -1,5 +1,103 @@ # frozen_string_literal: true +class GifReader + attr_reader :animated + + EXTENSION_LABELS = [0xf9, 0x01, 0xff].freeze + GIF_HEADERS = %w(GIF87a GIF89a).freeze + + class GifReaderException; end + + class UnknownImageType < GifReaderException; end + + class CannotParseImage < GifReaderException; end + + def self.animated?(path) + new(path).animated + rescue GifReaderException + false + end + + def initialize(path, max_frames = 2) + @path = path + @nb_frames = 0 + + File.open(path, 'rb') do |s| + raise UnknownImageType unless GIF_HEADERS.include?(s.read(6)) + + # Skip to "packed byte" + s.seek(4, IO::SEEK_CUR) + + # "Packed byte" gives us the size of the GIF color table + packed_byte, = s.read(1).unpack('C') + + # Skip background color and aspect ratio + s.seek(2, IO::SEEK_CUR) + + if packed_byte & 0x80 != 0 + # GIF uses a global color table, skip it + s.seek(3 * (1 << ((packed_byte & 0x07) + 1)), IO::SEEK_CUR) + end + + # Now read data + while @nb_frames < max_frames + separator = s.read(1) + + case separator + when ',' # Image block + @nb_frames += 1 + + # Skip to "packed byte" + s.seek(8, IO::SEEK_CUR) + packed_byte, = s.read(1).unpack('C') + + if packed_byte & 0x80 != 0 + # Image uses a local color table, skip it + s.seek(3 * (1 << ((packed_byte & 0x07) + 1)), IO::SEEK_CUR) + end + + # Skip lzw min code size + raise InvalidValue unless s.read(1).unpack('C')[0] >= 2 + + # Skip image data sub-blocks + skip_sub_blocks!(s) + when '!' # Extension block + skip_extension_block!(s) + when ';' # Trailer + break + else + raise CannotParseImage + end + end + end + + @animated = @nb_frames > 1 + end + + private + + def skip_extension_block!(file) + if EXTENSION_LABELS.include?(file.read(1).unpack('C')[0]) + block_size, = file.read(1).unpack('C') + file.seek(block_size, IO::SEEK_CUR) + end + + # Read until extension block end marker + skip_sub_blocks!(file) + end + + # Skip sub-blocks up until block end marker + def skip_sub_blocks!(file) + loop do + size, = file.read(1).unpack('C') + + break if size.zero? + + file.seek(size, IO::SEEK_CUR) + end + end +end + module Paperclip # This transcoder is only to be used for the MediaAttachment model # to convert animated gifs to webm @@ -19,8 +117,7 @@ module Paperclip private def needs_convert? - num_frames = identify('-format %n :file', file: file.path).to_i - options[:style] == :original && num_frames > 1 + options[:style] == :original && GifReader.animated?(file.path) end end end diff --git a/lib/paperclip/video_transcoder.rb b/lib/paperclip/video_transcoder.rb index c3504c17c..66f7feda5 100644 --- a/lib/paperclip/video_transcoder.rb +++ b/lib/paperclip/video_transcoder.rb @@ -6,7 +6,9 @@ module Paperclip class VideoTranscoder < Paperclip::Processor def make meta = ::Av.cli.identify(@file.path) + attachment.instance.type = MediaAttachment.types[:gifv] unless meta[:audio_encode] + options[:format] = File.extname(attachment.instance.file_file_name)[1..-1] if options[:keep_same_format] Paperclip::Transcoder.make(file, options, attachment) end -- cgit From bae268b2f6501ccb0ceeb8dd36180698839c4d3c Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Thu, 3 Oct 2019 04:13:22 +0200 Subject: Bump version to 3.0.0rc3 (#12063) --- CHANGELOG.md | 25 ++++++++++++++++++++++++- lib/mastodon/version.rb | 2 +- 2 files changed, 25 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e9ccdc8c..1f13ad450 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,7 @@ All notable changes to this project will be documented in this file. - Add profile directory REST API - Add special alert for throttled requests in web UI ([ThibG](https://github.com/tootsuite/mastodon/pull/11677)) - Add confirmation modal when logging out from the web UI ([ThibG](https://github.com/tootsuite/mastodon/pull/11671)) -- **Add audio player in web UI** ([Gargron](https://github.com/tootsuite/mastodon/pull/11644), [Gargron](https://github.com/tootsuite/mastodon/pull/11652), [Gargron](https://github.com/tootsuite/mastodon/pull/11654), [ThibG](https://github.com/tootsuite/mastodon/pull/11629)) +- **Add audio player in web UI** ([Gargron](https://github.com/tootsuite/mastodon/pull/11644), [Gargron](https://github.com/tootsuite/mastodon/pull/11652), [Gargron](https://github.com/tootsuite/mastodon/pull/11654), [ThibG](https://github.com/tootsuite/mastodon/pull/11629), [Gargron](https://github.com/tootsuite/mastodon/pull/12056)) - **Add autosuggestions for hashtags in web UI** ([Gargron](https://github.com/tootsuite/mastodon/pull/11422), [ThibG](https://github.com/tootsuite/mastodon/pull/11632), [Gargron](https://github.com/tootsuite/mastodon/pull/11764), [Gargron](https://github.com/tootsuite/mastodon/pull/11588), [Gargron](https://github.com/tootsuite/mastodon/pull/11442)) - **Add media editing modal with OCR tool in web UI** ([Gargron](https://github.com/tootsuite/mastodon/pull/11563), [Gargron](https://github.com/tootsuite/mastodon/pull/11566), [ThibG](https://github.com/tootsuite/mastodon/pull/11575), [ThibG](https://github.com/tootsuite/mastodon/pull/11576), [Gargron](https://github.com/tootsuite/mastodon/pull/11577), [Gargron](https://github.com/tootsuite/mastodon/pull/11573), [Gargron](https://github.com/tootsuite/mastodon/pull/11571)) - Add indicator of unread notifications to window title when web UI is out of focus ([Gargron](https://github.com/tootsuite/mastodon/pull/11560), [Gargron](https://github.com/tootsuite/mastodon/pull/11572)) @@ -22,6 +22,9 @@ All notable changes to this project will be documented in this file. - **Add option to disable real-time updates in web UI ("slow mode")** ([Gargron](https://github.com/tootsuite/mastodon/pull/9984), [ykzts](https://github.com/tootsuite/mastodon/pull/11880), [ThibG](https://github.com/tootsuite/mastodon/pull/11883), [Gargron](https://github.com/tootsuite/mastodon/pull/11898), [ThibG](https://github.com/tootsuite/mastodon/pull/11859)) - Add option to disable blurhash previews in web UI ([ThibG](https://github.com/tootsuite/mastodon/pull/11188)) - Add native smooth scrolling when supported in web UI ([ThibG](https://github.com/tootsuite/mastodon/pull/11207)) +- Add scrolling to the search bar on focus in web UI ([Kjwon15](https://github.com/tootsuite/mastodon/pull/12032)) +- Add refresh button to list of rebloggers/favouriters in web UI ([Gargron](https://github.com/tootsuite/mastodon/pull/12031)) +- Add error description and button to copy stack trace to web UI ([Gargron](https://github.com/tootsuite/mastodon/pull/12033)) - Add search and sort functions to hashtag admin UI ([mayaeh](https://github.com/tootsuite/mastodon/pull/11829), [Gargron](https://github.com/tootsuite/mastodon/pull/11897), [mayaeh](https://github.com/tootsuite/mastodon/pull/11875)) - Add setting for default search engine indexing in admin UI ([brortao](https://github.com/tootsuite/mastodon/pull/11804)) - Add account bio to account view in admin UI ([ThibG](https://github.com/tootsuite/mastodon/pull/11473)) @@ -64,6 +67,7 @@ All notable changes to this project will be documented in this file. - Add `tootctl media refresh` command ([Gargron](https://github.com/tootsuite/mastodon/pull/11775)) - Add `tootctl cache recount` command ([Gargron](https://github.com/tootsuite/mastodon/pull/11597)) - Add option to exclude suspended domains from `tootctl domains crawl` ([dariusk](https://github.com/tootsuite/mastodon/pull/11454)) +- Add parallelization to `tootctl search deploy` ([noellabo](https://github.com/tootsuite/mastodon/pull/12051)) - Add soft delete for statuses for instant deletes through API ([Gargron](https://github.com/tootsuite/mastodon/pull/11623), [Gargron](https://github.com/tootsuite/mastodon/pull/11648)) - Add rails-level JSON caching ([Gargron](https://github.com/tootsuite/mastodon/pull/11333), [Gargron](https://github.com/tootsuite/mastodon/pull/11271)) - **Add request pool to improve delivery performance** ([Gargron](https://github.com/tootsuite/mastodon/pull/10353), [ykzts](https://github.com/tootsuite/mastodon/pull/11756)) @@ -72,6 +76,13 @@ All notable changes to this project will be documented in this file. - **Add more accurate hashtag search** ([Gargron](https://github.com/tootsuite/mastodon/pull/11579), [Gargron](https://github.com/tootsuite/mastodon/pull/11427), [Gargron](https://github.com/tootsuite/mastodon/pull/11448)) - **Add more accurate account search** ([Gargron](https://github.com/tootsuite/mastodon/pull/11537), [Gargron](https://github.com/tootsuite/mastodon/pull/11580)) - **Add a spam check** ([Gargron](https://github.com/tootsuite/mastodon/pull/11217), [Gargron](https://github.com/tootsuite/mastodon/pull/11806), [ThibG](https://github.com/tootsuite/mastodon/pull/11296)) +- Add new languages ([Gargron](https://github.com/tootsuite/mastodon/pull/12062)) + - Breton + - Spanish (Argentina) + - Estonian + - Macedonian + - New Norwegian +- Add NodeInfo endpoint ([Gargron](https://github.com/tootsuite/mastodon/pull/12002), [Gargron](https://github.com/tootsuite/mastodon/pull/12058)) ### Changed @@ -100,6 +111,8 @@ All notable changes to this project will be documented in this file. - Change supported Node versions to include v12 ([abcang](https://github.com/tootsuite/mastodon/pull/11706)) - Change Portuguese language from `pt` to `pt-PT` ([Gargron](https://github.com/tootsuite/mastodon/pull/11820)) - Change domain block silence to always require approval on follow ([ThibG](https://github.com/tootsuite/mastodon/pull/11975)) +- Change link preview fetcher to not perform a HEAD request first ([Gargron](https://github.com/tootsuite/mastodon/pull/12028)) +- Change `tootctl domains purge` to accept multiple domains at once ([Gargron](https://github.com/tootsuite/mastodon/pull/12046)) ### Removed @@ -176,6 +189,16 @@ All notable changes to this project will be documented in this file. - Fix http_parser.rb gem not being compiled when no network available ([petabyteboy](https://github.com/tootsuite/mastodon/pull/11444)) - Fix muted text color not applying to all text ([trwnh](https://github.com/tootsuite/mastodon/pull/11996)) - Fix follower/following lists resetting on back-navigation in web UI ([Gargron](https://github.com/tootsuite/mastodon/pull/11986)) +- Fix n+1 query when approving multiple follow requests ([abcang](https://github.com/tootsuite/mastodon/pull/12004)) +- Fix records not being indexed into ElasticSearch sometimes ([Gargron](https://github.com/tootsuite/mastodon/pull/12024)) +- Fix needlessly indexing unsearchable statuses into ElasticSearch ([Gargron](https://github.com/tootsuite/mastodon/pull/12041)) +- Fix new user bootstrapping crashing when to-be-followed accouns are invalid ([ThibG](https://github.com/tootsuite/mastodon/pull/12037)) +- Fix featured hashtag URL being interpreted as media or with_replies ([Gargron](https://github.com/tootsuite/mastodon/pull/12048)) +- Fix account counters being overwritten by parallel writes ([Gargron](https://github.com/tootsuite/mastodon/pull/12045)) + +### Security + +- Fix performance of GIF re-encoding and always strip EXIF data from videos ([Gargron](https://github.com/tootsuite/mastodon/pull/12057)) ## [2.9.3] - 2019-08-10 ### Added diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb index 9c5686ed2..6e4cf551b 100644 --- a/lib/mastodon/version.rb +++ b/lib/mastodon/version.rb @@ -17,7 +17,7 @@ module Mastodon end def flags - 'rc2' + 'rc3' end def suffix -- cgit From 83d3e7733da892f5ad94ed2b8757db11250fbe6a Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Thu, 3 Oct 2019 22:44:22 +0200 Subject: Bump version to 3.0.0 (#12000) --- CHANGELOG.md | 8 ++++---- lib/mastodon/version.rb | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) (limited to 'lib') diff --git a/CHANGELOG.md b/CHANGELOG.md index 69ceb6028..6b0d23a22 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,8 +3,7 @@ Changelog All notable changes to this project will be documented in this file. -## Unreleased - +## [3.0.0] - 2019-10-03 ### Added - Add "not available" label to unloaded media attachments in web UI ([Gargron](https://github.com/tootsuite/mastodon/pull/11715), [Gargron](https://github.com/tootsuite/mastodon/pull/11745)) @@ -53,6 +52,7 @@ All notable changes to this project will be documented in this file. - **Add REST API for managing featured hashtags** ([noellabo](https://github.com/tootsuite/mastodon/pull/11778)) - **Add REST API for managing timeline read markers** ([Gargron](https://github.com/tootsuite/mastodon/pull/11762)) - Add `exclude_unreviewed` param to `GET /api/v2/search` REST API ([Gargron](https://github.com/tootsuite/mastodon/pull/11977)) +- Add `reason` param to `POST /api/v1/accounts` REST API ([Gargron](https://github.com/tootsuite/mastodon/pull/12064)) - **Add ActivityPub secure mode** ([Gargron](https://github.com/tootsuite/mastodon/pull/11269), [ThibG](https://github.com/tootsuite/mastodon/pull/11332), [ThibG](https://github.com/tootsuite/mastodon/pull/11295)) - Add HTTP signatures to all outgoing ActivityPub GET requests ([Gargron](https://github.com/tootsuite/mastodon/pull/11284), [ThibG](https://github.com/tootsuite/mastodon/pull/11300)) - Add support for ActivityPub Audio activities ([ThibG](https://github.com/tootsuite/mastodon/pull/11189)) @@ -191,8 +191,8 @@ All notable changes to this project will be documented in this file. - Fix n+1 query when approving multiple follow requests ([abcang](https://github.com/tootsuite/mastodon/pull/12004)) - Fix records not being indexed into ElasticSearch sometimes ([Gargron](https://github.com/tootsuite/mastodon/pull/12024)) - Fix needlessly indexing unsearchable statuses into ElasticSearch ([Gargron](https://github.com/tootsuite/mastodon/pull/12041)) -- Fix new user bootstrapping crashing when to-be-followed accouns are invalid ([ThibG](https://github.com/tootsuite/mastodon/pull/12037)) -- Fix featured hashtag URL being interpreted as media or with_replies ([Gargron](https://github.com/tootsuite/mastodon/pull/12048)) +- Fix new user bootstrapping crashing when to-be-followed accounts are invalid ([ThibG](https://github.com/tootsuite/mastodon/pull/12037)) +- Fix featured hashtag URL being interpreted as media or replies tab ([Gargron](https://github.com/tootsuite/mastodon/pull/12048)) - Fix account counters being overwritten by parallel writes ([Gargron](https://github.com/tootsuite/mastodon/pull/12045)) ### Security diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb index 6e4cf551b..a0f4678ec 100644 --- a/lib/mastodon/version.rb +++ b/lib/mastodon/version.rb @@ -17,7 +17,7 @@ module Mastodon end def flags - 'rc3' + '' end def suffix -- cgit From c4fbfaf0b85a589fd669d097471be6bfb82620c6 Mon Sep 17 00:00:00 2001 From: Jeong Arm Date: Sun, 6 Oct 2019 11:30:07 +0900 Subject: Remove unused option from tootctl accounts cull (#12074) --- lib/mastodon/accounts_cli.rb | 1 - 1 file changed, 1 deletion(-) (limited to 'lib') diff --git a/lib/mastodon/accounts_cli.rb b/lib/mastodon/accounts_cli.rb index a09a6ab04..6dbb75689 100644 --- a/lib/mastodon/accounts_cli.rb +++ b/lib/mastodon/accounts_cli.rb @@ -211,7 +211,6 @@ module Mastodon end option :concurrency, type: :numeric, default: 5, aliases: [:c] - option :verbose, type: :boolean, aliases: [:v] option :dry_run, type: :boolean desc 'cull', 'Remove remote accounts that no longer exist' long_desc <<-LONG_DESC -- cgit From 38b6c34e32e2624b1c110809428642dcd1f69c82 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Mon, 7 Oct 2019 04:24:05 +0200 Subject: Fix issues with tootctl's parallelization and progress reporting (#12093) --- lib/mastodon/cli_helper.rb | 36 +++++++++++++++++++++++++----------- 1 file changed, 25 insertions(+), 11 deletions(-) (limited to 'lib') diff --git a/lib/mastodon/cli_helper.rb b/lib/mastodon/cli_helper.rb index da7348349..c2950dffa 100644 --- a/lib/mastodon/cli_helper.rb +++ b/lib/mastodon/cli_helper.rb @@ -15,6 +15,11 @@ module Mastodon end def parallelize_with_progress(scope) + if options[:concurrency] < 1 + say('Cannot run with this concurrency setting, must be at least 1', :red) + exit(1) + end + ActiveRecord::Base.configurations[Rails.env]['pool'] = options[:concurrency] progress = create_progress_bar(scope.count) @@ -27,17 +32,26 @@ module Mastodon items.each do |item| futures << Concurrent::Future.execute(executor: pool) do - ActiveRecord::Base.connection_pool.with_connection do - begin - progress.log("Processing #{item.id}") if options[:verbose] - - result = yield(item) - aggregate.increment(result) if result.is_a?(Integer) - rescue => e - progress.log pastel.red("Error processing #{item.id}: #{e}") - ensure - progress.increment + begin + if !progress.total.nil? && progress.progress + 1 > progress.total + # The number of items has changed between start and now, + # since there is no good way to predict the final count from + # here, just change the progress bar to an indeterminate one + + progress.total = nil end + + progress.log("Processing #{item.id}") if options[:verbose] + + result = ActiveRecord::Base.connection_pool.with_connection do + yield(item) + end + + aggregate.increment(result) if result.is_a?(Integer) + rescue => e + progress.log pastel.red("Error processing #{item.id}: #{e}") + ensure + progress.increment end end end @@ -46,7 +60,7 @@ module Mastodon futures.map(&:value) end - progress.finish + progress.stop [total.value, aggregate.value] end -- cgit From 6c9b4f6b7710cbbdafcceafb59dfef0be6baba65 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Mon, 7 Oct 2019 05:05:02 +0200 Subject: Fix tootctl not allocating enough database connections for main thread (#12097) --- lib/mastodon/cli_helper.rb | 2 +- lib/mastodon/feeds_cli.rb | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/mastodon/cli_helper.rb b/lib/mastodon/cli_helper.rb index c2950dffa..ec4d9a81e 100644 --- a/lib/mastodon/cli_helper.rb +++ b/lib/mastodon/cli_helper.rb @@ -20,7 +20,7 @@ module Mastodon exit(1) end - ActiveRecord::Base.configurations[Rails.env]['pool'] = options[:concurrency] + ActiveRecord::Base.configurations[Rails.env]['pool'] = options[:concurrency] + 1 progress = create_progress_bar(scope.count) pool = Concurrent::FixedThreadPool.new(options[:concurrency]) diff --git a/lib/mastodon/feeds_cli.rb b/lib/mastodon/feeds_cli.rb index ea7c90dff..578ea15c5 100644 --- a/lib/mastodon/feeds_cli.rb +++ b/lib/mastodon/feeds_cli.rb @@ -27,7 +27,6 @@ module Mastodon dry_run = options[:dry_run] ? '(DRY RUN)' : '' if options[:all] || username.nil? - processed, = parallelize_with_progress(Account.joins(:user).merge(User.active)) do |account| PrecomputeFeedService.new.call(account) unless options[:dry_run] end -- cgit From 0aaa3afc2df2cc2fa8cdd277cf91dcfabf45afcd Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Mon, 7 Oct 2019 20:04:56 +0200 Subject: Add `tootctl media usage` command (#12115) --- lib/mastodon/media_cli.rb | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'lib') diff --git a/lib/mastodon/media_cli.rb b/lib/mastodon/media_cli.rb index ec2f36c30..bb97751d5 100644 --- a/lib/mastodon/media_cli.rb +++ b/lib/mastodon/media_cli.rb @@ -97,5 +97,17 @@ module Mastodon say("Downloaded #{processed} media attachments (approx. #{number_to_human_size(aggregate)})#{dry_run}", :green, true) end + + desc 'usage', 'Calculate disk space consumed by Mastodon' + def usage + say("Attachments:\t#{number_to_human_size(MediaAttachment.sum(:file_file_size))} (#{number_to_human_size(MediaAttachment.where(account: Account.local).sum(:file_file_size))} local)") + say("Custom emoji:\t#{number_to_human_size(CustomEmoji.sum(:image_file_size))} (#{number_to_human_size(CustomEmoji.local.sum(:image_file_size))} local)") + say("Preview cards:\t#{number_to_human_size(PreviewCard.sum(:image_file_size))}") + say("Avatars:\t#{number_to_human_size(Account.sum(:avatar_file_size))} (#{number_to_human_size(Account.local.sum(:avatar_file_size))} local)") + say("Headers:\t#{number_to_human_size(Account.sum(:header_file_size))} (#{number_to_human_size(Account.local.sum(:header_file_size))} local)") + say("Backups:\t#{number_to_human_size(Backup.sum(:dump_file_size))}") + say("Imports:\t#{number_to_human_size(Import.sum(:data_file_size))}") + say("Settings:\t#{number_to_human_size(SiteUpload.sum(:file_file_size))}") + end end end -- cgit From 4a98e77d0e7fec15e1830059a9a56bf56aa391db Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 8 Oct 2019 05:59:10 +0200 Subject: Change `tootctl media refresh` to skip already downloaded attachments (#12118) --- lib/mastodon/media_cli.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/mastodon/media_cli.rb b/lib/mastodon/media_cli.rb index bb97751d5..e48175134 100644 --- a/lib/mastodon/media_cli.rb +++ b/lib/mastodon/media_cli.rb @@ -50,6 +50,7 @@ module Mastodon option :concurrency, type: :numeric, default: 5, aliases: [:c] option :verbose, type: :boolean, default: false, aliases: [:v] option :dry_run, type: :boolean, default: false + option :force, type: :boolean, default: false desc 'refresh', 'Fetch remote media files' long_desc <<-DESC Re-downloads media attachments from other servers. You must specify the @@ -62,6 +63,9 @@ module Mastodon using username@domain handle of the account. Use the --domain option to download attachments from a specific domain. + + By default, attachments that are believed to be already downloaded will + not be re-downloaded. To force re-download of every URL, use --force. DESC def refresh dry_run = options[:dry_run] ? ' (DRY RUN)' : '' @@ -85,7 +89,7 @@ module Mastodon end processed, aggregate = parallelize_with_progress(scope) do |media_attachment| - next if media_attachment.remote_url.blank? + next if media_attachment.remote_url.blank? || (!options[:force] && media_attachment.file_file_name.present?) unless options[:dry_run] media_attachment.reset_file! -- cgit From e6d111f38b95b08a2880390b1519655b54ac2fd0 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Wed, 9 Oct 2019 07:36:57 +0200 Subject: Bump version to 3.0.1 (#12116) --- CHANGELOG.md | 30 ++++++++++++++++++++++++++++++ lib/mastodon/version.rb | 2 +- 2 files changed, 31 insertions(+), 1 deletion(-) (limited to 'lib') diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b0d23a22..47a466294 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,36 @@ Changelog All notable changes to this project will be documented in this file. +## [3.0.1] - 2019-10-09 +### Added + +- Add `tootctl media usage` command ([Gargron](https://github.com/tootsuite/mastodon/pull/12115)) +- Add admin setting to auto-approve trending hashtags ([Gargron](https://github.com/tootsuite/mastodon/pull/12122)) + +### Changed + +- Change `tootctl media refresh` to skip already downloaded attachments ([Gargron](https://github.com/tootsuite/mastodon/pull/12118)) + +### Removed + +- Remove auto-silence behaviour from spam check ([Gargron](https://github.com/tootsuite/mastodon/pull/12117)) +- Remove HTML `lang` attribute from individual statuses in web UI ([Gargron](https://github.com/tootsuite/mastodon/pull/12124)) +- Remove fallback to long description on sidebar and meta description ([Gargron](https://github.com/tootsuite/mastodon/pull/12119)) + +### Fixed + +- Fix attachment not being re-downloaded even if file is not stored ([Gargron](https://github.com/tootsuite/mastodon/pull/12125)) +- Fix old migration trying to use new column due to default status scope ([Gargron](https://github.com/tootsuite/mastodon/pull/12095)) +- Fix column back button missing for not found accounts ([trwnh](https://github.com/tootsuite/mastodon/pull/12094)) +- Fix issues with tootctl's parallelization and progress reporting ([Gargron](https://github.com/tootsuite/mastodon/pull/12093), [Gargron](https://github.com/tootsuite/mastodon/pull/12097)) +- Fix existing user records with now-renamed `pt` locale ([Gargron](https://github.com/tootsuite/mastodon/pull/12092)) +- Fix hashtag timeline REST API accepting too many hashtags ([Gargron](https://github.com/tootsuite/mastodon/pull/12091)) +- Fix `GET /api/v1/instance` REST APIs being unavailable in secure mode ([Gargron](https://github.com/tootsuite/mastodon/pull/12089)) +- Fix performance of home feed regeneration and merging ([Gargron](https://github.com/tootsuite/mastodon/pull/12084)) +- Fix ffmpeg performance issues due to stdout buffer overflow ([hugogameiro](https://github.com/tootsuite/mastodon/pull/12088)) +- Fix S3 adapter retrying failing uploads with exponential backoff ([Gargron](https://github.com/tootsuite/mastodon/pull/12085)) +- Fix `tootctl accounts cull` advertising unused option flag ([Kjwon15](https://github.com/tootsuite/mastodon/pull/12074)) + ## [3.0.0] - 2019-10-03 ### Added diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb index a0f4678ec..f3ead6d8d 100644 --- a/lib/mastodon/version.rb +++ b/lib/mastodon/version.rb @@ -13,7 +13,7 @@ module Mastodon end def patch - 0 + 1 end def flags -- cgit From 45eccaf8c530c91ff7b0c599bc2e783fe9eb1e6b Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Thu, 10 Oct 2019 06:48:53 +0200 Subject: Fix preloaded JSON-LD context for identity not being used (#12138) Regression from #11316 --- config/initializers/json_ld.rb | 1 + lib/json_ld/identity.rb | 87 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+) create mode 100644 lib/json_ld/identity.rb (limited to 'lib') diff --git a/config/initializers/json_ld.rb b/config/initializers/json_ld.rb index d5575d135..3ed3c4b31 100644 --- a/config/initializers/json_ld.rb +++ b/config/initializers/json_ld.rb @@ -1,3 +1,4 @@ # frozen_string_literal: true require_relative '../../lib/json_ld/security' +require_relative '../../lib/json_ld/identity' diff --git a/lib/json_ld/identity.rb b/lib/json_ld/identity.rb new file mode 100644 index 000000000..4fb3f8e9d --- /dev/null +++ b/lib/json_ld/identity.rb @@ -0,0 +1,87 @@ +# -*- encoding: utf-8 -*- +# frozen_string_literal: true +# This file generated automatically from http://w3id.org/identity/v1 +require 'json/ld' +class JSON::LD::Context + add_preloaded("http://w3id.org/identity/v1") do + new(term_definitions: { + "Credential" => TermDefinition.new("Credential", id: "https://w3id.org/credentials#Credential", simple: true), + "CryptographicKey" => TermDefinition.new("CryptographicKey", id: "https://w3id.org/security#Key", simple: true), + "CryptographicKeyCredential" => TermDefinition.new("CryptographicKeyCredential", id: "https://w3id.org/credentials#CryptographicKeyCredential", simple: true), + "EncryptedMessage" => TermDefinition.new("EncryptedMessage", id: "https://w3id.org/security#EncryptedMessage", simple: true), + "GraphSignature2012" => TermDefinition.new("GraphSignature2012", id: "https://w3id.org/security#GraphSignature2012", simple: true), + "Group" => TermDefinition.new("Group", id: "https://www.w3.org/ns/activitystreams#Group", simple: true), + "Identity" => TermDefinition.new("Identity", id: "https://w3id.org/identity#Identity", simple: true), + "LinkedDataSignature2015" => TermDefinition.new("LinkedDataSignature2015", id: "https://w3id.org/security#LinkedDataSignature2015", simple: true), + "Organization" => TermDefinition.new("Organization", id: "http://schema.org/Organization", simple: true), + "Person" => TermDefinition.new("Person", id: "http://schema.org/Person", simple: true), + "PostalAddress" => TermDefinition.new("PostalAddress", id: "http://schema.org/PostalAddress", simple: true), + "about" => TermDefinition.new("about", id: "http://schema.org/about", type_mapping: "@id"), + "accessControl" => TermDefinition.new("accessControl", id: "https://w3id.org/permissions#accessControl", type_mapping: "@id"), + "address" => TermDefinition.new("address", id: "http://schema.org/address", type_mapping: "@id"), + "addressCountry" => TermDefinition.new("addressCountry", id: "http://schema.org/addressCountry", simple: true), + "addressLocality" => TermDefinition.new("addressLocality", id: "http://schema.org/addressLocality", simple: true), + "addressRegion" => TermDefinition.new("addressRegion", id: "http://schema.org/addressRegion", simple: true), + "cipherAlgorithm" => TermDefinition.new("cipherAlgorithm", id: "https://w3id.org/security#cipherAlgorithm", simple: true), + "cipherData" => TermDefinition.new("cipherData", id: "https://w3id.org/security#cipherData", simple: true), + "cipherKey" => TermDefinition.new("cipherKey", id: "https://w3id.org/security#cipherKey", simple: true), + "claim" => TermDefinition.new("claim", id: "https://w3id.org/credentials#claim", type_mapping: "@id"), + "comment" => TermDefinition.new("comment", id: "http://www.w3.org/2000/01/rdf-schema#comment", simple: true), + "created" => TermDefinition.new("created", id: "http://purl.org/dc/terms/created", type_mapping: "http://www.w3.org/2001/XMLSchema#dateTime"), + "creator" => TermDefinition.new("creator", id: "http://purl.org/dc/terms/creator", type_mapping: "@id"), + "cred" => TermDefinition.new("cred", id: "https://w3id.org/credentials#", simple: true, prefix: true), + "credential" => TermDefinition.new("credential", id: "https://w3id.org/credentials#credential", type_mapping: "@id"), + "dc" => TermDefinition.new("dc", id: "http://purl.org/dc/terms/", simple: true, prefix: true), + "description" => TermDefinition.new("description", id: "http://schema.org/description", simple: true), + "digestAlgorithm" => TermDefinition.new("digestAlgorithm", id: "https://w3id.org/security#digestAlgorithm", simple: true), + "digestValue" => TermDefinition.new("digestValue", id: "https://w3id.org/security#digestValue", simple: true), + "domain" => TermDefinition.new("domain", id: "https://w3id.org/security#domain", simple: true), + "email" => TermDefinition.new("email", id: "http://schema.org/email", simple: true), + "expires" => TermDefinition.new("expires", id: "https://w3id.org/security#expiration", type_mapping: "http://www.w3.org/2001/XMLSchema#dateTime"), + "familyName" => TermDefinition.new("familyName", id: "http://schema.org/familyName", simple: true), + "givenName" => TermDefinition.new("givenName", id: "http://schema.org/givenName", simple: true), + "id" => TermDefinition.new("id", id: "@id", simple: true), + "identity" => TermDefinition.new("identity", id: "https://w3id.org/identity#", simple: true, prefix: true), + "identityService" => TermDefinition.new("identityService", id: "https://w3id.org/identity#identityService", type_mapping: "@id"), + "idp" => TermDefinition.new("idp", id: "https://w3id.org/identity#idp", type_mapping: "@id"), + "image" => TermDefinition.new("image", id: "http://schema.org/image", type_mapping: "@id"), + "initializationVector" => TermDefinition.new("initializationVector", id: "https://w3id.org/security#initializationVector", simple: true), + "issued" => TermDefinition.new("issued", id: "https://w3id.org/credentials#issued", type_mapping: "http://www.w3.org/2001/XMLSchema#dateTime"), + "issuer" => TermDefinition.new("issuer", id: "https://w3id.org/credentials#issuer", type_mapping: "@id"), + "label" => TermDefinition.new("label", id: "http://www.w3.org/2000/01/rdf-schema#label", simple: true), + "member" => TermDefinition.new("member", id: "http://schema.org/member", type_mapping: "@id"), + "memberOf" => TermDefinition.new("memberOf", id: "http://schema.org/memberOf", type_mapping: "@id"), + "name" => TermDefinition.new("name", id: "http://schema.org/name", simple: true), + "nonce" => TermDefinition.new("nonce", id: "https://w3id.org/security#nonce", simple: true), + "normalizationAlgorithm" => TermDefinition.new("normalizationAlgorithm", id: "https://w3id.org/security#normalizationAlgorithm", simple: true), + "owner" => TermDefinition.new("owner", id: "https://w3id.org/security#owner", type_mapping: "@id"), + "password" => TermDefinition.new("password", id: "https://w3id.org/security#password", simple: true), + "paymentProcessor" => TermDefinition.new("paymentProcessor", id: "https://w3id.org/payswarm#processor", simple: true), + "perm" => TermDefinition.new("perm", id: "https://w3id.org/permissions#", simple: true, prefix: true), + "postalCode" => TermDefinition.new("postalCode", id: "http://schema.org/postalCode", simple: true), + "preferences" => TermDefinition.new("preferences", id: "https://w3id.org/payswarm#preferences", type_mapping: "@vocab"), + "privateKey" => TermDefinition.new("privateKey", id: "https://w3id.org/security#privateKey", type_mapping: "@id"), + "privateKeyPem" => TermDefinition.new("privateKeyPem", id: "https://w3id.org/security#privateKeyPem", simple: true), + "ps" => TermDefinition.new("ps", id: "https://w3id.org/payswarm#", simple: true, prefix: true), + "publicKey" => TermDefinition.new("publicKey", id: "https://w3id.org/security#publicKey", type_mapping: "@id"), + "publicKeyPem" => TermDefinition.new("publicKeyPem", id: "https://w3id.org/security#publicKeyPem", simple: true), + "publicKeyService" => TermDefinition.new("publicKeyService", id: "https://w3id.org/security#publicKeyService", type_mapping: "@id"), + "rdf" => TermDefinition.new("rdf", id: "http://www.w3.org/1999/02/22-rdf-syntax-ns#", simple: true, prefix: true), + "rdfs" => TermDefinition.new("rdfs", id: "http://www.w3.org/2000/01/rdf-schema#", simple: true, prefix: true), + "recipient" => TermDefinition.new("recipient", id: "https://w3id.org/credentials#recipient", type_mapping: "@id"), + "revoked" => TermDefinition.new("revoked", id: "https://w3id.org/security#revoked", type_mapping: "http://www.w3.org/2001/XMLSchema#dateTime"), + "schema" => TermDefinition.new("schema", id: "http://schema.org/", simple: true, prefix: true), + "sec" => TermDefinition.new("sec", id: "https://w3id.org/security#", simple: true, prefix: true), + "signature" => TermDefinition.new("signature", id: "https://w3id.org/security#signature", simple: true), + "signatureAlgorithm" => TermDefinition.new("signatureAlgorithm", id: "https://w3id.org/security#signatureAlgorithm", simple: true), + "signatureValue" => TermDefinition.new("signatureValue", id: "https://w3id.org/security#signatureValue", simple: true), + "streetAddress" => TermDefinition.new("streetAddress", id: "http://schema.org/streetAddress", simple: true), + "title" => TermDefinition.new("title", id: "http://purl.org/dc/terms/title", simple: true), + "type" => TermDefinition.new("type", id: "@type", simple: true), + "url" => TermDefinition.new("url", id: "http://schema.org/url", type_mapping: "@id"), + "writePermission" => TermDefinition.new("writePermission", id: "https://w3id.org/permissions#writePermission", type_mapping: "@id"), + "xsd" => TermDefinition.new("xsd", id: "http://www.w3.org/2001/XMLSchema#", simple: true, prefix: true) + }) + end + alias_preloaded("https://w3id.org/identity/v1", "http://w3id.org/identity/v1") +end -- cgit From 23ed9303b890f07715b7afaf0e1a18c13e40ea15 Mon Sep 17 00:00:00 2001 From: Nathaniel Suchy Date: Mon, 4 Nov 2019 06:55:20 -0500 Subject: Add `tootctl media lookup` command (#12283) * Add a lookup tool to the media cli * Improved lookup logic * Clarified wording in the output * Code style changes * Code style changes * Code style changes * Code style changes * Add error handling code incase an attachment isn't found * Code style changes * Code style changes * Make requested changes * Fix styling issues * Handle other media types * Remove an inadvertently added log * Make requested changes * Make the code safe no matter what the path, S3 or not * Code style changes * Code style changes * Replace select method with Ruby Enumerable grep method --- lib/mastodon/media_cli.rb | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) (limited to 'lib') diff --git a/lib/mastodon/media_cli.rb b/lib/mastodon/media_cli.rb index e48175134..3b702f155 100644 --- a/lib/mastodon/media_cli.rb +++ b/lib/mastodon/media_cli.rb @@ -113,5 +113,27 @@ module Mastodon say("Imports:\t#{number_to_human_size(Import.sum(:data_file_size))}") say("Settings:\t#{number_to_human_size(SiteUpload.sum(:file_file_size))}") end + + desc 'lookup', 'Lookup where media is displayed by passing a media URL' + def lookup + prompt = TTY::Prompt.new + + url = prompt.ask('Please enter a URL to the media to lookup:', required: true) + + attachment_id = url + .split('/')[0..-2] + .grep(/\A\d+\z/) + .join('') + + if url.split('/')[0..-2].include? 'media_attachments' + model = MediaAttachment.find(attachment_id).status + prompt.say(ActivityPub::TagManager.instance.url_for(model)) + elsif url.split('/')[0..-2].include? 'accounts' + model = Account.find(attachment_id) + prompt.say(ActivityPub::TagManager.instance.url_for(model)) + else + prompt.say('Not found') + end + end end end -- cgit From 6be16d02cb3f03d21226f9b6664d33e3e8650043 Mon Sep 17 00:00:00 2001 From: Yamagishi Kazutoshi Date: Tue, 3 Dec 2019 02:25:43 +0900 Subject: Update ESLint and RuboCop in Code Climate (#12534) --- .codeclimate.yml | 4 ++-- .rubocop.yml | 6 ++++++ app/lib/activitypub/adapter.rb | 1 + app/lib/language_detector.rb | 2 +- app/models/media_attachment.rb | 2 +- lib/paperclip/lazy_thumbnail.rb | 4 ++-- 6 files changed, 13 insertions(+), 6 deletions(-) (limited to 'lib') diff --git a/.codeclimate.yml b/.codeclimate.yml index 571507a54..9817d7f1c 100644 --- a/.codeclimate.yml +++ b/.codeclimate.yml @@ -27,10 +27,10 @@ plugins: enabled: true eslint: enabled: true - channel: eslint-5 + channel: eslint-6 rubocop: enabled: true - channel: rubocop-0-71 + channel: rubocop-0-76 sass-lint: enabled: true exclude_patterns: diff --git a/.rubocop.yml b/.rubocop.yml index 8bd4c867f..9a68becbb 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -71,6 +71,9 @@ Naming/MemoizedInstanceVariableName: Rails: Enabled: true +Rails/EnumHash: + Enabled: false + Rails/HasAndBelongsToMany: Enabled: false @@ -102,6 +105,9 @@ Style/Documentation: Style/DoubleNegation: Enabled: true +Style/FormatStringToken: + Enabled: false + Style/FrozenStringLiteralComment: Enabled: true diff --git a/app/lib/activitypub/adapter.rb b/app/lib/activitypub/adapter.rb index 2a8f72333..78138fb73 100644 --- a/app/lib/activitypub/adapter.rb +++ b/app/lib/activitypub/adapter.rb @@ -35,6 +35,7 @@ class ActivityPub::Adapter < ActiveModelSerializers::Adapter::Base def serializable_hash(options = nil) named_contexts = {} context_extensions = {} + options = serialization_options(options) serialized_hash = serializer.serializable_hash(options.merge(named_contexts: named_contexts, context_extensions: context_extensions)) serialized_hash = serialized_hash.select { |k, _| options[:fields].include?(k) } if options[:fields] diff --git a/app/lib/language_detector.rb b/app/lib/language_detector.rb index 6f9511a54..302072bcc 100644 --- a/app/lib/language_detector.rb +++ b/app/lib/language_detector.rb @@ -44,7 +44,7 @@ class LanguageDetector words = text.scan(RELIABLE_CHARACTERS_RE) if words.present? - words.reduce(0) { |acc, elem| acc + elem.size }.to_f / text.size.to_f > 0.3 + words.reduce(0) { |acc, elem| acc + elem.size }.to_f / text.size > 0.3 else false end diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb index 1f9d92b22..5d5034a52 100644 --- a/app/models/media_attachment.rb +++ b/app/models/media_attachment.rb @@ -287,7 +287,7 @@ class MediaAttachment < ApplicationRecord width: width, height: height, size: "#{width}x#{height}", - aspect: width.to_f / height.to_f, + aspect: width.to_f / height, } end diff --git a/lib/paperclip/lazy_thumbnail.rb b/lib/paperclip/lazy_thumbnail.rb index 542c17fb2..10b14860c 100644 --- a/lib/paperclip/lazy_thumbnail.rb +++ b/lib/paperclip/lazy_thumbnail.rb @@ -9,8 +9,8 @@ module Paperclip min_side = [@current_geometry.width, @current_geometry.height].min.to_i options[:geometry] = "#{min_side}x#{min_side}#" if @target_geometry.square? && min_side < @target_geometry.width elsif options[:pixels] - width = Math.sqrt(options[:pixels] * (@current_geometry.width.to_f / @current_geometry.height.to_f)).round.to_i - height = Math.sqrt(options[:pixels] * (@current_geometry.height.to_f / @current_geometry.width.to_f)).round.to_i + width = Math.sqrt(options[:pixels] * (@current_geometry.width.to_f / @current_geometry.height)).round.to_i + height = Math.sqrt(options[:pixels] * (@current_geometry.height.to_f / @current_geometry.width)).round.to_i options[:geometry] = "#{width}x#{height}>" end -- cgit