From df1653174be233f2737d8ec281325dee54011947 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Thu, 19 Nov 2020 17:38:06 +0100 Subject: Add cache buster feature for media files (#15155) Nginx can be configured to bypass proxy cache when a special header is in the request. If the response is cacheable, it will replace the cache for that request. Proxy caching of media files is desirable when using object storage as a way of minimizing bandwidth costs, but has the drawback of leaving deleted media files for a configured amount of cache time. A cache buster can make those media files immediately unavailable. This especially makes sense when suspending and unsuspending an account. --- config/initializers/cache_buster.rb | 10 ++++++++++ config/initializers/paperclip.rb | 1 - 2 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 config/initializers/cache_buster.rb (limited to 'config') diff --git a/config/initializers/cache_buster.rb b/config/initializers/cache_buster.rb new file mode 100644 index 000000000..227e450f3 --- /dev/null +++ b/config/initializers/cache_buster.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +Rails.application.configure do + config.x.cache_buster_enabled = ENV['CACHE_BUSTER_ENABLED'] == 'true' + + config.x.cache_buster = { + secret_header: ENV['CACHE_BUSTER_SECRET_HEADER'], + secret: ENV['CACHE_BUSTER_SECRET'], + } +end diff --git a/config/initializers/paperclip.rb b/config/initializers/paperclip.rb index b841d5220..25adcd8d6 100644 --- a/config/initializers/paperclip.rb +++ b/config/initializers/paperclip.rb @@ -107,7 +107,6 @@ elsif ENV['SWIFT_ENABLED'] == 'true' else Paperclip::Attachment.default_options.merge!( storage: :filesystem, - use_timestamp: true, path: File.join(ENV.fetch('PAPERCLIP_ROOT_PATH', File.join(':rails_root', 'public', 'system')), ':prefix_path:class', ':attachment', ':id_partition', ':style', ':filename'), url: ENV.fetch('PAPERCLIP_ROOT_URL', '/system') + '/:prefix_url:class/:attachment/:id_partition/:style/:filename', ) -- cgit From 96c1e7132971877fba308c51cd42306f0b1bf166 Mon Sep 17 00:00:00 2001 From: ThibG Date: Thu, 19 Nov 2020 17:48:13 +0100 Subject: Add import/export feature for bookmarks (#14956) * Add ability to export bookmarks * Add support for importing bookmarks * Add bookmark import tests * Add bookmarks export test --- .../settings/exports/bookmarks_controller.rb | 19 +++++++++ app/models/export.rb | 12 ++++++ app/models/import.rb | 2 +- app/services/import_service.rb | 45 ++++++++++++++++++++++ app/views/settings/exports/show.html.haml | 4 ++ config/locales/en.yml | 2 + config/routes.rb | 1 + .../settings/exports/bookmarks_controller_specs.rb | 17 ++++++++ spec/fixtures/files/bookmark-imports.txt | 4 ++ spec/services/import_service_spec.rb | 42 ++++++++++++++++++++ 10 files changed, 147 insertions(+), 1 deletion(-) create mode 100644 app/controllers/settings/exports/bookmarks_controller.rb create mode 100644 spec/controllers/settings/exports/bookmarks_controller_specs.rb create mode 100644 spec/fixtures/files/bookmark-imports.txt (limited to 'config') diff --git a/app/controllers/settings/exports/bookmarks_controller.rb b/app/controllers/settings/exports/bookmarks_controller.rb new file mode 100644 index 000000000..c12e2f147 --- /dev/null +++ b/app/controllers/settings/exports/bookmarks_controller.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Settings + module Exports + class BookmarksController < BaseController + include ExportControllerConcern + + def index + send_export_file + end + + private + + def export_data + @export.to_bookmarks_csv + end + end + end +end diff --git a/app/models/export.rb b/app/models/export.rb index cab01f11a..5216eed5e 100644 --- a/app/models/export.rb +++ b/app/models/export.rb @@ -9,6 +9,14 @@ class Export @account = account end + def to_bookmarks_csv + CSV.generate do |csv| + account.bookmarks.includes(:status).reorder(id: :desc).each do |bookmark| + csv << [ActivityPub::TagManager.instance.uri_for(bookmark.status)] + end + end + end + def to_blocked_accounts_csv to_csv account.blocking.select(:username, :domain) end @@ -55,6 +63,10 @@ class Export account.statuses_count end + def total_bookmarks + account.bookmarks.count + end + def total_follows account.following_count end diff --git a/app/models/import.rb b/app/models/import.rb index c78a04d07..702453289 100644 --- a/app/models/import.rb +++ b/app/models/import.rb @@ -24,7 +24,7 @@ class Import < ApplicationRecord belongs_to :account - enum type: [:following, :blocking, :muting, :domain_blocking] + enum type: [:following, :blocking, :muting, :domain_blocking, :bookmarks] validates :type, presence: true diff --git a/app/services/import_service.rb b/app/services/import_service.rb index 7e55452de..288e47f1e 100644 --- a/app/services/import_service.rb +++ b/app/services/import_service.rb @@ -18,6 +18,8 @@ class ImportService < BaseService import_mutes! when 'domain_blocking' import_domain_blocks! + when 'bookmarks' + import_bookmarks! end end @@ -88,6 +90,39 @@ class ImportService < BaseService end end + def import_bookmarks! + parse_import_data!(['#uri']) + items = @data.take(ROWS_PROCESSING_LIMIT).map { |row| row['#uri'].strip } + + if @import.overwrite? + presence_hash = items.each_with_object({}) { |id, mapping| mapping[id] = true } + + @account.bookmarks.find_each do |bookmark| + if presence_hash[bookmark.status.uri] + items.delete(bookmark.status.uri) + else + bookmark.destroy! + end + end + end + + statuses = items.map do |uri| + status = ActivityPub::TagManager.instance.uri_to_resource(uri, Status) + next if status.nil? && ActivityPub::TagManager.instance.local_uri?(uri) + + status || ActivityPub::FetchRemoteStatusService.new.call(uri) + end.compact + + account_ids = statuses.map(&:account_id) + preloaded_relations = relations_map_for_account(@account, account_ids) + + statuses.keep_if { |status| StatusPolicy.new(@account, status, preloaded_relations).show? } + + statuses.each do |status| + @account.bookmarks.find_or_create_by!(account: @account, status: status) + end + end + def parse_import_data!(default_headers) data = CSV.parse(import_data, headers: true) data = CSV.parse(import_data, headers: default_headers) unless data.headers&.first&.strip&.include?(' ') @@ -101,4 +136,14 @@ class ImportService < BaseService def follow_limit FollowLimitValidator.limit_for_account(@account) end + + def relations_map_for_account(account, account_ids) + { + blocking: {}, + blocked_by: Account.blocked_by_map(account_ids, account.id), + muting: {}, + following: Account.following_map(account_ids, account.id), + domain_blocking_by_domain: {}, + } + end end diff --git a/app/views/settings/exports/show.html.haml b/app/views/settings/exports/show.html.haml index 0bb80e937..18b52c0c2 100644 --- a/app/views/settings/exports/show.html.haml +++ b/app/views/settings/exports/show.html.haml @@ -36,6 +36,10 @@ %th= t('exports.domain_blocks') %td= number_with_delimiter @export.total_domain_blocks %td= table_link_to 'download', t('exports.csv'), settings_exports_domain_blocks_path(format: :csv) + %tr + %th= t('exports.bookmarks') + %td= number_with_delimiter @export.total_bookmarks + %td= table_link_to 'download', t('bookmarks.csv'), settings_exports_bookmarks_path(format: :csv) %hr.spacer/ diff --git a/config/locales/en.yml b/config/locales/en.yml index bec099082..263ffcdc7 100644 --- a/config/locales/en.yml +++ b/config/locales/en.yml @@ -842,6 +842,7 @@ en: request: Request your archive size: Size blocks: You block + bookmarks: Bookmarks csv: CSV domain_blocks: Domain blocks lists: Lists @@ -918,6 +919,7 @@ en: success: Your data was successfully uploaded and will now be processed in due time types: blocking: Blocking list + bookmarks: Bookmarks domain_blocking: Domain blocking list following: Following list muting: Muting list diff --git a/config/routes.rb b/config/routes.rb index 54c76799c..a534b433e 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -125,6 +125,7 @@ Rails.application.routes.draw do resources :mutes, only: :index, controller: :muted_accounts resources :lists, only: :index, controller: :lists resources :domain_blocks, only: :index, controller: :blocked_domains + resources :bookmarks, only: :index, controller: :bookmarks end resources :two_factor_authentication_methods, only: [:index] do diff --git a/spec/controllers/settings/exports/bookmarks_controller_specs.rb b/spec/controllers/settings/exports/bookmarks_controller_specs.rb new file mode 100644 index 000000000..85761577b --- /dev/null +++ b/spec/controllers/settings/exports/bookmarks_controller_specs.rb @@ -0,0 +1,17 @@ +require 'rails_helper' + +describe Settings::Exports::BookmarksController do + render_views + + describe 'GET #index' do + it 'returns a csv of the bookmarked toots' do + user = Fabricate(:user) + user.account.bookmarks.create!(status: Fabricate(:status, uri: 'https://foo.bar/statuses/1312')) + + sign_in user, scope: :user + get :index, format: :csv + + expect(response.body).to eq "https://foo.bar/statuses/1312\n" + end + end +end diff --git a/spec/fixtures/files/bookmark-imports.txt b/spec/fixtures/files/bookmark-imports.txt new file mode 100644 index 000000000..7cc8901a0 --- /dev/null +++ b/spec/fixtures/files/bookmark-imports.txt @@ -0,0 +1,4 @@ +https://example.com/statuses/1312 +https://local.com/users/foo/statuses/42 +https://unknown-remote.com/users/bar/statuses/1 +https://example.com/statuses/direct diff --git a/spec/services/import_service_spec.rb b/spec/services/import_service_spec.rb index b1909d4fd..764225aa7 100644 --- a/spec/services/import_service_spec.rb +++ b/spec/services/import_service_spec.rb @@ -1,6 +1,8 @@ require 'rails_helper' RSpec.describe ImportService, type: :service do + include RoutingHelper + let!(:account) { Fabricate(:account, locked: false) } let!(:bob) { Fabricate(:account, username: 'bob', locked: false) } let!(:eve) { Fabricate(:account, username: 'eve', domain: 'example.com', locked: false, protocol: :activitypub, inbox_url: 'https://example.com/inbox') } @@ -169,4 +171,44 @@ RSpec.describe ImportService, type: :service do end end end + + context 'import bookmarks' do + subject { ImportService.new } + + let(:csv) { attachment_fixture('bookmark-imports.txt') } + + around(:each) do |example| + local_before = Rails.configuration.x.local_domain + web_before = Rails.configuration.x.web_domain + Rails.configuration.x.local_domain = 'local.com' + Rails.configuration.x.web_domain = 'local.com' + example.run + Rails.configuration.x.web_domain = web_before + Rails.configuration.x.local_domain = local_before + end + + let(:local_account) { Fabricate(:account, username: 'foo', domain: '') } + let!(:remote_status) { Fabricate(:status, uri: 'https://example.com/statuses/1312') } + let!(:direct_status) { Fabricate(:status, uri: 'https://example.com/statuses/direct', visibility: :direct) } + + before do + service = double + allow(ActivityPub::FetchRemoteStatusService).to receive(:new).and_return(service) + allow(service).to receive(:call).with('https://unknown-remote.com/users/bar/statuses/1') do + Fabricate(:status, uri: 'https://unknown-remote.com/users/bar/statuses/1') + end + end + + describe 'when no bookmarks are set' do + let(:import) { Import.create(account: account, type: 'bookmarks', data: csv) } + it 'adds the toots the user has access to to bookmarks' do + local_status = Fabricate(:status, account: local_account, uri: 'https://local.com/users/foo/statuses/42', id: 42, local: true) + subject.call(import) + expect(account.bookmarks.map(&:status).map(&:id)).to include(local_status.id) + expect(account.bookmarks.map(&:status).map(&:id)).to include(remote_status.id) + expect(account.bookmarks.map(&:status).map(&:id)).not_to include(direct_status.id) + expect(account.bookmarks.count).to eq 3 + end + end + end end -- cgit