From 0a6b0986685e6a54042b22ae01df4dd0aad9f6b0 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Tue, 12 Sep 2017 00:16:03 +0200 Subject: Fix count numbers from ActivityPub not being saved (#4899) They are marked as read-only by Rails, but we know what we are doing, so we are un-marking them as such. The mastodon:maintenance:update_counter_caches task is not really supposed to be run anymore (it was a one-time thing during an upgrade) however, just in case, I have modified it to not touch ActivityPub accounts. Also, no point writing to logger from these rake tasks, since they are not to be run from cron. Better to give stdout feedback. --- lib/tasks/mastodon.rake | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/lib/tasks/mastodon.rake b/lib/tasks/mastodon.rake index 3c65ece4b..2aef752ca 100644 --- a/lib/tasks/mastodon.rake +++ b/lib/tasks/mastodon.rake @@ -90,9 +90,9 @@ namespace :mastodon do desc 'Set unknown attachment type for remote-only attachments' task set_unknown: :environment do - Rails.logger.debug 'Setting unknown attachment type for remote-only attachments...' + puts 'Setting unknown attachment type for remote-only attachments...' MediaAttachment.where(file_file_name: nil).where.not(type: :unknown).in_batches.update_all(type: :unknown) - Rails.logger.debug 'Done!' + puts 'Done!' end desc 'Redownload avatars/headers of remote users. Optionally limit to a particular domain with DOMAIN' @@ -188,24 +188,24 @@ namespace :mastodon do namespace :maintenance do desc 'Update counter caches' task update_counter_caches: :environment do - Rails.logger.debug 'Updating counter caches for accounts...' + puts 'Updating counter caches for accounts...' - Account.unscoped.select('id').find_in_batches do |batch| + Account.unscoped.where.not(protocol: :activitypub).select('id').find_in_batches do |batch| Account.where(id: batch.map(&:id)).update_all('statuses_count = (select count(*) from statuses where account_id = accounts.id), followers_count = (select count(*) from follows where target_account_id = accounts.id), following_count = (select count(*) from follows where account_id = accounts.id)') end - Rails.logger.debug 'Updating counter caches for statuses...' + puts 'Updating counter caches for statuses...' Status.unscoped.select('id').find_in_batches do |batch| Status.where(id: batch.map(&:id)).update_all('favourites_count = (select count(*) from favourites where favourites.status_id = statuses.id), reblogs_count = (select count(*) from statuses as reblogs where reblogs.reblog_of_id = statuses.id)') end - Rails.logger.debug 'Done!' + puts 'Done!' end desc 'Generate static versions of GIF avatars/headers' task add_static_avatars: :environment do - Rails.logger.debug 'Generating static avatars/headers for GIF ones...' + puts 'Generating static avatars/headers for GIF ones...' Account.unscoped.where(avatar_content_type: 'image/gif').or(Account.unscoped.where(header_content_type: 'image/gif')).find_each do |account| begin @@ -217,7 +217,7 @@ namespace :mastodon do end end - Rails.logger.debug 'Done!' + puts 'Done!' end desc 'Ensure referencial integrity' -- cgit From 1aad015bbbe7957827c2b921a21c53ce11c6ac36 Mon Sep 17 00:00:00 2001 From: abcang Date: Thu, 14 Sep 2017 22:12:43 +0900 Subject: Revert unique retry job (#4937) * Revert "Enable UniqueRetryJobMiddleware even when called from sidekiq worker (#4836)" This reverts commit 6859d4c0289e767955aac3f345074220fe200604. * Revert "Do not execute the job with the same arguments as the retry job (#4814)" This reverts commit be7ffa2d7539d5a1946a3933cb9d242b9fac0ddc. --- app/workers/pubsubhubbub/subscribe_worker.rb | 2 +- config/application.rb | 1 - config/initializers/sidekiq.rb | 6 ------ lib/mastodon/unique_retry_job_middleware.rb | 20 -------------------- 4 files changed, 1 insertion(+), 28 deletions(-) delete mode 100644 lib/mastodon/unique_retry_job_middleware.rb (limited to 'lib') diff --git a/app/workers/pubsubhubbub/subscribe_worker.rb b/app/workers/pubsubhubbub/subscribe_worker.rb index 130c967e0..7560c2671 100644 --- a/app/workers/pubsubhubbub/subscribe_worker.rb +++ b/app/workers/pubsubhubbub/subscribe_worker.rb @@ -3,7 +3,7 @@ class Pubsubhubbub::SubscribeWorker include Sidekiq::Worker - sidekiq_options queue: 'push', retry: 10, unique: :until_executed, dead: false, unique_retry: true + sidekiq_options queue: 'push', retry: 10, unique: :until_executed, dead: false sidekiq_retry_in do |count| case count diff --git a/config/application.rb b/config/application.rb index f98f7af16..b6ce74147 100644 --- a/config/application.rb +++ b/config/application.rb @@ -10,7 +10,6 @@ require_relative '../app/lib/exceptions' require_relative '../lib/paperclip/gif_transcoder' require_relative '../lib/paperclip/video_transcoder' require_relative '../lib/mastodon/version' -require_relative '../lib/mastodon/unique_retry_job_middleware' Dotenv::Railtie.load diff --git a/config/initializers/sidekiq.rb b/config/initializers/sidekiq.rb index 0ee77730e..b70784d79 100644 --- a/config/initializers/sidekiq.rb +++ b/config/initializers/sidekiq.rb @@ -9,14 +9,8 @@ end Sidekiq.configure_server do |config| config.redis = redis_params - config.client_middleware do |chain| - chain.add Mastodon::UniqueRetryJobMiddleware - end end Sidekiq.configure_client do |config| config.redis = redis_params - config.client_middleware do |chain| - chain.add Mastodon::UniqueRetryJobMiddleware - end end diff --git a/lib/mastodon/unique_retry_job_middleware.rb b/lib/mastodon/unique_retry_job_middleware.rb deleted file mode 100644 index 75da8a0c9..000000000 --- a/lib/mastodon/unique_retry_job_middleware.rb +++ /dev/null @@ -1,20 +0,0 @@ -# frozen_string_literal: true - -class Mastodon::UniqueRetryJobMiddleware - def call(_worker_class, item, _queue, _redis_pool) - return if item['unique_retry'] && retried?(item) - yield - end - - private - - def retried?(item) - # Use unique digest key of SidekiqUniqueJobs - unique_key = SidekiqUniqueJobs::UNIQUE_DIGEST_KEY - unique_digest = item[unique_key] - class_name = item['class'] - retries = Sidekiq::RetrySet.new - - retries.any? { |job| job.item['class'] == class_name && job.item[unique_key] == unique_digest } - end -end -- cgit From 54edb4b853522b322922b65989ddfc5fdf928014 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sat, 16 Sep 2017 03:01:45 +0200 Subject: When accessing uncached media attachment, redownload it (#4955) * When accessing uncached media attachment, redownload it * Prevent re-download of rejected media --- app/controllers/media_proxy_controller.rb | 40 ++++++++++++++++++++++ app/models/media_attachment.rb | 10 ++++-- .../rest/media_attachment_serializer.rb | 12 +++++-- config/routes.rb | 2 ++ lib/tasks/mastodon.rake | 1 - 5 files changed, 60 insertions(+), 5 deletions(-) create mode 100644 app/controllers/media_proxy_controller.rb (limited to 'lib') diff --git a/app/controllers/media_proxy_controller.rb b/app/controllers/media_proxy_controller.rb new file mode 100644 index 000000000..6a83cf9dc --- /dev/null +++ b/app/controllers/media_proxy_controller.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +class MediaProxyController < ApplicationController + include RoutingHelper + + def show + RedisLock.acquire(lock_options) do |lock| + if lock.acquired? + @media_attachment = MediaAttachment.remote.find(params[:id]) + redownload! if @media_attachment.needs_redownload? && !reject_media? + end + end + + redirect_to full_asset_url(@media_attachment.file.url(version)) + end + + private + + def redownload! + @media_attachment.file_remote_url = @media_attachment.remote_url + @media_attachment.touch(:created_at) + @media_attachment.save! + end + + def version + if request.path.ends_with?('/small') + :small + else + :original + end + end + + def lock_options + { redis: Redis.current, key: "media_download:#{params[:id]}" } + end + + def reject_media? + DomainBlock.find_by(domain: @media_attachment.account.domain)&.reject_media? + end +end diff --git a/app/models/media_attachment.rb b/app/models/media_attachment.rb index d83ca44f1..d913e7372 100644 --- a/app/models/media_attachment.rb +++ b/app/models/media_attachment.rb @@ -56,15 +56,21 @@ class MediaAttachment < ApplicationRecord validates :account, presence: true - scope :attached, -> { where.not(status_id: nil) } + scope :attached, -> { where.not(status_id: nil) } scope :unattached, -> { where(status_id: nil) } - scope :local, -> { where(remote_url: '') } + scope :local, -> { where(remote_url: '') } + scope :remote, -> { where.not(remote_url: '') } + default_scope { order(id: :asc) } def local? remote_url.blank? end + def needs_redownload? + file.blank? && remote_url.present? + end + def to_param shortcode end diff --git a/app/serializers/rest/media_attachment_serializer.rb b/app/serializers/rest/media_attachment_serializer.rb index 9055b8db4..31189406a 100644 --- a/app/serializers/rest/media_attachment_serializer.rb +++ b/app/serializers/rest/media_attachment_serializer.rb @@ -7,11 +7,19 @@ class REST::MediaAttachmentSerializer < ActiveModel::Serializer :remote_url, :text_url, :meta def url - full_asset_url(object.file.url(:original)) + if object.needs_redownload? + media_proxy_url(object.id, :original) + else + full_asset_url(object.file.url(:original)) + end end def preview_url - full_asset_url(object.file.url(:small)) + if object.needs_redownload? + media_proxy_url(object.id, :small) + else + full_asset_url(object.file.url(:small)) + end end def text_url diff --git a/config/routes.rb b/config/routes.rb index 63e94fb49..bf5428869 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -98,6 +98,8 @@ Rails.application.routes.draw do resources :media, only: [:show] resources :tags, only: [:show] + get '/media_proxy/:id/(*any)', to: 'media_proxy#show', as: :media_proxy + # Remote follow resource :authorize_follow, only: [:show, :create] resource :share, only: [:show, :create] diff --git a/lib/tasks/mastodon.rake b/lib/tasks/mastodon.rake index 2aef752ca..5614ddf48 100644 --- a/lib/tasks/mastodon.rake +++ b/lib/tasks/mastodon.rake @@ -83,7 +83,6 @@ namespace :mastodon do MediaAttachment.where.not(remote_url: '').where('created_at < ?', time_ago).find_each do |media| media.file.destroy - media.type = :unknown media.save end end -- cgit From efec5072304217019bd401c2202c6eb61019db89 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Sat, 16 Sep 2017 03:08:29 +0200 Subject: Bump to 1.6.1 --- lib/mastodon/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/mastodon/version.rb b/lib/mastodon/version.rb index b6fdb10c6..8b692c29d 100644 --- a/lib/mastodon/version.rb +++ b/lib/mastodon/version.rb @@ -13,7 +13,7 @@ module Mastodon end def patch - 0 + 1 end def pre -- cgit