diff options
Diffstat (limited to 'app/lib')
28 files changed, 537 insertions, 70 deletions
diff --git a/app/lib/activitypub/activity.rb b/app/lib/activitypub/activity.rb index 7ff06ea39..f4c67cccd 100644 --- a/app/lib/activitypub/activity.rb +++ b/app/lib/activitypub/activity.rb @@ -116,12 +116,12 @@ class ActivityPub::Activity def dereference_object! return unless @object.is_a?(String) - dereferencer = ActivityPub::Dereferencer.new(@object, permitted_origin: @account.uri, signature_account: signed_fetch_account) + dereferencer = ActivityPub::Dereferencer.new(@object, permitted_origin: @account.uri, signature_actor: signed_fetch_actor) @object = dereferencer.object unless dereferencer.object.nil? end - def signed_fetch_account + def signed_fetch_actor return Account.find(@options[:delivered_to_account_id]) if @options[:delivered_to_account_id].present? first_mentioned_local_account || first_local_follower @@ -163,15 +163,15 @@ class ActivityPub::Activity end def followed_by_local_accounts? - @account.passive_relationships.exists? || @options[:relayed_through_account]&.passive_relationships&.exists? + @account.passive_relationships.exists? || (@options[:relayed_through_actor].is_a?(Account) && @options[:relayed_through_actor].passive_relationships&.exists?) end def requested_through_relay? - @options[:relayed_through_account] && Relay.find_by(inbox_url: @options[:relayed_through_account].inbox_url)&.enabled? + @options[:relayed_through_actor] && Relay.find_by(inbox_url: @options[:relayed_through_actor].inbox_url)&.enabled? end def reject_payload! - Rails.logger.info("Rejected #{@json['type']} activity #{@json['id']} from #{@account.uri}#{@options[:relayed_through_account] && "via #{@options[:relayed_through_account].uri}"}") + Rails.logger.info("Rejected #{@json['type']} activity #{@json['id']} from #{@account.uri}#{@options[:relayed_through_actor] && "via #{@options[:relayed_through_actor].uri}"}") nil end end diff --git a/app/lib/activitypub/activity/add.rb b/app/lib/activitypub/activity/add.rb index 845eeaef7..9e2483983 100644 --- a/app/lib/activitypub/activity/add.rb +++ b/app/lib/activitypub/activity/add.rb @@ -2,12 +2,32 @@ class ActivityPub::Activity::Add < ActivityPub::Activity def perform - return unless @json['target'].present? && value_or_id(@json['target']) == @account.featured_collection_url + return if @json['target'].blank? + case value_or_id(@json['target']) + when @account.featured_collection_url + case @object['type'] + when 'Hashtag' + add_featured_tags + else + add_featured + end + end + end + + private + + def add_featured status = status_from_object return unless !status.nil? && status.account_id == @account.id && !@account.pinned?(status) StatusPin.create!(account: @account, status: status) end + + def add_featured_tags + name = @object['name']&.delete_prefix('#') + + FeaturedTag.create!(account: @account, name: name) if name.present? + end end diff --git a/app/lib/activitypub/activity/remove.rb b/app/lib/activitypub/activity/remove.rb index f523ead9f..f5cbef675 100644 --- a/app/lib/activitypub/activity/remove.rb +++ b/app/lib/activitypub/activity/remove.rb @@ -2,8 +2,22 @@ class ActivityPub::Activity::Remove < ActivityPub::Activity def perform - return unless @json['target'].present? && value_or_id(@json['target']) == @account.featured_collection_url + return if @json['target'].blank? + case value_or_id(@json['target']) + when @account.featured_collection_url + case @object['type'] + when 'Hashtag' + remove_featured_tags + else + remove_featured + end + end + end + + private + + def remove_featured status = status_from_uri(object_uri) return unless !status.nil? && status.account_id == @account.id @@ -11,4 +25,13 @@ class ActivityPub::Activity::Remove < ActivityPub::Activity pin = StatusPin.find_by(account: @account, status: status) pin&.destroy! end + + def remove_featured_tags + name = @object['name']&.delete_prefix('#') + + return if name.blank? + + featured_tag = FeaturedTag.by_name(name).find_by(account: @account) + featured_tag&.destroy! + end end diff --git a/app/lib/activitypub/dereferencer.rb b/app/lib/activitypub/dereferencer.rb index bea69608f..4d7756d71 100644 --- a/app/lib/activitypub/dereferencer.rb +++ b/app/lib/activitypub/dereferencer.rb @@ -3,10 +3,10 @@ class ActivityPub::Dereferencer include JsonLdHelper - def initialize(uri, permitted_origin: nil, signature_account: nil) + def initialize(uri, permitted_origin: nil, signature_actor: nil) @uri = uri @permitted_origin = permitted_origin - @signature_account = signature_account + @signature_actor = signature_actor end def object @@ -46,7 +46,7 @@ class ActivityPub::Dereferencer req.add_headers('Accept' => 'application/activity+json, application/ld+json') req.add_headers(headers) if headers - req.on_behalf_of(@signature_account) if @signature_account + req.on_behalf_of(@signature_actor) if @signature_actor req.perform do |res| if res.code == 200 diff --git a/app/lib/activitypub/linked_data_signature.rb b/app/lib/activitypub/linked_data_signature.rb index e853a970e..f90adaf6c 100644 --- a/app/lib/activitypub/linked_data_signature.rb +++ b/app/lib/activitypub/linked_data_signature.rb @@ -9,7 +9,7 @@ class ActivityPub::LinkedDataSignature @json = json.with_indifferent_access end - def verify_account! + def verify_actor! return unless @json['signature'].is_a?(Hash) type = @json['signature']['type'] @@ -18,7 +18,7 @@ class ActivityPub::LinkedDataSignature return unless type == 'RsaSignature2017' - creator = ActivityPub::TagManager.instance.uri_to_resource(creator_uri, Account) + creator = ActivityPub::TagManager.instance.uri_to_actor(creator_uri) creator ||= ActivityPub::FetchRemoteKeyService.new.call(creator_uri, id: false) return if creator.nil? @@ -35,7 +35,7 @@ class ActivityPub::LinkedDataSignature def sign!(creator, sign_with: nil) options = { 'type' => 'RsaSignature2017', - 'creator' => [ActivityPub::TagManager.instance.uri_for(creator), '#main-key'].join, + 'creator' => ActivityPub::TagManager.instance.key_uri_for(creator), 'created' => Time.now.utc.iso8601, } diff --git a/app/lib/activitypub/tag_manager.rb b/app/lib/activitypub/tag_manager.rb index f6b9741fa..3d6b28ef5 100644 --- a/app/lib/activitypub/tag_manager.rb +++ b/app/lib/activitypub/tag_manager.rb @@ -44,6 +44,10 @@ class ActivityPub::TagManager end end + def key_uri_for(target) + [uri_for(target), '#main-key'].join + end + def uri_for_username(username) account_url(username: username) end @@ -155,6 +159,10 @@ class ActivityPub::TagManager path_params[param] end + def uri_to_actor(uri) + uri_to_resource(uri, Account) + end + def uri_to_resource(uri, klass) return if uri.nil? diff --git a/app/lib/feed_manager.rb b/app/lib/feed_manager.rb index c607223fc..9fe9ec346 100644 --- a/app/lib/feed_manager.rb +++ b/app/lib/feed_manager.rb @@ -278,7 +278,7 @@ class FeedManager next if last_status_score < oldest_home_score end - statuses = target_account.statuses.where(visibility: [:public, :unlisted, :private]).includes(:preloadable_poll, :media_attachments, reblog: :account).limit(limit) + statuses = target_account.statuses.where(visibility: [:public, :unlisted, :private]).includes(:preloadable_poll, :media_attachments, :account, reblog: :account).limit(limit) crutches = build_crutches(account.id, statuses) statuses.each do |status| @@ -403,6 +403,7 @@ class FeedManager def filter_from_home?(status, receiver_id, crutches) return false if receiver_id == status.account_id return true if status.reply? && (status.in_reply_to_id.nil? || status.in_reply_to_account_id.nil?) + return true if crutches[:languages][status.account_id].present? && status.language.present? && !crutches[:languages][status.account_id].include?(status.language) check_for_blocks = crutches[:active_mentions][status.id] || [] check_for_blocks.concat([status.account_id]) @@ -483,7 +484,7 @@ class FeedManager # @param [Hash] crutches # @return [Boolean] def filter_from_tags?(status, receiver_id, crutches) - receiver_id != status.account_id && (((crutches[:active_mentions][status.id] || []) + [status.account_id]).any? { |target_account_id| crutches[:blocking][target_account_id] || crutches[:muting][target_account_id] } || crutches[:blocked_by][status.account_id] || crutches[:domain_blocking][status.account.domain]) + receiver_id == status.account_id || ((crutches[:active_mentions][status.id] || []) + [status.account_id]).any? { |target_account_id| crutches[:blocking][target_account_id] || crutches[:muting][target_account_id] } || crutches[:blocked_by][status.account_id] || crutches[:domain_blocking][status.account.domain] end # Adds a status to an account's feed, returning true if a status was @@ -600,10 +601,11 @@ class FeedManager end crutches[:following] = Follow.where(account_id: receiver_id, target_account_id: statuses.map(&:in_reply_to_account_id).compact).pluck(:target_account_id).index_with(true) + crutches[:languages] = Follow.where(account_id: receiver_id, target_account_id: statuses.map(&:account_id)).pluck(:target_account_id, :languages).to_h crutches[:hiding_reblogs] = Follow.where(account_id: receiver_id, target_account_id: statuses.map { |s| s.account_id if s.reblog? }.compact, show_reblogs: false).pluck(:target_account_id).index_with(true) crutches[:blocking] = Block.where(account_id: receiver_id, target_account_id: check_for_blocks).pluck(:target_account_id).index_with(true) crutches[:muting] = Mute.where(account_id: receiver_id, target_account_id: check_for_blocks).pluck(:target_account_id).index_with(true) - crutches[:domain_blocking] = AccountDomainBlock.where(account_id: receiver_id, domain: statuses.map { |s| s.reblog&.account&.domain }.compact).pluck(:domain).index_with(true) + crutches[:domain_blocking] = AccountDomainBlock.where(account_id: receiver_id, domain: statuses.flat_map { |s| [s.account.domain, s.reblog&.account&.domain] }.compact).pluck(:domain).index_with(true) crutches[:blocked_by] = Block.where(target_account_id: receiver_id, account_id: statuses.map { |s| s.reblog&.account_id }.compact).pluck(:account_id).index_with(true) crutches diff --git a/app/lib/hashtag_normalizer.rb b/app/lib/hashtag_normalizer.rb index c1f99e163..49fa6101d 100644 --- a/app/lib/hashtag_normalizer.rb +++ b/app/lib/hashtag_normalizer.rb @@ -8,7 +8,7 @@ class HashtagNormalizer private def remove_invalid_characters(str) - str.gsub(/[^[:alnum:]#{Tag::HASHTAG_SEPARATORS}]/, '') + str.gsub(Tag::HASHTAG_INVALID_CHARS_RE, '') end def ascii_folding(str) diff --git a/app/lib/inline_renderer.rb b/app/lib/inline_renderer.rb index b70814748..4bb240b48 100644 --- a/app/lib/inline_renderer.rb +++ b/app/lib/inline_renderer.rb @@ -11,6 +11,7 @@ class InlineRenderer case @template when :status serializer = REST::StatusSerializer + preload_associations_for_status when :notification serializer = REST::NotificationSerializer when :conversation @@ -35,6 +36,16 @@ class InlineRenderer private + def preload_associations_for_status + ActiveRecord::Associations::Preloader.new.preload(@object, { + active_mentions: :account, + + reblog: { + active_mentions: :account, + }, + }) + end + def current_user @current_account&.user end diff --git a/app/lib/permalink_redirector.rb b/app/lib/permalink_redirector.rb index e48bce060..cf1a37625 100644 --- a/app/lib/permalink_redirector.rb +++ b/app/lib/permalink_redirector.rb @@ -8,20 +8,14 @@ class PermalinkRedirector end def redirect_path - if path_segments[0] == 'web' - if path_segments[1].present? && path_segments[1].start_with?('@') && path_segments[2] =~ /\d/ - find_status_url_by_id(path_segments[2]) - elsif path_segments[1].present? && path_segments[1].start_with?('@') - find_account_url_by_name(path_segments[1]) - elsif path_segments[1] == 'statuses' && path_segments[2] =~ /\d/ - find_status_url_by_id(path_segments[2]) - elsif path_segments[1] == 'accounts' && path_segments[2] =~ /\d/ - find_account_url_by_id(path_segments[2]) - elsif path_segments[1] == 'timelines' && path_segments[2] == 'tag' && path_segments[3].present? - find_tag_url_by_name(path_segments[3]) - elsif path_segments[1] == 'tags' && path_segments[2].present? - find_tag_url_by_name(path_segments[2]) - end + if path_segments[0].present? && path_segments[0].start_with?('@') && path_segments[1] =~ /\d/ + find_status_url_by_id(path_segments[1]) + elsif path_segments[0].present? && path_segments[0].start_with?('@') + find_account_url_by_name(path_segments[0]) + elsif path_segments[0] == 'statuses' && path_segments[1] =~ /\d/ + find_status_url_by_id(path_segments[1]) + elsif path_segments[0] == 'accounts' && path_segments[1] =~ /\d/ + find_account_url_by_id(path_segments[1]) end end @@ -33,18 +27,12 @@ class PermalinkRedirector def find_status_url_by_id(id) status = Status.find_by(id: id) - - return unless status&.distributable? - - ActivityPub::TagManager.instance.url_for(status) + ActivityPub::TagManager.instance.url_for(status) if status&.distributable? && !status.account.local? end def find_account_url_by_id(id) account = Account.find_by(id: id) - - return unless account - - ActivityPub::TagManager.instance.url_for(account) + ActivityPub::TagManager.instance.url_for(account) if account.present? && !account.local? end def find_account_url_by_name(name) @@ -52,12 +40,6 @@ class PermalinkRedirector domain = nil if TagManager.instance.local_domain?(domain) account = Account.find_remote(username, domain) - return unless account - - ActivityPub::TagManager.instance.url_for(account) - end - - def find_tag_url_by_name(name) - tag_path(CGI.unescape(name)) + ActivityPub::TagManager.instance.url_for(account) if account.present? && !account.local? end end diff --git a/app/lib/redis_configuration.rb b/app/lib/redis_configuration.rb index e14d6c8b6..f0e86d985 100644 --- a/app/lib/redis_configuration.rb +++ b/app/lib/redis_configuration.rb @@ -7,9 +7,7 @@ class RedisConfiguration @pool = ConnectionPool.new(size: new_pool_size) { new.connection } end - def with - pool.with { |redis| yield redis } - end + delegate :with, to: :pool def pool @pool ||= establish_pool(pool_size) @@ -17,7 +15,7 @@ class RedisConfiguration def pool_size if Sidekiq.server? - Sidekiq.options[:concurrency] + Sidekiq[:concurrency] else ENV['MAX_THREADS'] || 5 end diff --git a/app/lib/request.rb b/app/lib/request.rb index f5123d776..dd198f399 100644 --- a/app/lib/request.rb +++ b/app/lib/request.rb @@ -40,12 +40,11 @@ class Request set_digest! if options.key?(:body) end - def on_behalf_of(account, key_id_format = :uri, sign_with: nil) - raise ArgumentError, 'account must not be nil' if account.nil? + def on_behalf_of(actor, sign_with: nil) + raise ArgumentError, 'actor must not be nil' if actor.nil? - @account = account - @keypair = sign_with.present? ? OpenSSL::PKey::RSA.new(sign_with) : @account.keypair - @key_id_format = key_id_format + @actor = actor + @keypair = sign_with.present? ? OpenSSL::PKey::RSA.new(sign_with) : @actor.keypair self end @@ -63,8 +62,6 @@ class Request end begin - response = response.extend(ClientLimit) - # If we are using a persistent connection, we have to # read every response to be able to move forward at all. # However, simply calling #to_s or #flush may not be safe, @@ -79,7 +76,7 @@ class Request end def headers - (@account ? @headers.merge('Signature' => signature) : @headers).without(REQUEST_TARGET) + (@actor ? @headers.merge('Signature' => signature) : @headers).without(REQUEST_TARGET) end class << self @@ -128,12 +125,7 @@ class Request end def key_id - case @key_id_format - when :acct - @account.to_webfinger_s - when :uri - [ActivityPub::TagManager.instance.uri_for(@account), '#main-key'].join - end + ActivityPub::TagManager.instance.key_uri_for(@actor) end def http_client @@ -187,6 +179,14 @@ class Request end end + if ::HTTP::Response.methods.include?(:body_with_limit) && !Rails.env.production? + abort 'HTTP::Response#body_with_limit is already defined, the monkey patch will not be applied' + else + class ::HTTP::Response + include Request::ClientLimit + end + end + class Socket < TCPSocket class << self def open(host, *args) @@ -199,7 +199,8 @@ class Request rescue IPAddr::InvalidAddressError Resolv::DNS.open do |dns| dns.timeouts = 5 - addresses = dns.getaddresses(host).take(2) + addresses = dns.getaddresses(host) + addresses = addresses.filter { |addr| addr.is_a?(Resolv::IPv6) }.take(2) + addresses.filter { |addr| !addr.is_a?(Resolv::IPv6) }.take(2) end end @@ -208,7 +209,7 @@ class Request addresses.each do |address| begin - check_private_address(address) + check_private_address(address, host) sock = ::Socket.new(address.is_a?(Resolv::IPv6) ? ::Socket::AF_INET6 : ::Socket::AF_INET, ::Socket::SOCK_STREAM, 0) sockaddr = ::Socket.pack_sockaddr_in(port, address.to_s) @@ -264,10 +265,10 @@ class Request alias new open - def check_private_address(address) + def check_private_address(address, host) addr = IPAddr.new(address.to_s) return if private_address_exceptions.any? { |range| range.include?(addr) } - raise Mastodon::HostValidationError if PrivateAddressCheck.private_address?(addr) + raise Mastodon::PrivateNetworkAddressError, host if PrivateAddressCheck.private_address?(addr) end def private_address_exceptions diff --git a/app/lib/status_cache_hydrator.rb b/app/lib/status_cache_hydrator.rb new file mode 100644 index 000000000..a84d25694 --- /dev/null +++ b/app/lib/status_cache_hydrator.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +class StatusCacheHydrator + def initialize(status) + @status = status + end + + def hydrate(account_id) + # The cache of the serialized hash is generated by the fan-out-on-write service + payload = Rails.cache.fetch("fan-out/#{@status.id}") { InlineRenderer.render(@status, nil, :status) } + + # If we're delivering to the author who disabled the display of the application used to create the + # status, we need to hydrate the application, since it was not rendered for the basic payload + payload[:application] = @status.application.present? ? ActiveModelSerializers::SerializableResource.new(@status.application, serializer: REST::StatusSerializer::ApplicationSerializer).as_json : nil if payload[:application].nil? && @status.account_id == account_id + + # We take advantage of the fact that some relationships can only occur with an original status, not + # the reblog that wraps it, so we can assume that some values are always false + if payload[:reblog] + payload[:muted] = false + payload[:bookmarked] = false + payload[:pinned] = false if @status.account_id == account_id + payload[:filtered] = CustomFilter.apply_cached_filters(CustomFilter.cached_filters_for(account_id), @status.reblog).map { |filter| ActiveModelSerializers::SerializableResource.new(filter, serializer: REST::FilterResultSerializer).as_json } + + # If the reblogged status is being delivered to the author who disabled the display of the application + # used to create the status, we need to hydrate it here too + payload[:reblog][:application] = @status.reblog.application.present? ? ActiveModelSerializers::SerializableResource.new(@status.reblog.application, serializer: REST::StatusSerializer::ApplicationSerializer).as_json : nil if payload[:reblog][:application].nil? && @status.reblog.account_id == account_id + + payload[:reblog][:favourited] = Favourite.where(account_id: account_id, status_id: @status.reblog_of_id).exists? + payload[:reblog][:reblogged] = Status.where(account_id: account_id, reblog_of_id: @status.reblog_of_id).exists? + payload[:reblog][:muted] = ConversationMute.where(account_id: account_id, conversation_id: @status.reblog.conversation_id).exists? + payload[:reblog][:bookmarked] = Bookmark.where(account_id: account_id, status_id: @status.reblog_of_id).exists? + payload[:reblog][:pinned] = StatusPin.where(account_id: account_id, status_id: @status.reblog_of_id).exists? if @status.reblog.account_id == account_id + payload[:reblog][:filtered] = payload[:filtered] + + if payload[:reblog][:poll] + if @status.reblog.account_id == account_id + payload[:reblog][:poll][:voted] = true + payload[:reblog][:poll][:own_votes] = [] + else + own_votes = PollVote.where(poll_id: @status.reblog.poll_id, account_id: account_id).pluck(:choice) + payload[:reblog][:poll][:voted] = !own_votes.empty? + payload[:reblog][:poll][:own_votes] = own_votes + end + end + + payload[:favourited] = payload[:reblog][:favourited] + payload[:reblogged] = payload[:reblog][:reblogged] + else + payload[:favourited] = Favourite.where(account_id: account_id, status_id: @status.id).exists? + payload[:reblogged] = Status.where(account_id: account_id, reblog_of_id: @status.id).exists? + payload[:muted] = ConversationMute.where(account_id: account_id, conversation_id: @status.conversation_id).exists? + payload[:bookmarked] = Bookmark.where(account_id: account_id, status_id: @status.id).exists? + payload[:pinned] = StatusPin.where(account_id: account_id, status_id: @status.id).exists? if @status.account_id == account_id + payload[:filtered] = CustomFilter.apply_cached_filters(CustomFilter.cached_filters_for(account_id), @status).map { |filter| ActiveModelSerializers::SerializableResource.new(filter, serializer: REST::FilterResultSerializer).as_json } + + if payload[:poll] + payload[:poll][:voted] = @status.account_id == account_id + payload[:poll][:own_votes] = [] + end + end + + payload + end +end diff --git a/app/lib/status_reach_finder.rb b/app/lib/status_reach_finder.rb index 98e502bb6..ccf1e9e3a 100644 --- a/app/lib/status_reach_finder.rb +++ b/app/lib/status_reach_finder.rb @@ -55,7 +55,7 @@ class StatusReachFinder # Beware: Reblogs can be created without the author having had access to the status def reblogs_account_ids - @status.reblogs.pluck(:account_id) if distributable? || unsafe? + @status.reblogs.rewhere(deleted_at: [nil, @status.deleted_at]).pluck(:account_id) if distributable? || unsafe? end # Beware: Favourites can be created without the author having had access to the status diff --git a/app/lib/translation_service.rb b/app/lib/translation_service.rb new file mode 100644 index 000000000..285f30939 --- /dev/null +++ b/app/lib/translation_service.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +class TranslationService + class Error < StandardError; end + class NotConfiguredError < Error; end + class TooManyRequestsError < Error; end + class QuotaExceededError < Error; end + class UnexpectedResponseError < Error; end + + def self.configured + if ENV['DEEPL_API_KEY'].present? + TranslationService::DeepL.new(ENV.fetch('DEEPL_PLAN', 'free'), ENV['DEEPL_API_KEY']) + elsif ENV['LIBRE_TRANSLATE_ENDPOINT'].present? + TranslationService::LibreTranslate.new(ENV['LIBRE_TRANSLATE_ENDPOINT'], ENV['LIBRE_TRANSLATE_API_KEY']) + else + raise NotConfiguredError + end + end + + def self.configured? + ENV['DEEPL_API_KEY'].present? || ENV['LIBRE_TRANSLATE_ENDPOINT'].present? + end + + def translate(_text, _source_language, _target_language) + raise NotImplementedError + end +end diff --git a/app/lib/translation_service/deepl.rb b/app/lib/translation_service/deepl.rb new file mode 100644 index 000000000..537fd24c0 --- /dev/null +++ b/app/lib/translation_service/deepl.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +class TranslationService::DeepL < TranslationService + include JsonLdHelper + + def initialize(plan, api_key) + super() + + @plan = plan + @api_key = api_key + end + + def translate(text, source_language, target_language) + request(text, source_language, target_language).perform do |res| + case res.code + when 429 + raise TooManyRequestsError + when 456 + raise QuotaExceededError + when 200...300 + transform_response(res.body_with_limit) + else + raise UnexpectedResponseError + end + end + end + + private + + def request(text, source_language, target_language) + req = Request.new(:post, endpoint_url, form: { text: text, source_lang: source_language&.upcase, target_lang: target_language, tag_handling: 'html' }) + req.add_headers('Authorization': "DeepL-Auth-Key #{@api_key}") + req + end + + def endpoint_url + if @plan == 'free' + 'https://api-free.deepl.com/v2/translate' + else + 'https://api.deepl.com/v2/translate' + end + end + + def transform_response(str) + json = Oj.load(str, mode: :strict) + + raise UnexpectedResponseError unless json.is_a?(Hash) + + Translation.new(text: json.dig('translations', 0, 'text'), detected_source_language: json.dig('translations', 0, 'detected_source_language')&.downcase, provider: 'DeepL.com') + rescue Oj::ParseError + raise UnexpectedResponseError + end +end diff --git a/app/lib/translation_service/libre_translate.rb b/app/lib/translation_service/libre_translate.rb new file mode 100644 index 000000000..43576e306 --- /dev/null +++ b/app/lib/translation_service/libre_translate.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +class TranslationService::LibreTranslate < TranslationService + def initialize(base_url, api_key) + super() + + @base_url = base_url + @api_key = api_key + end + + def translate(text, source_language, target_language) + request(text, source_language, target_language).perform do |res| + case res.code + when 429 + raise TooManyRequestsError + when 403 + raise QuotaExceededError + when 200...300 + transform_response(res.body_with_limit, source_language) + else + raise UnexpectedResponseError + end + end + end + + private + + def request(text, source_language, target_language) + body = Oj.dump(q: text, source: source_language.presence || 'auto', target: target_language, format: 'html', api_key: @api_key) + req = Request.new(:post, "#{@base_url}/translate", body: body) + req.add_headers('Content-Type': 'application/json') + req + end + + def transform_response(str, source_language) + json = Oj.load(str, mode: :strict) + + raise UnexpectedResponseError unless json.is_a?(Hash) + + Translation.new(text: json['translatedText'], detected_source_language: source_language, provider: 'LibreTranslate') + rescue Oj::ParseError + raise UnexpectedResponseError + end +end diff --git a/app/lib/translation_service/translation.rb b/app/lib/translation_service/translation.rb new file mode 100644 index 000000000..19318c7e9 --- /dev/null +++ b/app/lib/translation_service/translation.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +class TranslationService::Translation < ActiveModelSerializers::Model + attributes :text, :detected_source_language, :provider +end diff --git a/app/lib/user_settings_decorator.rb b/app/lib/user_settings_decorator.rb index 1d70ed36a..260077a1c 100644 --- a/app/lib/user_settings_decorator.rb +++ b/app/lib/user_settings_decorator.rb @@ -32,6 +32,7 @@ class UserSettingsDecorator user.settings['system_font_ui'] = system_font_ui_preference if change?('setting_system_font_ui') user.settings['system_emoji_font'] = system_emoji_font_preference if change?('setting_system_emoji_font') user.settings['noindex'] = noindex_preference if change?('setting_noindex') + user.settings['hide_followers_count'] = hide_followers_count_preference if change?('setting_hide_followers_count') user.settings['flavour'] = flavour_preference if change?('setting_flavour') user.settings['skin'] = skin_preference if change?('setting_skin') user.settings['aggregate_reblogs'] = aggregate_reblogs_preference if change?('setting_aggregate_reblogs') @@ -117,6 +118,10 @@ class UserSettingsDecorator settings['setting_skin'] end + def hide_followers_count_preference + boolean_cast_setting 'setting_hide_followers_count' + end + def show_application_preference boolean_cast_setting 'setting_show_application' end diff --git a/app/lib/vacuum.rb b/app/lib/vacuum.rb new file mode 100644 index 000000000..9db1ec90b --- /dev/null +++ b/app/lib/vacuum.rb @@ -0,0 +1,3 @@ +# frozen_string_literal: true + +module Vacuum; end diff --git a/app/lib/vacuum/access_tokens_vacuum.rb b/app/lib/vacuum/access_tokens_vacuum.rb new file mode 100644 index 000000000..4f3878027 --- /dev/null +++ b/app/lib/vacuum/access_tokens_vacuum.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +class Vacuum::AccessTokensVacuum + def perform + vacuum_revoked_access_tokens! + vacuum_revoked_access_grants! + end + + private + + def vacuum_revoked_access_tokens! + Doorkeeper::AccessToken.where('revoked_at IS NOT NULL').where('revoked_at < NOW()').delete_all + end + + def vacuum_revoked_access_grants! + Doorkeeper::AccessGrant.where('revoked_at IS NOT NULL').where('revoked_at < NOW()').delete_all + end +end diff --git a/app/lib/vacuum/backups_vacuum.rb b/app/lib/vacuum/backups_vacuum.rb new file mode 100644 index 000000000..3b83072f3 --- /dev/null +++ b/app/lib/vacuum/backups_vacuum.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +class Vacuum::BackupsVacuum + def initialize(retention_period) + @retention_period = retention_period + end + + def perform + vacuum_expired_backups! if retention_period? + end + + private + + def vacuum_expired_backups! + backups_past_retention_period.in_batches.destroy_all + end + + def backups_past_retention_period + Backup.unscoped.where(Backup.arel_table[:created_at].lt(@retention_period.ago)) + end + + def retention_period? + @retention_period.present? + end +end diff --git a/app/lib/vacuum/feeds_vacuum.rb b/app/lib/vacuum/feeds_vacuum.rb new file mode 100644 index 000000000..00b9fd646 --- /dev/null +++ b/app/lib/vacuum/feeds_vacuum.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +class Vacuum::FeedsVacuum + def perform + vacuum_inactive_home_feeds! + vacuum_inactive_list_feeds! + vacuum_inactive_direct_feeds! + end + + private + + def vacuum_inactive_home_feeds! + inactive_users.select(:id, :account_id).find_in_batches do |users| + feed_manager.clean_feeds!(:home, users.map(&:account_id)) + end + end + + def vacuum_inactive_list_feeds! + inactive_users_lists.select(:id).find_in_batches do |lists| + feed_manager.clean_feeds!(:list, lists.map(&:id)) + end + end + + def vacuum_inactive_direct_feeds! + inactive_users_lists.select(:id).find_in_batches do |lists| + feed_manager.clean_feeds!(:direct, lists.map(&:id)) + end + end + + def inactive_users + User.confirmed.inactive + end + + def inactive_users_lists + List.where(account_id: inactive_users.select(:account_id)) + end + + def feed_manager + FeedManager.instance + end +end diff --git a/app/lib/vacuum/media_attachments_vacuum.rb b/app/lib/vacuum/media_attachments_vacuum.rb new file mode 100644 index 000000000..7c0a85a9d --- /dev/null +++ b/app/lib/vacuum/media_attachments_vacuum.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +class Vacuum::MediaAttachmentsVacuum + TTL = 1.day.freeze + + def initialize(retention_period) + @retention_period = retention_period + end + + def perform + vacuum_orphaned_records! + vacuum_cached_files! if retention_period? + end + + private + + def vacuum_cached_files! + media_attachments_past_retention_period.find_each do |media_attachment| + media_attachment.file.destroy + media_attachment.thumbnail.destroy + media_attachment.save + end + end + + def vacuum_orphaned_records! + orphaned_media_attachments.in_batches.destroy_all + end + + def media_attachments_past_retention_period + MediaAttachment.unscoped.remote.cached.where(MediaAttachment.arel_table[:created_at].lt(@retention_period.ago)).where(MediaAttachment.arel_table[:updated_at].lt(@retention_period.ago)) + end + + def orphaned_media_attachments + MediaAttachment.unscoped.unattached.where(MediaAttachment.arel_table[:created_at].lt(TTL.ago)) + end + + def retention_period? + @retention_period.present? + end +end diff --git a/app/lib/vacuum/preview_cards_vacuum.rb b/app/lib/vacuum/preview_cards_vacuum.rb new file mode 100644 index 000000000..14fdeda1c --- /dev/null +++ b/app/lib/vacuum/preview_cards_vacuum.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +class Vacuum::PreviewCardsVacuum + TTL = 1.day.freeze + + def initialize(retention_period) + @retention_period = retention_period + end + + def perform + vacuum_cached_images! if retention_period? + end + + private + + def vacuum_cached_images! + preview_cards_past_retention_period.find_each do |preview_card| + preview_card.image.destroy + preview_card.save + end + end + + def preview_cards_past_retention_period + PreviewCard.cached.where(PreviewCard.arel_table[:updated_at].lt(@retention_period.ago)) + end + + def retention_period? + @retention_period.present? + end +end diff --git a/app/lib/vacuum/statuses_vacuum.rb b/app/lib/vacuum/statuses_vacuum.rb new file mode 100644 index 000000000..41d6ba270 --- /dev/null +++ b/app/lib/vacuum/statuses_vacuum.rb @@ -0,0 +1,54 @@ +# frozen_string_literal: true + +class Vacuum::StatusesVacuum + include Redisable + + def initialize(retention_period) + @retention_period = retention_period + end + + def perform + vacuum_statuses! if retention_period? + end + + private + + def vacuum_statuses! + statuses_scope.find_in_batches do |statuses| + # Side-effects not covered by foreign keys, such + # as the search index, must be handled first. + + remove_from_account_conversations(statuses) + remove_from_search_index(statuses) + + # Foreign keys take care of most associated records + # for us. Media attachments will be orphaned. + + Status.where(id: statuses.map(&:id)).delete_all + end + end + + def statuses_scope + Status.unscoped.kept.where(account: Account.remote).where(Status.arel_table[:id].lt(retention_period_as_id)).select(:id, :visibility) + end + + def retention_period_as_id + Mastodon::Snowflake.id_at(@retention_period.ago, with_random: false) + end + + def analyze_statuses! + ActiveRecord::Base.connection.execute('ANALYZE statuses') + end + + def remove_from_account_conversations(statuses) + Status.where(id: statuses.select(&:direct_visibility?).map(&:id)).includes(:account, mentions: :account).each(&:unlink_from_conversations) + end + + def remove_from_search_index(statuses) + with_redis { |redis| redis.sadd('chewy:queue:StatusesIndex', statuses.map(&:id)) } if Chewy.enabled? + end + + def retention_period? + @retention_period.present? + end +end diff --git a/app/lib/vacuum/system_keys_vacuum.rb b/app/lib/vacuum/system_keys_vacuum.rb new file mode 100644 index 000000000..ceee2fd16 --- /dev/null +++ b/app/lib/vacuum/system_keys_vacuum.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class Vacuum::SystemKeysVacuum + def perform + vacuum_expired_system_keys! + end + + private + + def vacuum_expired_system_keys! + SystemKey.expired.delete_all + end +end diff --git a/app/lib/webfinger.rb b/app/lib/webfinger.rb index a681e0815..7c0c10c33 100644 --- a/app/lib/webfinger.rb +++ b/app/lib/webfinger.rb @@ -3,7 +3,7 @@ class Webfinger class Error < StandardError; end class GoneError < Error; end - class RedirectError < StandardError; end + class RedirectError < Error; end class Response attr_reader :uri |