From 5a8c651e8f0252c7135042e79396f782361302d9 Mon Sep 17 00:00:00 2001 From: Christian Schmidt Date: Fri, 3 Mar 2023 21:06:31 +0100 Subject: Only offer translation for supported languages (#23879) --- .../mastodon/components/status_content.jsx | 4 +- app/javascript/mastodon/initial_state.js | 2 - app/lib/translation_service.rb | 4 ++ app/lib/translation_service/deepl.rb | 46 ++++++++++++++++------ app/lib/translation_service/libre_translate.rb | 38 ++++++++++++------ app/models/status.rb | 10 +++++ app/serializers/initial_state_serializer.rb | 1 - app/serializers/rest/status_serializer.rb | 6 ++- app/services/translate_status_service.rb | 2 +- 9 files changed, 82 insertions(+), 31 deletions(-) (limited to 'app') diff --git a/app/javascript/mastodon/components/status_content.jsx b/app/javascript/mastodon/components/status_content.jsx index a1c38171f..f9c9fe079 100644 --- a/app/javascript/mastodon/components/status_content.jsx +++ b/app/javascript/mastodon/components/status_content.jsx @@ -6,7 +6,7 @@ import { Link } from 'react-router-dom'; import classnames from 'classnames'; import PollContainer from 'mastodon/containers/poll_container'; import Icon from 'mastodon/components/icon'; -import { autoPlayGif, languages as preloadedLanguages, translationEnabled } from 'mastodon/initial_state'; +import { autoPlayGif, languages as preloadedLanguages } from 'mastodon/initial_state'; const MAX_HEIGHT = 706; // 22px * 32 (+ 2px padding at the top) @@ -220,7 +220,7 @@ class StatusContent extends React.PureComponent { const hidden = this.props.onExpandedToggle ? !this.props.expanded : this.state.hidden; const renderReadMore = this.props.onClick && status.get('collapsed'); - const renderTranslate = translationEnabled && this.context.identity.signedIn && this.props.onTranslate && ['public', 'unlisted'].includes(status.get('visibility')) && status.get('contentHtml').length > 0 && status.get('language') !== null && intl.locale !== status.get('language'); + const renderTranslate = this.props.onTranslate && status.get('translatable'); const content = { __html: status.get('translation') ? status.getIn(['translation', 'content']) : status.get('contentHtml') }; const spoilerContent = { __html: status.get('spoilerHtml') }; diff --git a/app/javascript/mastodon/initial_state.js b/app/javascript/mastodon/initial_state.js index d04c4a42d..919e0fc28 100644 --- a/app/javascript/mastodon/initial_state.js +++ b/app/javascript/mastodon/initial_state.js @@ -80,7 +80,6 @@ * @property {boolean} use_blurhash * @property {boolean=} use_pending_items * @property {string} version - * @property {boolean} translation_enabled */ /** @@ -132,7 +131,6 @@ export const unfollowModal = getMeta('unfollow_modal'); export const useBlurhash = getMeta('use_blurhash'); export const usePendingItems = getMeta('use_pending_items'); export const version = getMeta('version'); -export const translationEnabled = getMeta('translation_enabled'); export const languages = initialState?.languages; export const statusPageUrl = getMeta('status_page_url'); diff --git a/app/lib/translation_service.rb b/app/lib/translation_service.rb index 285f30939..5ff93674a 100644 --- a/app/lib/translation_service.rb +++ b/app/lib/translation_service.rb @@ -21,6 +21,10 @@ class TranslationService ENV['DEEPL_API_KEY'].present? || ENV['LIBRE_TRANSLATE_ENDPOINT'].present? end + def supported?(_source_language, _target_language) + false + end + def translate(_text, _source_language, _target_language) raise NotImplementedError end diff --git a/app/lib/translation_service/deepl.rb b/app/lib/translation_service/deepl.rb index 151d33d90..deff95a1d 100644 --- a/app/lib/translation_service/deepl.rb +++ b/app/lib/translation_service/deepl.rb @@ -11,33 +11,53 @@ class TranslationService::DeepL < TranslationService end def translate(text, source_language, target_language) - request(text, source_language, target_language).perform do |res| + form = { text: text, source_lang: source_language&.upcase, target_lang: target_language, tag_handling: 'html' } + request(:post, '/v2/translate', form: form) do |res| + transform_response(res.body_with_limit) + end + end + + def supported?(source_language, target_language) + source_language.in?(languages('source')) && target_language.in?(languages('target')) + end + + private + + def languages(type) + Rails.cache.fetch("translation_service/deepl/languages/#{type}", expires_in: 7.days, race_condition_ttl: 1.minute) do + request(:get, "/v2/languages?type=#{type}") do |res| + # In DeepL, EN and PT are deprecated in favor of EN-GB/EN-US and PT-BR/PT-PT, so + # they are supported but not returned by the API. + extra = type == 'source' ? [nil] : %w(en pt) + languages = Oj.load(res.body_with_limit).map { |language| language['language'].downcase } + + languages + extra + end + end + end + + def request(verb, path, **options) + req = Request.new(verb, "#{base_url}#{path}", **options) + req.add_headers(Authorization: "DeepL-Auth-Key #{@api_key}") + req.perform do |res| case res.code when 429 raise TooManyRequestsError when 456 raise QuotaExceededError when 200...300 - transform_response(res.body_with_limit) + yield res else raise UnexpectedResponseError end end end - private - - def request(text, source_language, target_language) - req = Request.new(:post, endpoint_url, form: { text: text, source_lang: source_language&.upcase, target_lang: target_language, tag_handling: 'html' }) - req.add_headers(Authorization: "DeepL-Auth-Key #{@api_key}") - req - end - - def endpoint_url + def base_url if @plan == 'free' - 'https://api-free.deepl.com/v2/translate' + 'https://api-free.deepl.com' else - 'https://api.deepl.com/v2/translate' + 'https://api.deepl.com' end end diff --git a/app/lib/translation_service/libre_translate.rb b/app/lib/translation_service/libre_translate.rb index 4ebe21e45..743e4d77f 100644 --- a/app/lib/translation_service/libre_translate.rb +++ b/app/lib/translation_service/libre_translate.rb @@ -9,29 +9,45 @@ class TranslationService::LibreTranslate < TranslationService end def translate(text, source_language, target_language) - request(text, source_language, target_language).perform do |res| + body = Oj.dump(q: text, source: source_language.presence || 'auto', target: target_language, format: 'html', api_key: @api_key) + request(:post, '/translate', body: body) do |res| + transform_response(res.body_with_limit, source_language) + end + end + + def supported?(source_language, target_language) + languages.key?(source_language) && languages[source_language].include?(target_language) + end + + private + + def languages + Rails.cache.fetch('translation_service/libre_translate/languages', expires_in: 7.days, race_condition_ttl: 1.minute) do + request(:get, '/languages') do |res| + languages = Oj.load(res.body_with_limit).to_h { |language| [language['code'], language['targets']] } + languages[nil] = languages.values.flatten.uniq + languages + end + end + end + + def request(verb, path, **options) + req = Request.new(verb, "#{@base_url}#{path}", allow_local: true, **options) + req.add_headers('Content-Type': 'application/json') + req.perform do |res| case res.code when 429 raise TooManyRequestsError when 403 raise QuotaExceededError when 200...300 - transform_response(res.body_with_limit, source_language) + yield res else raise UnexpectedResponseError end end end - private - - def request(text, source_language, target_language) - body = Oj.dump(q: text, source: source_language.presence || 'auto', target: target_language, format: 'html', api_key: @api_key) - req = Request.new(:post, "#{@base_url}/translate", body: body, allow_local: true) - req.add_headers('Content-Type': 'application/json') - req - end - def transform_response(str, source_language) json = Oj.load(str, mode: :strict) diff --git a/app/models/status.rb b/app/models/status.rb index e7ea191a8..dd7ac2edb 100644 --- a/app/models/status.rb +++ b/app/models/status.rb @@ -232,6 +232,16 @@ class Status < ApplicationRecord public_visibility? || unlisted_visibility? end + def translatable? + translate_target_locale = I18n.locale.to_s.split(/[_-]/).first + + distributable? && + content.present? && + language != translate_target_locale && + TranslationService.configured? && + TranslationService.configured.supported?(language, translate_target_locale) + end + alias sign? distributable? def with_media? diff --git a/app/serializers/initial_state_serializer.rb b/app/serializers/initial_state_serializer.rb index 7905444e9..769ba653e 100644 --- a/app/serializers/initial_state_serializer.rb +++ b/app/serializers/initial_state_serializer.rb @@ -30,7 +30,6 @@ class InitialStateSerializer < ActiveModel::Serializer timeline_preview: Setting.timeline_preview, activity_api_enabled: Setting.activity_api_enabled, single_user_mode: Rails.configuration.x.single_user_mode, - translation_enabled: TranslationService.configured?, trends_as_landing_page: Setting.trends_as_landing_page, status_page_url: Setting.status_page_url, } diff --git a/app/serializers/rest/status_serializer.rb b/app/serializers/rest/status_serializer.rb index e0b8f32a6..a422f5b25 100644 --- a/app/serializers/rest/status_serializer.rb +++ b/app/serializers/rest/status_serializer.rb @@ -4,7 +4,7 @@ class REST::StatusSerializer < ActiveModel::Serializer include FormattingHelper attributes :id, :created_at, :in_reply_to_id, :in_reply_to_account_id, - :sensitive, :spoiler_text, :visibility, :language, + :sensitive, :spoiler_text, :visibility, :language, :translatable, :uri, :url, :replies_count, :reblogs_count, :favourites_count, :edited_at @@ -50,6 +50,10 @@ class REST::StatusSerializer < ActiveModel::Serializer object.account.user_shows_application? || (current_user? && current_user.account_id == object.account_id) end + def translatable + current_user? && object.translatable? + end + def visibility # This visibility is masked behind "private" # to avoid API changes because there are no diff --git a/app/services/translate_status_service.rb b/app/services/translate_status_service.rb index 539a0d9db..92d8b62a0 100644 --- a/app/services/translate_status_service.rb +++ b/app/services/translate_status_service.rb @@ -6,7 +6,7 @@ class TranslateStatusService < BaseService include FormattingHelper def call(status, target_language) - raise Mastodon::NotPermittedError unless status.public_visibility? || status.unlisted_visibility? + raise Mastodon::NotPermittedError unless status.translatable? @status = status @content = status_content_format(@status) -- cgit