From 634368c491c8e89d93b54dd2dc170f0873beafcb Mon Sep 17 00:00:00 2001 From: Nick Schonning Date: Fri, 17 Feb 2023 21:23:49 -0500 Subject: Autofix Rubocop Lint/SymbolConversion (#23683) --- app/lib/translation_service/deepl.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'app/lib/translation_service') diff --git a/app/lib/translation_service/deepl.rb b/app/lib/translation_service/deepl.rb index 537fd24c0..151d33d90 100644 --- a/app/lib/translation_service/deepl.rb +++ b/app/lib/translation_service/deepl.rb @@ -29,7 +29,7 @@ class TranslationService::DeepL < TranslationService def request(text, source_language, target_language) req = Request.new(:post, endpoint_url, form: { text: text, source_lang: source_language&.upcase, target_lang: target_language, tag_handling: 'html' }) - req.add_headers('Authorization': "DeepL-Auth-Key #{@api_key}") + req.add_headers(Authorization: "DeepL-Auth-Key #{@api_key}") req end -- cgit From 5a8c651e8f0252c7135042e79396f782361302d9 Mon Sep 17 00:00:00 2001 From: Christian Schmidt Date: Fri, 3 Mar 2023 21:06:31 +0100 Subject: Only offer translation for supported languages (#23879) --- .rubocop.yml | 4 + .../mastodon/components/status_content.jsx | 4 +- app/javascript/mastodon/initial_state.js | 2 - app/lib/translation_service.rb | 4 + app/lib/translation_service/deepl.rb | 46 +++++++--- app/lib/translation_service/libre_translate.rb | 38 +++++--- app/models/status.rb | 10 +++ app/serializers/initial_state_serializer.rb | 1 - app/serializers/rest/status_serializer.rb | 6 +- app/services/translate_status_service.rb | 2 +- spec/lib/translation_service/deepl_spec.rb | 100 +++++++++++++++++++++ .../translation_service/libre_translate_spec.rb | 71 +++++++++++++++ spec/models/status_spec.rb | 79 ++++++++++++++++ 13 files changed, 336 insertions(+), 31 deletions(-) create mode 100644 spec/lib/translation_service/deepl_spec.rb create mode 100644 spec/lib/translation_service/libre_translate_spec.rb (limited to 'app/lib/translation_service') diff --git a/.rubocop.yml b/.rubocop.yml index 27d778edf..0a41c54b9 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -97,6 +97,10 @@ Rails/Exit: - 'lib/mastodon/cli_helper.rb' - 'lib/cli.rb' +RSpec/FilePath: + CustomTransform: + DeepL: deepl + RSpec/NotToNot: EnforcedStyle: to_not diff --git a/app/javascript/mastodon/components/status_content.jsx b/app/javascript/mastodon/components/status_content.jsx index a1c38171f..f9c9fe079 100644 --- a/app/javascript/mastodon/components/status_content.jsx +++ b/app/javascript/mastodon/components/status_content.jsx @@ -6,7 +6,7 @@ import { Link } from 'react-router-dom'; import classnames from 'classnames'; import PollContainer from 'mastodon/containers/poll_container'; import Icon from 'mastodon/components/icon'; -import { autoPlayGif, languages as preloadedLanguages, translationEnabled } from 'mastodon/initial_state'; +import { autoPlayGif, languages as preloadedLanguages } from 'mastodon/initial_state'; const MAX_HEIGHT = 706; // 22px * 32 (+ 2px padding at the top) @@ -220,7 +220,7 @@ class StatusContent extends React.PureComponent { const hidden = this.props.onExpandedToggle ? !this.props.expanded : this.state.hidden; const renderReadMore = this.props.onClick && status.get('collapsed'); - const renderTranslate = translationEnabled && this.context.identity.signedIn && this.props.onTranslate && ['public', 'unlisted'].includes(status.get('visibility')) && status.get('contentHtml').length > 0 && status.get('language') !== null && intl.locale !== status.get('language'); + const renderTranslate = this.props.onTranslate && status.get('translatable'); const content = { __html: status.get('translation') ? status.getIn(['translation', 'content']) : status.get('contentHtml') }; const spoilerContent = { __html: status.get('spoilerHtml') }; diff --git a/app/javascript/mastodon/initial_state.js b/app/javascript/mastodon/initial_state.js index d04c4a42d..919e0fc28 100644 --- a/app/javascript/mastodon/initial_state.js +++ b/app/javascript/mastodon/initial_state.js @@ -80,7 +80,6 @@ * @property {boolean} use_blurhash * @property {boolean=} use_pending_items * @property {string} version - * @property {boolean} translation_enabled */ /** @@ -132,7 +131,6 @@ export const unfollowModal = getMeta('unfollow_modal'); export const useBlurhash = getMeta('use_blurhash'); export const usePendingItems = getMeta('use_pending_items'); export const version = getMeta('version'); -export const translationEnabled = getMeta('translation_enabled'); export const languages = initialState?.languages; export const statusPageUrl = getMeta('status_page_url'); diff --git a/app/lib/translation_service.rb b/app/lib/translation_service.rb index 285f30939..5ff93674a 100644 --- a/app/lib/translation_service.rb +++ b/app/lib/translation_service.rb @@ -21,6 +21,10 @@ class TranslationService ENV['DEEPL_API_KEY'].present? || ENV['LIBRE_TRANSLATE_ENDPOINT'].present? end + def supported?(_source_language, _target_language) + false + end + def translate(_text, _source_language, _target_language) raise NotImplementedError end diff --git a/app/lib/translation_service/deepl.rb b/app/lib/translation_service/deepl.rb index 151d33d90..deff95a1d 100644 --- a/app/lib/translation_service/deepl.rb +++ b/app/lib/translation_service/deepl.rb @@ -11,33 +11,53 @@ class TranslationService::DeepL < TranslationService end def translate(text, source_language, target_language) - request(text, source_language, target_language).perform do |res| + form = { text: text, source_lang: source_language&.upcase, target_lang: target_language, tag_handling: 'html' } + request(:post, '/v2/translate', form: form) do |res| + transform_response(res.body_with_limit) + end + end + + def supported?(source_language, target_language) + source_language.in?(languages('source')) && target_language.in?(languages('target')) + end + + private + + def languages(type) + Rails.cache.fetch("translation_service/deepl/languages/#{type}", expires_in: 7.days, race_condition_ttl: 1.minute) do + request(:get, "/v2/languages?type=#{type}") do |res| + # In DeepL, EN and PT are deprecated in favor of EN-GB/EN-US and PT-BR/PT-PT, so + # they are supported but not returned by the API. + extra = type == 'source' ? [nil] : %w(en pt) + languages = Oj.load(res.body_with_limit).map { |language| language['language'].downcase } + + languages + extra + end + end + end + + def request(verb, path, **options) + req = Request.new(verb, "#{base_url}#{path}", **options) + req.add_headers(Authorization: "DeepL-Auth-Key #{@api_key}") + req.perform do |res| case res.code when 429 raise TooManyRequestsError when 456 raise QuotaExceededError when 200...300 - transform_response(res.body_with_limit) + yield res else raise UnexpectedResponseError end end end - private - - def request(text, source_language, target_language) - req = Request.new(:post, endpoint_url, form: { text: text, source_lang: source_language&.upcase, target_lang: target_language, tag_handling: 'html' }) - req.add_headers(Authorization: "DeepL-Auth-Key #{@api_key}") - req - end - - def endpoint_url + def base_url if @plan == 'free' - 'https://api-free.deepl.com/v2/translate' + 'https://api-free.deepl.com' else - 'https://api.deepl.com/v2/translate' + 'https://api.deepl.com' end end diff --git a/app/lib/translation_service/libre_translate.rb b/app/lib/translation_service/libre_translate.rb index 4ebe21e45..743e4d77f 100644 --- a/app/lib/translation_service/libre_translate.rb +++ b/app/lib/translation_service/libre_translate.rb @@ -9,29 +9,45 @@ class TranslationService::LibreTranslate < TranslationService end def translate(text, source_language, target_language) - request(text, source_language, target_language).perform do |res| + body = Oj.dump(q: text, source: source_language.presence || 'auto', target: target_language, format: 'html', api_key: @api_key) + request(:post, '/translate', body: body) do |res| + transform_response(res.body_with_limit, source_language) + end + end + + def supported?(source_language, target_language) + languages.key?(source_language) && languages[source_language].include?(target_language) + end + + private + + def languages + Rails.cache.fetch('translation_service/libre_translate/languages', expires_in: 7.days, race_condition_ttl: 1.minute) do + request(:get, '/languages') do |res| + languages = Oj.load(res.body_with_limit).to_h { |language| [language['code'], language['targets']] } + languages[nil] = languages.values.flatten.uniq + languages + end + end + end + + def request(verb, path, **options) + req = Request.new(verb, "#{@base_url}#{path}", allow_local: true, **options) + req.add_headers('Content-Type': 'application/json') + req.perform do |res| case res.code when 429 raise TooManyRequestsError when 403 raise QuotaExceededError when 200...300 - transform_response(res.body_with_limit, source_language) + yield res else raise UnexpectedResponseError end end end - private - - def request(text, source_language, target_language) - body = Oj.dump(q: text, source: source_language.presence || 'auto', target: target_language, format: 'html', api_key: @api_key) - req = Request.new(:post, "#{@base_url}/translate", body: body, allow_local: true) - req.add_headers('Content-Type': 'application/json') - req - end - def transform_response(str, source_language) json = Oj.load(str, mode: :strict) diff --git a/app/models/status.rb b/app/models/status.rb index e7ea191a8..dd7ac2edb 100644 --- a/app/models/status.rb +++ b/app/models/status.rb @@ -232,6 +232,16 @@ class Status < ApplicationRecord public_visibility? || unlisted_visibility? end + def translatable? + translate_target_locale = I18n.locale.to_s.split(/[_-]/).first + + distributable? && + content.present? && + language != translate_target_locale && + TranslationService.configured? && + TranslationService.configured.supported?(language, translate_target_locale) + end + alias sign? distributable? def with_media? diff --git a/app/serializers/initial_state_serializer.rb b/app/serializers/initial_state_serializer.rb index 7905444e9..769ba653e 100644 --- a/app/serializers/initial_state_serializer.rb +++ b/app/serializers/initial_state_serializer.rb @@ -30,7 +30,6 @@ class InitialStateSerializer < ActiveModel::Serializer timeline_preview: Setting.timeline_preview, activity_api_enabled: Setting.activity_api_enabled, single_user_mode: Rails.configuration.x.single_user_mode, - translation_enabled: TranslationService.configured?, trends_as_landing_page: Setting.trends_as_landing_page, status_page_url: Setting.status_page_url, } diff --git a/app/serializers/rest/status_serializer.rb b/app/serializers/rest/status_serializer.rb index e0b8f32a6..a422f5b25 100644 --- a/app/serializers/rest/status_serializer.rb +++ b/app/serializers/rest/status_serializer.rb @@ -4,7 +4,7 @@ class REST::StatusSerializer < ActiveModel::Serializer include FormattingHelper attributes :id, :created_at, :in_reply_to_id, :in_reply_to_account_id, - :sensitive, :spoiler_text, :visibility, :language, + :sensitive, :spoiler_text, :visibility, :language, :translatable, :uri, :url, :replies_count, :reblogs_count, :favourites_count, :edited_at @@ -50,6 +50,10 @@ class REST::StatusSerializer < ActiveModel::Serializer object.account.user_shows_application? || (current_user? && current_user.account_id == object.account_id) end + def translatable + current_user? && object.translatable? + end + def visibility # This visibility is masked behind "private" # to avoid API changes because there are no diff --git a/app/services/translate_status_service.rb b/app/services/translate_status_service.rb index 539a0d9db..92d8b62a0 100644 --- a/app/services/translate_status_service.rb +++ b/app/services/translate_status_service.rb @@ -6,7 +6,7 @@ class TranslateStatusService < BaseService include FormattingHelper def call(status, target_language) - raise Mastodon::NotPermittedError unless status.public_visibility? || status.unlisted_visibility? + raise Mastodon::NotPermittedError unless status.translatable? @status = status @content = status_content_format(@status) diff --git a/spec/lib/translation_service/deepl_spec.rb b/spec/lib/translation_service/deepl_spec.rb new file mode 100644 index 000000000..aa2473186 --- /dev/null +++ b/spec/lib/translation_service/deepl_spec.rb @@ -0,0 +1,100 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe TranslationService::DeepL do + subject(:service) { described_class.new(plan, 'my-api-key') } + + let(:plan) { 'advanced' } + + before do + stub_request(:get, 'https://api.deepl.com/v2/languages?type=source').to_return( + body: '[{"language":"EN","name":"English"},{"language":"UK","name":"Ukrainian"}]' + ) + stub_request(:get, 'https://api.deepl.com/v2/languages?type=target').to_return( + body: '[{"language":"EN-GB","name":"English (British)"},{"language":"ZH","name":"Chinese"}]' + ) + end + + describe '#supported?' do + it 'supports included languages as source and target languages' do + expect(service.supported?('uk', 'en')).to be true + end + + it 'supports auto-detecting source language' do + expect(service.supported?(nil, 'en')).to be true + end + + it 'supports "en" and "pt" as target languages though not included in language list' do + expect(service.supported?('uk', 'en')).to be true + expect(service.supported?('uk', 'pt')).to be true + end + + it 'does not support non-included language as target language' do + expect(service.supported?('uk', 'nl')).to be false + end + + it 'does not support non-included language as source language' do + expect(service.supported?('da', 'en')).to be false + end + end + + describe '#translate' do + it 'returns translation with specified source language' do + stub_request(:post, 'https://api.deepl.com/v2/translate') + .with(body: 'text=Hasta+la+vista&source_lang=ES&target_lang=en&tag_handling=html') + .to_return(body: '{"translations":[{"detected_source_language":"ES","text":"See you soon"}]}') + + translation = service.translate('Hasta la vista', 'es', 'en') + expect(translation.detected_source_language).to eq 'es' + expect(translation.provider).to eq 'DeepL.com' + expect(translation.text).to eq 'See you soon' + end + + it 'returns translation with auto-detected source language' do + stub_request(:post, 'https://api.deepl.com/v2/translate') + .with(body: 'text=Guten+Tag&source_lang&target_lang=en&tag_handling=html') + .to_return(body: '{"translations":[{"detected_source_language":"DE","text":"Good Morning"}]}') + + translation = service.translate('Guten Tag', nil, 'en') + expect(translation.detected_source_language).to eq 'de' + expect(translation.provider).to eq 'DeepL.com' + expect(translation.text).to eq 'Good Morning' + end + end + + describe '#languages?' do + it 'returns source languages' do + expect(service.send(:languages, 'source')).to eq ['en', 'uk', nil] + end + + it 'returns target languages' do + expect(service.send(:languages, 'target')).to eq %w(en-gb zh en pt) + end + end + + describe '#request' do + before do + stub_request(:any, //) + # rubocop:disable Lint/EmptyBlock + service.send(:request, :get, '/v2/languages') { |res| } + # rubocop:enable Lint/EmptyBlock + end + + it 'uses paid plan base URL' do + expect(a_request(:get, 'https://api.deepl.com/v2/languages')).to have_been_made.once + end + + context 'with free plan' do + let(:plan) { 'free' } + + it 'uses free plan base URL' do + expect(a_request(:get, 'https://api-free.deepl.com/v2/languages')).to have_been_made.once + end + end + + it 'sends API key' do + expect(a_request(:get, 'https://api.deepl.com/v2/languages').with(headers: { Authorization: 'DeepL-Auth-Key my-api-key' })).to have_been_made.once + end + end +end diff --git a/spec/lib/translation_service/libre_translate_spec.rb b/spec/lib/translation_service/libre_translate_spec.rb new file mode 100644 index 000000000..a6cb01884 --- /dev/null +++ b/spec/lib/translation_service/libre_translate_spec.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +require 'rails_helper' + +RSpec.describe TranslationService::LibreTranslate do + subject(:service) { described_class.new('https://libretranslate.example.com', 'my-api-key') } + + before do + stub_request(:get, 'https://libretranslate.example.com/languages').to_return( + body: '[{"code": "en","name": "English","targets": ["de","es"]},{"code": "da","name": "Danish","targets": ["en","de"]}]' + ) + end + + describe '#supported?' do + it 'supports included language pair' do + expect(service.supported?('en', 'de')).to be true + end + + it 'does not support reversed language pair' do + expect(service.supported?('de', 'en')).to be false + end + + it 'supports auto-detecting source language' do + expect(service.supported?(nil, 'de')).to be true + end + + it 'does not support auto-detecting for unsupported target language' do + expect(service.supported?(nil, 'pt')).to be false + end + end + + describe '#languages' do + subject(:languages) { service.send(:languages) } + + it 'includes supported source languages' do + expect(languages.keys).to eq ['en', 'da', nil] + end + + it 'includes supported target languages for source language' do + expect(languages['en']).to eq %w(de es) + end + + it 'includes supported target languages for auto-detected language' do + expect(languages[nil]).to eq %w(de es en) + end + end + + describe '#translate' do + it 'returns translation with specified source language' do + stub_request(:post, 'https://libretranslate.example.com/translate') + .with(body: '{"q":"Hasta la vista","source":"es","target":"en","format":"html","api_key":"my-api-key"}') + .to_return(body: '{"translatedText": "See you"}') + + translation = service.translate('Hasta la vista', 'es', 'en') + expect(translation.detected_source_language).to eq 'es' + expect(translation.provider).to eq 'LibreTranslate' + expect(translation.text).to eq 'See you' + end + + it 'returns translation with auto-detected source language' do + stub_request(:post, 'https://libretranslate.example.com/translate') + .with(body: '{"q":"Guten Morgen","source":"auto","target":"en","format":"html","api_key":"my-api-key"}') + .to_return(body: '{"detectedLanguage":{"confidence":92,"language":"de"},"translatedText":"Good morning"}') + + translation = service.translate('Guten Morgen', nil, 'en') + expect(translation.detected_source_language).to be_nil + expect(translation.provider).to eq 'LibreTranslate' + expect(translation.text).to eq 'Good morning' + end + end +end diff --git a/spec/models/status_spec.rb b/spec/models/status_spec.rb index 1e58c6d0d..1f6cfc796 100644 --- a/spec/models/status_spec.rb +++ b/spec/models/status_spec.rb @@ -114,6 +114,85 @@ RSpec.describe Status, type: :model do end end + describe '#translatable?' do + before do + allow(TranslationService).to receive(:configured?).and_return(true) + allow(TranslationService).to receive(:configured).and_return(TranslationService.new) + allow(TranslationService.configured).to receive(:supported?).with('es', 'en').and_return(true) + + subject.language = 'es' + subject.visibility = :public + end + + context 'all conditions are satisfied' do + it 'returns true' do + expect(subject.translatable?).to be true + end + end + + context 'translation service is not configured' do + it 'returns false' do + allow(TranslationService).to receive(:configured?).and_return(false) + allow(TranslationService).to receive(:configured).and_raise(TranslationService::NotConfiguredError) + expect(subject.translatable?).to be false + end + end + + context 'status language is nil' do + it 'returns true' do + subject.language = nil + allow(TranslationService.configured).to receive(:supported?).with(nil, 'en').and_return(true) + expect(subject.translatable?).to be true + end + end + + context 'status language is same as default locale' do + it 'returns false' do + subject.language = I18n.locale + expect(subject.translatable?).to be false + end + end + + context 'status language is unsupported' do + it 'returns false' do + subject.language = 'af' + allow(TranslationService.configured).to receive(:supported?).with('af', 'en').and_return(false) + expect(subject.translatable?).to be false + end + end + + context 'default locale is unsupported' do + it 'returns false' do + allow(TranslationService.configured).to receive(:supported?).with('es', 'af').and_return(false) + I18n.with_locale('af') do + expect(subject.translatable?).to be false + end + end + end + + context 'default locale has region' do + it 'returns true' do + I18n.with_locale('en-GB') do + expect(subject.translatable?).to be true + end + end + end + + context 'status text is blank' do + it 'returns false' do + subject.text = ' ' + expect(subject.translatable?).to be false + end + end + + context 'status visiblity is hidden' do + it 'returns false' do + subject.visibility = 'limited' + expect(subject.translatable?).to be false + end + end + end + describe '#content' do it 'returns the text of the status if it is not a reblog' do expect(subject.content).to eql subject.text -- cgit From bd047acc356671727c112336bb237f979bba517d Mon Sep 17 00:00:00 2001 From: Christian Schmidt Date: Thu, 16 Mar 2023 11:07:24 +0100 Subject: Replace `Status#translatable?` with language matrix in separate endpoint (#24037) --- .rubocop_todo.yml | 1 - .../instances/translation_languages_controller.rb | 23 +++++++ app/javascript/mastodon/actions/server.js | 27 ++++++++ .../mastodon/components/status_content.jsx | 13 +++- app/javascript/mastodon/features/ui/index.jsx | 3 +- app/javascript/mastodon/reducers/server.js | 9 +++ app/lib/translation_service.rb | 4 +- app/lib/translation_service/deepl.rb | 30 ++++---- app/lib/translation_service/libre_translate.rb | 18 ++--- app/models/status.rb | 10 --- app/serializers/rest/status_serializer.rb | 6 +- app/services/translate_status_service.rb | 16 ++++- config/routes.rb | 1 + .../translation_languages_controller_spec.rb | 31 +++++++++ .../v1/statuses/translations_controller_spec.rb | 3 +- spec/lib/translation_service/deepl_spec.rb | 36 +++------- .../translation_service/libre_translate_spec.rb | 31 ++------- spec/models/status_spec.rb | 79 ---------------------- spec/presenters/instance_presenter_spec.rb | 2 +- 19 files changed, 164 insertions(+), 179 deletions(-) create mode 100644 app/controllers/api/v1/instances/translation_languages_controller.rb create mode 100644 spec/controllers/api/v1/instances/translation_languages_controller_spec.rb (limited to 'app/lib/translation_service') diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index e79f4f8e9..85f078dcf 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -484,7 +484,6 @@ RSpec/DescribedClass: - 'spec/models/user_spec.rb' - 'spec/policies/account_moderation_note_policy_spec.rb' - 'spec/presenters/account_relationships_presenter_spec.rb' - - 'spec/presenters/instance_presenter_spec.rb' - 'spec/presenters/status_relationships_presenter_spec.rb' - 'spec/serializers/activitypub/note_serializer_spec.rb' - 'spec/serializers/activitypub/update_poll_serializer_spec.rb' diff --git a/app/controllers/api/v1/instances/translation_languages_controller.rb b/app/controllers/api/v1/instances/translation_languages_controller.rb new file mode 100644 index 000000000..3910a499e --- /dev/null +++ b/app/controllers/api/v1/instances/translation_languages_controller.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +class Api::V1::Instances::TranslationLanguagesController < Api::BaseController + skip_before_action :require_authenticated_user!, unless: :whitelist_mode? + + before_action :set_languages + + def show + expires_in 1.day, public: true + render json: @languages + end + + private + + def set_languages + if TranslationService.configured? + @languages = Rails.cache.fetch('translation_service/languages', expires_in: 7.days, race_condition_ttl: 1.hour) { TranslationService.configured.languages } + @languages['und'] = @languages.delete(nil) if @languages.key?(nil) + else + @languages = {} + end + end +end diff --git a/app/javascript/mastodon/actions/server.js b/app/javascript/mastodon/actions/server.js index 31d4aea10..091af0f0f 100644 --- a/app/javascript/mastodon/actions/server.js +++ b/app/javascript/mastodon/actions/server.js @@ -5,6 +5,10 @@ export const SERVER_FETCH_REQUEST = 'Server_FETCH_REQUEST'; export const SERVER_FETCH_SUCCESS = 'Server_FETCH_SUCCESS'; export const SERVER_FETCH_FAIL = 'Server_FETCH_FAIL'; +export const SERVER_TRANSLATION_LANGUAGES_FETCH_REQUEST = 'SERVER_TRANSLATION_LANGUAGES_FETCH_REQUEST'; +export const SERVER_TRANSLATION_LANGUAGES_FETCH_SUCCESS = 'SERVER_TRANSLATION_LANGUAGES_FETCH_SUCCESS'; +export const SERVER_TRANSLATION_LANGUAGES_FETCH_FAIL = 'SERVER_TRANSLATION_LANGUAGES_FETCH_FAIL'; + export const EXTENDED_DESCRIPTION_REQUEST = 'EXTENDED_DESCRIPTION_REQUEST'; export const EXTENDED_DESCRIPTION_SUCCESS = 'EXTENDED_DESCRIPTION_SUCCESS'; export const EXTENDED_DESCRIPTION_FAIL = 'EXTENDED_DESCRIPTION_FAIL'; @@ -37,6 +41,29 @@ const fetchServerFail = error => ({ error, }); +export const fetchServerTranslationLanguages = () => (dispatch, getState) => { + dispatch(fetchServerTranslationLanguagesRequest()); + + api(getState) + .get('/api/v1/instance/translation_languages').then(({ data }) => { + dispatch(fetchServerTranslationLanguagesSuccess(data)); + }).catch(err => dispatch(fetchServerTranslationLanguagesFail(err))); +}; + +const fetchServerTranslationLanguagesRequest = () => ({ + type: SERVER_TRANSLATION_LANGUAGES_FETCH_REQUEST, +}); + +const fetchServerTranslationLanguagesSuccess = translationLanguages => ({ + type: SERVER_TRANSLATION_LANGUAGES_FETCH_SUCCESS, + translationLanguages, +}); + +const fetchServerTranslationLanguagesFail = error => ({ + type: SERVER_TRANSLATION_LANGUAGES_FETCH_FAIL, + error, +}); + export const fetchExtendedDescription = () => (dispatch, getState) => { dispatch(fetchExtendedDescriptionRequest()); diff --git a/app/javascript/mastodon/components/status_content.jsx b/app/javascript/mastodon/components/status_content.jsx index f9c9fe079..67a487b00 100644 --- a/app/javascript/mastodon/components/status_content.jsx +++ b/app/javascript/mastodon/components/status_content.jsx @@ -3,6 +3,7 @@ import ImmutablePropTypes from 'react-immutable-proptypes'; import PropTypes from 'prop-types'; import { FormattedMessage, injectIntl } from 'react-intl'; import { Link } from 'react-router-dom'; +import { connect } from 'react-redux'; import classnames from 'classnames'; import PollContainer from 'mastodon/containers/poll_container'; import Icon from 'mastodon/components/icon'; @@ -47,7 +48,12 @@ class TranslateButton extends React.PureComponent { } -export default @injectIntl +const mapStateToProps = state => ({ + languages: state.getIn(['server', 'translationLanguages', 'items']), +}); + +export default @connect(mapStateToProps) +@injectIntl class StatusContent extends React.PureComponent { static contextTypes = { @@ -63,6 +69,7 @@ class StatusContent extends React.PureComponent { onClick: PropTypes.func, collapsable: PropTypes.bool, onCollapsedToggle: PropTypes.func, + languages: ImmutablePropTypes.map, intl: PropTypes.object, }; @@ -220,7 +227,9 @@ class StatusContent extends React.PureComponent { const hidden = this.props.onExpandedToggle ? !this.props.expanded : this.state.hidden; const renderReadMore = this.props.onClick && status.get('collapsed'); - const renderTranslate = this.props.onTranslate && status.get('translatable'); + const contentLocale = intl.locale.replace(/[_-].*/, ''); + const targetLanguages = this.props.languages?.get(status.get('language') || 'und'); + const renderTranslate = this.props.onTranslate && this.context.identity.signedIn && ['public', 'unlisted'].includes(status.get('visibility')) && status.get('contentHtml').length > 0 && targetLanguages?.includes(contentLocale); const content = { __html: status.get('translation') ? status.getIn(['translation', 'content']) : status.get('contentHtml') }; const spoilerContent = { __html: status.get('spoilerHtml') }; diff --git a/app/javascript/mastodon/features/ui/index.jsx b/app/javascript/mastodon/features/ui/index.jsx index 2dd59f95d..083707220 100644 --- a/app/javascript/mastodon/features/ui/index.jsx +++ b/app/javascript/mastodon/features/ui/index.jsx @@ -13,7 +13,7 @@ import { debounce } from 'lodash'; import { uploadCompose, resetCompose, changeComposeSpoilerness } from '../../actions/compose'; import { expandHomeTimeline } from '../../actions/timelines'; import { expandNotifications } from '../../actions/notifications'; -import { fetchServer } from '../../actions/server'; +import { fetchServer, fetchServerTranslationLanguages } from '../../actions/server'; import { clearHeight } from '../../actions/height_cache'; import { focusApp, unfocusApp, changeLayout } from 'mastodon/actions/app'; import { synchronouslySubmitMarkers, submitMarkers, fetchMarkers } from 'mastodon/actions/markers'; @@ -399,6 +399,7 @@ class UI extends React.PureComponent { this.props.dispatch(fetchMarkers()); this.props.dispatch(expandHomeTimeline()); this.props.dispatch(expandNotifications()); + this.props.dispatch(fetchServerTranslationLanguages()); setTimeout(() => this.props.dispatch(fetchServer()), 3000); } diff --git a/app/javascript/mastodon/reducers/server.js b/app/javascript/mastodon/reducers/server.js index db9f2b5e6..909ab2a66 100644 --- a/app/javascript/mastodon/reducers/server.js +++ b/app/javascript/mastodon/reducers/server.js @@ -2,6 +2,9 @@ import { SERVER_FETCH_REQUEST, SERVER_FETCH_SUCCESS, SERVER_FETCH_FAIL, + SERVER_TRANSLATION_LANGUAGES_FETCH_REQUEST, + SERVER_TRANSLATION_LANGUAGES_FETCH_SUCCESS, + SERVER_TRANSLATION_LANGUAGES_FETCH_FAIL, EXTENDED_DESCRIPTION_REQUEST, EXTENDED_DESCRIPTION_SUCCESS, EXTENDED_DESCRIPTION_FAIL, @@ -35,6 +38,12 @@ export default function server(state = initialState, action) { return state.set('server', fromJS(action.server)).setIn(['server', 'isLoading'], false); case SERVER_FETCH_FAIL: return state.setIn(['server', 'isLoading'], false); + case SERVER_TRANSLATION_LANGUAGES_FETCH_REQUEST: + return state.setIn(['translationLanguages', 'isLoading'], true); + case SERVER_TRANSLATION_LANGUAGES_FETCH_SUCCESS: + return state.setIn(['translationLanguages', 'items'], fromJS(action.translationLanguages)).setIn(['translationLanguages', 'isLoading'], false); + case SERVER_TRANSLATION_LANGUAGES_FETCH_FAIL: + return state.setIn(['translationLanguages', 'isLoading'], false); case EXTENDED_DESCRIPTION_REQUEST: return state.setIn(['extendedDescription', 'isLoading'], true); case EXTENDED_DESCRIPTION_SUCCESS: diff --git a/app/lib/translation_service.rb b/app/lib/translation_service.rb index 5ff93674a..bfe5de44f 100644 --- a/app/lib/translation_service.rb +++ b/app/lib/translation_service.rb @@ -21,8 +21,8 @@ class TranslationService ENV['DEEPL_API_KEY'].present? || ENV['LIBRE_TRANSLATE_ENDPOINT'].present? end - def supported?(_source_language, _target_language) - false + def languages + {} end def translate(_text, _source_language, _target_language) diff --git a/app/lib/translation_service/deepl.rb b/app/lib/translation_service/deepl.rb index deff95a1d..afcb7ecb2 100644 --- a/app/lib/translation_service/deepl.rb +++ b/app/lib/translation_service/deepl.rb @@ -17,25 +17,31 @@ class TranslationService::DeepL < TranslationService end end - def supported?(source_language, target_language) - source_language.in?(languages('source')) && target_language.in?(languages('target')) + def languages + source_languages = [nil] + fetch_languages('source') + + # In DeepL, EN and PT are deprecated in favor of EN-GB/EN-US and PT-BR/PT-PT, so + # they are supported but not returned by the API. + target_languages = %w(en pt) + fetch_languages('target') + + source_languages.index_with { |language| target_languages.without(nil, language) } end private - def languages(type) - Rails.cache.fetch("translation_service/deepl/languages/#{type}", expires_in: 7.days, race_condition_ttl: 1.minute) do - request(:get, "/v2/languages?type=#{type}") do |res| - # In DeepL, EN and PT are deprecated in favor of EN-GB/EN-US and PT-BR/PT-PT, so - # they are supported but not returned by the API. - extra = type == 'source' ? [nil] : %w(en pt) - languages = Oj.load(res.body_with_limit).map { |language| language['language'].downcase } - - languages + extra - end + def fetch_languages(type) + request(:get, "/v2/languages?type=#{type}") do |res| + Oj.load(res.body_with_limit).map { |language| normalize_language(language['language']) } end end + def normalize_language(language) + subtags = language.split(/[_-]/) + subtags[0].downcase! + subtags[1]&.upcase! + subtags.join('-') + end + def request(verb, path, **options) req = Request.new(verb, "#{base_url}#{path}", **options) req.add_headers(Authorization: "DeepL-Auth-Key #{@api_key}") diff --git a/app/lib/translation_service/libre_translate.rb b/app/lib/translation_service/libre_translate.rb index 743e4d77f..8bb194a9c 100644 --- a/app/lib/translation_service/libre_translate.rb +++ b/app/lib/translation_service/libre_translate.rb @@ -15,22 +15,18 @@ class TranslationService::LibreTranslate < TranslationService end end - def supported?(source_language, target_language) - languages.key?(source_language) && languages[source_language].include?(target_language) - end - - private - def languages - Rails.cache.fetch('translation_service/libre_translate/languages', expires_in: 7.days, race_condition_ttl: 1.minute) do - request(:get, '/languages') do |res| - languages = Oj.load(res.body_with_limit).to_h { |language| [language['code'], language['targets']] } - languages[nil] = languages.values.flatten.uniq - languages + request(:get, '/languages') do |res| + languages = Oj.load(res.body_with_limit).to_h do |language| + [language['code'], language['targets'].without(language['code'])] end + languages[nil] = languages.values.flatten.uniq.sort + languages end end + private + def request(verb, path, **options) req = Request.new(verb, "#{@base_url}#{path}", allow_local: true, **options) req.add_headers('Content-Type': 'application/json') diff --git a/app/models/status.rb b/app/models/status.rb index dd7ac2edb..e7ea191a8 100644 --- a/app/models/status.rb +++ b/app/models/status.rb @@ -232,16 +232,6 @@ class Status < ApplicationRecord public_visibility? || unlisted_visibility? end - def translatable? - translate_target_locale = I18n.locale.to_s.split(/[_-]/).first - - distributable? && - content.present? && - language != translate_target_locale && - TranslationService.configured? && - TranslationService.configured.supported?(language, translate_target_locale) - end - alias sign? distributable? def with_media? diff --git a/app/serializers/rest/status_serializer.rb b/app/serializers/rest/status_serializer.rb index a422f5b25..e0b8f32a6 100644 --- a/app/serializers/rest/status_serializer.rb +++ b/app/serializers/rest/status_serializer.rb @@ -4,7 +4,7 @@ class REST::StatusSerializer < ActiveModel::Serializer include FormattingHelper attributes :id, :created_at, :in_reply_to_id, :in_reply_to_account_id, - :sensitive, :spoiler_text, :visibility, :language, :translatable, + :sensitive, :spoiler_text, :visibility, :language, :uri, :url, :replies_count, :reblogs_count, :favourites_count, :edited_at @@ -50,10 +50,6 @@ class REST::StatusSerializer < ActiveModel::Serializer object.account.user_shows_application? || (current_user? && current_user.account_id == object.account_id) end - def translatable - current_user? && object.translatable? - end - def visibility # This visibility is masked behind "private" # to avoid API changes because there are no diff --git a/app/services/translate_status_service.rb b/app/services/translate_status_service.rb index 92d8b62a0..796f13a0d 100644 --- a/app/services/translate_status_service.rb +++ b/app/services/translate_status_service.rb @@ -6,19 +6,29 @@ class TranslateStatusService < BaseService include FormattingHelper def call(status, target_language) - raise Mastodon::NotPermittedError unless status.translatable? - @status = status @content = status_content_format(@status) @target_language = target_language + raise Mastodon::NotPermittedError unless permitted? + Rails.cache.fetch("translations/#{@status.language}/#{@target_language}/#{content_hash}", expires_in: CACHE_TTL) { translation_backend.translate(@content, @status.language, @target_language) } end private def translation_backend - TranslationService.configured + @translation_backend ||= TranslationService.configured + end + + def permitted? + return false unless @status.distributable? && @status.content.present? && TranslationService.configured? + + languages[@status.language]&.include?(@target_language) + end + + def languages + Rails.cache.fetch('translation_service/languages', expires_in: 7.days, race_condition_ttl: 1.hour) { TranslationService.configured.languages } end def content_hash diff --git a/config/routes.rb b/config/routes.rb index 530b46a5a..ea595e1e1 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -546,6 +546,7 @@ Rails.application.routes.draw do resources :domain_blocks, only: [:index], controller: 'instances/domain_blocks' resource :privacy_policy, only: [:show], controller: 'instances/privacy_policies' resource :extended_description, only: [:show], controller: 'instances/extended_descriptions' + resource :translation_languages, only: [:show], controller: 'instances/translation_languages' resource :activity, only: [:show], controller: 'instances/activity' end diff --git a/spec/controllers/api/v1/instances/translation_languages_controller_spec.rb b/spec/controllers/api/v1/instances/translation_languages_controller_spec.rb new file mode 100644 index 000000000..5b7e4abb6 --- /dev/null +++ b/spec/controllers/api/v1/instances/translation_languages_controller_spec.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +require 'rails_helper' + +describe Api::V1::Instances::TranslationLanguagesController do + describe 'GET #show' do + context 'when no translation service is configured' do + it 'returns empty language matrix' do + get :show + + expect(response).to have_http_status(200) + expect(body_as_json).to eq({}) + end + end + + context 'when a translation service is configured' do + before do + service = instance_double(TranslationService::DeepL, languages: { nil => %w(en de), 'en' => ['de'] }) + allow(TranslationService).to receive(:configured?).and_return(true) + allow(TranslationService).to receive(:configured).and_return(service) + end + + it 'returns language matrix' do + get :show + + expect(response).to have_http_status(200) + expect(body_as_json).to eq({ und: %w(en de), en: ['de'] }) + end + end + end +end diff --git a/spec/controllers/api/v1/statuses/translations_controller_spec.rb b/spec/controllers/api/v1/statuses/translations_controller_spec.rb index 2deea9fc0..8495779bf 100644 --- a/spec/controllers/api/v1/statuses/translations_controller_spec.rb +++ b/spec/controllers/api/v1/statuses/translations_controller_spec.rb @@ -19,9 +19,10 @@ describe Api::V1::Statuses::TranslationsController do before do translation = TranslationService::Translation.new(text: 'Hello') - service = instance_double(TranslationService::DeepL, translate: translation, supported?: true) + service = instance_double(TranslationService::DeepL, translate: translation) allow(TranslationService).to receive(:configured?).and_return(true) allow(TranslationService).to receive(:configured).and_return(service) + Rails.cache.write('translation_service/languages', { 'es' => ['en'] }) post :create, params: { status_id: status.id } end diff --git a/spec/lib/translation_service/deepl_spec.rb b/spec/lib/translation_service/deepl_spec.rb index aa2473186..2363f8f13 100644 --- a/spec/lib/translation_service/deepl_spec.rb +++ b/spec/lib/translation_service/deepl_spec.rb @@ -16,29 +16,6 @@ RSpec.describe TranslationService::DeepL do ) end - describe '#supported?' do - it 'supports included languages as source and target languages' do - expect(service.supported?('uk', 'en')).to be true - end - - it 'supports auto-detecting source language' do - expect(service.supported?(nil, 'en')).to be true - end - - it 'supports "en" and "pt" as target languages though not included in language list' do - expect(service.supported?('uk', 'en')).to be true - expect(service.supported?('uk', 'pt')).to be true - end - - it 'does not support non-included language as target language' do - expect(service.supported?('uk', 'nl')).to be false - end - - it 'does not support non-included language as source language' do - expect(service.supported?('da', 'en')).to be false - end - end - describe '#translate' do it 'returns translation with specified source language' do stub_request(:post, 'https://api.deepl.com/v2/translate') @@ -63,13 +40,18 @@ RSpec.describe TranslationService::DeepL do end end - describe '#languages?' do + describe '#languages' do it 'returns source languages' do - expect(service.send(:languages, 'source')).to eq ['en', 'uk', nil] + expect(service.languages.keys).to eq [nil, 'en', 'uk'] + end + + it 'returns target languages for each source language' do + expect(service.languages['en']).to eq %w(pt en-GB zh) + expect(service.languages['uk']).to eq %w(en pt en-GB zh) end - it 'returns target languages' do - expect(service.send(:languages, 'target')).to eq %w(en-gb zh en pt) + it 'returns target languages for auto-detection' do + expect(service.languages[nil]).to eq %w(en pt en-GB zh) end end diff --git a/spec/lib/translation_service/libre_translate_spec.rb b/spec/lib/translation_service/libre_translate_spec.rb index a6cb01884..fbd726a7e 100644 --- a/spec/lib/translation_service/libre_translate_spec.rb +++ b/spec/lib/translation_service/libre_translate_spec.rb @@ -7,41 +7,24 @@ RSpec.describe TranslationService::LibreTranslate do before do stub_request(:get, 'https://libretranslate.example.com/languages').to_return( - body: '[{"code": "en","name": "English","targets": ["de","es"]},{"code": "da","name": "Danish","targets": ["en","de"]}]' + body: '[{"code": "en","name": "English","targets": ["de","en","es"]},{"code": "da","name": "Danish","targets": ["en","pt"]}]' ) end - describe '#supported?' do - it 'supports included language pair' do - expect(service.supported?('en', 'de')).to be true - end - - it 'does not support reversed language pair' do - expect(service.supported?('de', 'en')).to be false - end - - it 'supports auto-detecting source language' do - expect(service.supported?(nil, 'de')).to be true - end - - it 'does not support auto-detecting for unsupported target language' do - expect(service.supported?(nil, 'pt')).to be false - end - end - describe '#languages' do - subject(:languages) { service.send(:languages) } + subject(:languages) { service.languages } - it 'includes supported source languages' do + it 'returns source languages' do expect(languages.keys).to eq ['en', 'da', nil] end - it 'includes supported target languages for source language' do + it 'returns target languages for each source language' do expect(languages['en']).to eq %w(de es) + expect(languages['da']).to eq %w(en pt) end - it 'includes supported target languages for auto-detected language' do - expect(languages[nil]).to eq %w(de es en) + it 'returns target languages for auto-detected language' do + expect(languages[nil]).to eq %w(de en es pt) end end diff --git a/spec/models/status_spec.rb b/spec/models/status_spec.rb index 1f6cfc796..1e58c6d0d 100644 --- a/spec/models/status_spec.rb +++ b/spec/models/status_spec.rb @@ -114,85 +114,6 @@ RSpec.describe Status, type: :model do end end - describe '#translatable?' do - before do - allow(TranslationService).to receive(:configured?).and_return(true) - allow(TranslationService).to receive(:configured).and_return(TranslationService.new) - allow(TranslationService.configured).to receive(:supported?).with('es', 'en').and_return(true) - - subject.language = 'es' - subject.visibility = :public - end - - context 'all conditions are satisfied' do - it 'returns true' do - expect(subject.translatable?).to be true - end - end - - context 'translation service is not configured' do - it 'returns false' do - allow(TranslationService).to receive(:configured?).and_return(false) - allow(TranslationService).to receive(:configured).and_raise(TranslationService::NotConfiguredError) - expect(subject.translatable?).to be false - end - end - - context 'status language is nil' do - it 'returns true' do - subject.language = nil - allow(TranslationService.configured).to receive(:supported?).with(nil, 'en').and_return(true) - expect(subject.translatable?).to be true - end - end - - context 'status language is same as default locale' do - it 'returns false' do - subject.language = I18n.locale - expect(subject.translatable?).to be false - end - end - - context 'status language is unsupported' do - it 'returns false' do - subject.language = 'af' - allow(TranslationService.configured).to receive(:supported?).with('af', 'en').and_return(false) - expect(subject.translatable?).to be false - end - end - - context 'default locale is unsupported' do - it 'returns false' do - allow(TranslationService.configured).to receive(:supported?).with('es', 'af').and_return(false) - I18n.with_locale('af') do - expect(subject.translatable?).to be false - end - end - end - - context 'default locale has region' do - it 'returns true' do - I18n.with_locale('en-GB') do - expect(subject.translatable?).to be true - end - end - end - - context 'status text is blank' do - it 'returns false' do - subject.text = ' ' - expect(subject.translatable?).to be false - end - end - - context 'status visiblity is hidden' do - it 'returns false' do - subject.visibility = 'limited' - expect(subject.translatable?).to be false - end - end - end - describe '#content' do it 'returns the text of the status if it is not a reblog' do expect(subject.content).to eql subject.text diff --git a/spec/presenters/instance_presenter_spec.rb b/spec/presenters/instance_presenter_spec.rb index 795abd8b4..2a1d668ce 100644 --- a/spec/presenters/instance_presenter_spec.rb +++ b/spec/presenters/instance_presenter_spec.rb @@ -3,7 +3,7 @@ require 'rails_helper' describe InstancePresenter do - let(:instance_presenter) { InstancePresenter.new } + let(:instance_presenter) { described_class.new } describe '#description' do around do |example| -- cgit