diff options
19 files changed, 164 insertions, 179 deletions
diff --git a/.rubocop_todo.yml b/.rubocop_todo.yml index e79f4f8e9..85f078dcf 100644 --- a/.rubocop_todo.yml +++ b/.rubocop_todo.yml @@ -484,7 +484,6 @@ RSpec/DescribedClass: - 'spec/models/user_spec.rb' - 'spec/policies/account_moderation_note_policy_spec.rb' - 'spec/presenters/account_relationships_presenter_spec.rb' - - 'spec/presenters/instance_presenter_spec.rb' - 'spec/presenters/status_relationships_presenter_spec.rb' - 'spec/serializers/activitypub/note_serializer_spec.rb' - 'spec/serializers/activitypub/update_poll_serializer_spec.rb' diff --git a/app/controllers/api/v1/instances/translation_languages_controller.rb b/app/controllers/api/v1/instances/translation_languages_controller.rb new file mode 100644 index 000000000..3910a499e --- /dev/null +++ b/app/controllers/api/v1/instances/translation_languages_controller.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +class Api::V1::Instances::TranslationLanguagesController < Api::BaseController + skip_before_action :require_authenticated_user!, unless: :whitelist_mode? + + before_action :set_languages + + def show + expires_in 1.day, public: true + render json: @languages + end + + private + + def set_languages + if TranslationService.configured? + @languages = Rails.cache.fetch('translation_service/languages', expires_in: 7.days, race_condition_ttl: 1.hour) { TranslationService.configured.languages } + @languages['und'] = @languages.delete(nil) if @languages.key?(nil) + else + @languages = {} + end + end +end diff --git a/app/javascript/mastodon/actions/server.js b/app/javascript/mastodon/actions/server.js index 31d4aea10..091af0f0f 100644 --- a/app/javascript/mastodon/actions/server.js +++ b/app/javascript/mastodon/actions/server.js @@ -5,6 +5,10 @@ export const SERVER_FETCH_REQUEST = 'Server_FETCH_REQUEST'; export const SERVER_FETCH_SUCCESS = 'Server_FETCH_SUCCESS'; export const SERVER_FETCH_FAIL = 'Server_FETCH_FAIL'; +export const SERVER_TRANSLATION_LANGUAGES_FETCH_REQUEST = 'SERVER_TRANSLATION_LANGUAGES_FETCH_REQUEST'; +export const SERVER_TRANSLATION_LANGUAGES_FETCH_SUCCESS = 'SERVER_TRANSLATION_LANGUAGES_FETCH_SUCCESS'; +export const SERVER_TRANSLATION_LANGUAGES_FETCH_FAIL = 'SERVER_TRANSLATION_LANGUAGES_FETCH_FAIL'; + export const EXTENDED_DESCRIPTION_REQUEST = 'EXTENDED_DESCRIPTION_REQUEST'; export const EXTENDED_DESCRIPTION_SUCCESS = 'EXTENDED_DESCRIPTION_SUCCESS'; export const EXTENDED_DESCRIPTION_FAIL = 'EXTENDED_DESCRIPTION_FAIL'; @@ -37,6 +41,29 @@ const fetchServerFail = error => ({ error, }); +export const fetchServerTranslationLanguages = () => (dispatch, getState) => { + dispatch(fetchServerTranslationLanguagesRequest()); + + api(getState) + .get('/api/v1/instance/translation_languages').then(({ data }) => { + dispatch(fetchServerTranslationLanguagesSuccess(data)); + }).catch(err => dispatch(fetchServerTranslationLanguagesFail(err))); +}; + +const fetchServerTranslationLanguagesRequest = () => ({ + type: SERVER_TRANSLATION_LANGUAGES_FETCH_REQUEST, +}); + +const fetchServerTranslationLanguagesSuccess = translationLanguages => ({ + type: SERVER_TRANSLATION_LANGUAGES_FETCH_SUCCESS, + translationLanguages, +}); + +const fetchServerTranslationLanguagesFail = error => ({ + type: SERVER_TRANSLATION_LANGUAGES_FETCH_FAIL, + error, +}); + export const fetchExtendedDescription = () => (dispatch, getState) => { dispatch(fetchExtendedDescriptionRequest()); diff --git a/app/javascript/mastodon/components/status_content.jsx b/app/javascript/mastodon/components/status_content.jsx index f9c9fe079..67a487b00 100644 --- a/app/javascript/mastodon/components/status_content.jsx +++ b/app/javascript/mastodon/components/status_content.jsx @@ -3,6 +3,7 @@ import ImmutablePropTypes from 'react-immutable-proptypes'; import PropTypes from 'prop-types'; import { FormattedMessage, injectIntl } from 'react-intl'; import { Link } from 'react-router-dom'; +import { connect } from 'react-redux'; import classnames from 'classnames'; import PollContainer from 'mastodon/containers/poll_container'; import Icon from 'mastodon/components/icon'; @@ -47,7 +48,12 @@ class TranslateButton extends React.PureComponent { } -export default @injectIntl +const mapStateToProps = state => ({ + languages: state.getIn(['server', 'translationLanguages', 'items']), +}); + +export default @connect(mapStateToProps) +@injectIntl class StatusContent extends React.PureComponent { static contextTypes = { @@ -63,6 +69,7 @@ class StatusContent extends React.PureComponent { onClick: PropTypes.func, collapsable: PropTypes.bool, onCollapsedToggle: PropTypes.func, + languages: ImmutablePropTypes.map, intl: PropTypes.object, }; @@ -220,7 +227,9 @@ class StatusContent extends React.PureComponent { const hidden = this.props.onExpandedToggle ? !this.props.expanded : this.state.hidden; const renderReadMore = this.props.onClick && status.get('collapsed'); - const renderTranslate = this.props.onTranslate && status.get('translatable'); + const contentLocale = intl.locale.replace(/[_-].*/, ''); + const targetLanguages = this.props.languages?.get(status.get('language') || 'und'); + const renderTranslate = this.props.onTranslate && this.context.identity.signedIn && ['public', 'unlisted'].includes(status.get('visibility')) && status.get('contentHtml').length > 0 && targetLanguages?.includes(contentLocale); const content = { __html: status.get('translation') ? status.getIn(['translation', 'content']) : status.get('contentHtml') }; const spoilerContent = { __html: status.get('spoilerHtml') }; diff --git a/app/javascript/mastodon/features/ui/index.jsx b/app/javascript/mastodon/features/ui/index.jsx index 2dd59f95d..083707220 100644 --- a/app/javascript/mastodon/features/ui/index.jsx +++ b/app/javascript/mastodon/features/ui/index.jsx @@ -13,7 +13,7 @@ import { debounce } from 'lodash'; import { uploadCompose, resetCompose, changeComposeSpoilerness } from '../../actions/compose'; import { expandHomeTimeline } from '../../actions/timelines'; import { expandNotifications } from '../../actions/notifications'; -import { fetchServer } from '../../actions/server'; +import { fetchServer, fetchServerTranslationLanguages } from '../../actions/server'; import { clearHeight } from '../../actions/height_cache'; import { focusApp, unfocusApp, changeLayout } from 'mastodon/actions/app'; import { synchronouslySubmitMarkers, submitMarkers, fetchMarkers } from 'mastodon/actions/markers'; @@ -399,6 +399,7 @@ class UI extends React.PureComponent { this.props.dispatch(fetchMarkers()); this.props.dispatch(expandHomeTimeline()); this.props.dispatch(expandNotifications()); + this.props.dispatch(fetchServerTranslationLanguages()); setTimeout(() => this.props.dispatch(fetchServer()), 3000); } diff --git a/app/javascript/mastodon/reducers/server.js b/app/javascript/mastodon/reducers/server.js index db9f2b5e6..909ab2a66 100644 --- a/app/javascript/mastodon/reducers/server.js +++ b/app/javascript/mastodon/reducers/server.js @@ -2,6 +2,9 @@ import { SERVER_FETCH_REQUEST, SERVER_FETCH_SUCCESS, SERVER_FETCH_FAIL, + SERVER_TRANSLATION_LANGUAGES_FETCH_REQUEST, + SERVER_TRANSLATION_LANGUAGES_FETCH_SUCCESS, + SERVER_TRANSLATION_LANGUAGES_FETCH_FAIL, EXTENDED_DESCRIPTION_REQUEST, EXTENDED_DESCRIPTION_SUCCESS, EXTENDED_DESCRIPTION_FAIL, @@ -35,6 +38,12 @@ export default function server(state = initialState, action) { return state.set('server', fromJS(action.server)).setIn(['server', 'isLoading'], false); case SERVER_FETCH_FAIL: return state.setIn(['server', 'isLoading'], false); + case SERVER_TRANSLATION_LANGUAGES_FETCH_REQUEST: + return state.setIn(['translationLanguages', 'isLoading'], true); + case SERVER_TRANSLATION_LANGUAGES_FETCH_SUCCESS: + return state.setIn(['translationLanguages', 'items'], fromJS(action.translationLanguages)).setIn(['translationLanguages', 'isLoading'], false); + case SERVER_TRANSLATION_LANGUAGES_FETCH_FAIL: + return state.setIn(['translationLanguages', 'isLoading'], false); case EXTENDED_DESCRIPTION_REQUEST: return state.setIn(['extendedDescription', 'isLoading'], true); case EXTENDED_DESCRIPTION_SUCCESS: diff --git a/app/lib/translation_service.rb b/app/lib/translation_service.rb index 5ff93674a..bfe5de44f 100644 --- a/app/lib/translation_service.rb +++ b/app/lib/translation_service.rb @@ -21,8 +21,8 @@ class TranslationService ENV['DEEPL_API_KEY'].present? || ENV['LIBRE_TRANSLATE_ENDPOINT'].present? end - def supported?(_source_language, _target_language) - false + def languages + {} end def translate(_text, _source_language, _target_language) diff --git a/app/lib/translation_service/deepl.rb b/app/lib/translation_service/deepl.rb index deff95a1d..afcb7ecb2 100644 --- a/app/lib/translation_service/deepl.rb +++ b/app/lib/translation_service/deepl.rb @@ -17,25 +17,31 @@ class TranslationService::DeepL < TranslationService end end - def supported?(source_language, target_language) - source_language.in?(languages('source')) && target_language.in?(languages('target')) + def languages + source_languages = [nil] + fetch_languages('source') + + # In DeepL, EN and PT are deprecated in favor of EN-GB/EN-US and PT-BR/PT-PT, so + # they are supported but not returned by the API. + target_languages = %w(en pt) + fetch_languages('target') + + source_languages.index_with { |language| target_languages.without(nil, language) } end private - def languages(type) - Rails.cache.fetch("translation_service/deepl/languages/#{type}", expires_in: 7.days, race_condition_ttl: 1.minute) do - request(:get, "/v2/languages?type=#{type}") do |res| - # In DeepL, EN and PT are deprecated in favor of EN-GB/EN-US and PT-BR/PT-PT, so - # they are supported but not returned by the API. - extra = type == 'source' ? [nil] : %w(en pt) - languages = Oj.load(res.body_with_limit).map { |language| language['language'].downcase } - - languages + extra - end + def fetch_languages(type) + request(:get, "/v2/languages?type=#{type}") do |res| + Oj.load(res.body_with_limit).map { |language| normalize_language(language['language']) } end end + def normalize_language(language) + subtags = language.split(/[_-]/) + subtags[0].downcase! + subtags[1]&.upcase! + subtags.join('-') + end + def request(verb, path, **options) req = Request.new(verb, "#{base_url}#{path}", **options) req.add_headers(Authorization: "DeepL-Auth-Key #{@api_key}") diff --git a/app/lib/translation_service/libre_translate.rb b/app/lib/translation_service/libre_translate.rb index 743e4d77f..8bb194a9c 100644 --- a/app/lib/translation_service/libre_translate.rb +++ b/app/lib/translation_service/libre_translate.rb @@ -15,22 +15,18 @@ class TranslationService::LibreTranslate < TranslationService end end - def supported?(source_language, target_language) - languages.key?(source_language) && languages[source_language].include?(target_language) - end - - private - def languages - Rails.cache.fetch('translation_service/libre_translate/languages', expires_in: 7.days, race_condition_ttl: 1.minute) do - request(:get, '/languages') do |res| - languages = Oj.load(res.body_with_limit).to_h { |language| [language['code'], language['targets']] } - languages[nil] = languages.values.flatten.uniq - languages + request(:get, '/languages') do |res| + languages = Oj.load(res.body_with_limit).to_h do |language| + [language['code'], language['targets'].without(language['code'])] end + languages[nil] = languages.values.flatten.uniq.sort + languages end end + private + def request(verb, path, **options) req = Request.new(verb, "#{@base_url}#{path}", allow_local: true, **options) req.add_headers('Content-Type': 'application/json') diff --git a/app/models/status.rb b/app/models/status.rb index dd7ac2edb..e7ea191a8 100644 --- a/app/models/status.rb +++ b/app/models/status.rb @@ -232,16 +232,6 @@ class Status < ApplicationRecord public_visibility? || unlisted_visibility? end - def translatable? - translate_target_locale = I18n.locale.to_s.split(/[_-]/).first - - distributable? && - content.present? && - language != translate_target_locale && - TranslationService.configured? && - TranslationService.configured.supported?(language, translate_target_locale) - end - alias sign? distributable? def with_media? diff --git a/app/serializers/rest/status_serializer.rb b/app/serializers/rest/status_serializer.rb index a422f5b25..e0b8f32a6 100644 --- a/app/serializers/rest/status_serializer.rb +++ b/app/serializers/rest/status_serializer.rb @@ -4,7 +4,7 @@ class REST::StatusSerializer < ActiveModel::Serializer include FormattingHelper attributes :id, :created_at, :in_reply_to_id, :in_reply_to_account_id, - :sensitive, :spoiler_text, :visibility, :language, :translatable, + :sensitive, :spoiler_text, :visibility, :language, :uri, :url, :replies_count, :reblogs_count, :favourites_count, :edited_at @@ -50,10 +50,6 @@ class REST::StatusSerializer < ActiveModel::Serializer object.account.user_shows_application? || (current_user? && current_user.account_id == object.account_id) end - def translatable - current_user? && object.translatable? - end - def visibility # This visibility is masked behind "private" # to avoid API changes because there are no diff --git a/app/services/translate_status_service.rb b/app/services/translate_status_service.rb index 92d8b62a0..796f13a0d 100644 --- a/app/services/translate_status_service.rb +++ b/app/services/translate_status_service.rb @@ -6,19 +6,29 @@ class TranslateStatusService < BaseService include FormattingHelper def call(status, target_language) - raise Mastodon::NotPermittedError unless status.translatable? - @status = status @content = status_content_format(@status) @target_language = target_language + raise Mastodon::NotPermittedError unless permitted? + Rails.cache.fetch("translations/#{@status.language}/#{@target_language}/#{content_hash}", expires_in: CACHE_TTL) { translation_backend.translate(@content, @status.language, @target_language) } end private def translation_backend - TranslationService.configured + @translation_backend ||= TranslationService.configured + end + + def permitted? + return false unless @status.distributable? && @status.content.present? && TranslationService.configured? + + languages[@status.language]&.include?(@target_language) + end + + def languages + Rails.cache.fetch('translation_service/languages', expires_in: 7.days, race_condition_ttl: 1.hour) { TranslationService.configured.languages } end def content_hash diff --git a/config/routes.rb b/config/routes.rb index 530b46a5a..ea595e1e1 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -546,6 +546,7 @@ Rails.application.routes.draw do resources :domain_blocks, only: [:index], controller: 'instances/domain_blocks' resource :privacy_policy, only: [:show], controller: 'instances/privacy_policies' resource :extended_description, only: [:show], controller: 'instances/extended_descriptions' + resource :translation_languages, only: [:show], controller: 'instances/translation_languages' resource :activity, only: [:show], controller: 'instances/activity' end diff --git a/spec/controllers/api/v1/instances/translation_languages_controller_spec.rb b/spec/controllers/api/v1/instances/translation_languages_controller_spec.rb new file mode 100644 index 000000000..5b7e4abb6 --- /dev/null +++ b/spec/controllers/api/v1/instances/translation_languages_controller_spec.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +require 'rails_helper' + +describe Api::V1::Instances::TranslationLanguagesController do + describe 'GET #show' do + context 'when no translation service is configured' do + it 'returns empty language matrix' do + get :show + + expect(response).to have_http_status(200) + expect(body_as_json).to eq({}) + end + end + + context 'when a translation service is configured' do + before do + service = instance_double(TranslationService::DeepL, languages: { nil => %w(en de), 'en' => ['de'] }) + allow(TranslationService).to receive(:configured?).and_return(true) + allow(TranslationService).to receive(:configured).and_return(service) + end + + it 'returns language matrix' do + get :show + + expect(response).to have_http_status(200) + expect(body_as_json).to eq({ und: %w(en de), en: ['de'] }) + end + end + end +end diff --git a/spec/controllers/api/v1/statuses/translations_controller_spec.rb b/spec/controllers/api/v1/statuses/translations_controller_spec.rb index 2deea9fc0..8495779bf 100644 --- a/spec/controllers/api/v1/statuses/translations_controller_spec.rb +++ b/spec/controllers/api/v1/statuses/translations_controller_spec.rb @@ -19,9 +19,10 @@ describe Api::V1::Statuses::TranslationsController do before do translation = TranslationService::Translation.new(text: 'Hello') - service = instance_double(TranslationService::DeepL, translate: translation, supported?: true) + service = instance_double(TranslationService::DeepL, translate: translation) allow(TranslationService).to receive(:configured?).and_return(true) allow(TranslationService).to receive(:configured).and_return(service) + Rails.cache.write('translation_service/languages', { 'es' => ['en'] }) post :create, params: { status_id: status.id } end diff --git a/spec/lib/translation_service/deepl_spec.rb b/spec/lib/translation_service/deepl_spec.rb index aa2473186..2363f8f13 100644 --- a/spec/lib/translation_service/deepl_spec.rb +++ b/spec/lib/translation_service/deepl_spec.rb @@ -16,29 +16,6 @@ RSpec.describe TranslationService::DeepL do ) end - describe '#supported?' do - it 'supports included languages as source and target languages' do - expect(service.supported?('uk', 'en')).to be true - end - - it 'supports auto-detecting source language' do - expect(service.supported?(nil, 'en')).to be true - end - - it 'supports "en" and "pt" as target languages though not included in language list' do - expect(service.supported?('uk', 'en')).to be true - expect(service.supported?('uk', 'pt')).to be true - end - - it 'does not support non-included language as target language' do - expect(service.supported?('uk', 'nl')).to be false - end - - it 'does not support non-included language as source language' do - expect(service.supported?('da', 'en')).to be false - end - end - describe '#translate' do it 'returns translation with specified source language' do stub_request(:post, 'https://api.deepl.com/v2/translate') @@ -63,13 +40,18 @@ RSpec.describe TranslationService::DeepL do end end - describe '#languages?' do + describe '#languages' do it 'returns source languages' do - expect(service.send(:languages, 'source')).to eq ['en', 'uk', nil] + expect(service.languages.keys).to eq [nil, 'en', 'uk'] + end + + it 'returns target languages for each source language' do + expect(service.languages['en']).to eq %w(pt en-GB zh) + expect(service.languages['uk']).to eq %w(en pt en-GB zh) end - it 'returns target languages' do - expect(service.send(:languages, 'target')).to eq %w(en-gb zh en pt) + it 'returns target languages for auto-detection' do + expect(service.languages[nil]).to eq %w(en pt en-GB zh) end end diff --git a/spec/lib/translation_service/libre_translate_spec.rb b/spec/lib/translation_service/libre_translate_spec.rb index a6cb01884..fbd726a7e 100644 --- a/spec/lib/translation_service/libre_translate_spec.rb +++ b/spec/lib/translation_service/libre_translate_spec.rb @@ -7,41 +7,24 @@ RSpec.describe TranslationService::LibreTranslate do before do stub_request(:get, 'https://libretranslate.example.com/languages').to_return( - body: '[{"code": "en","name": "English","targets": ["de","es"]},{"code": "da","name": "Danish","targets": ["en","de"]}]' + body: '[{"code": "en","name": "English","targets": ["de","en","es"]},{"code": "da","name": "Danish","targets": ["en","pt"]}]' ) end - describe '#supported?' do - it 'supports included language pair' do - expect(service.supported?('en', 'de')).to be true - end - - it 'does not support reversed language pair' do - expect(service.supported?('de', 'en')).to be false - end - - it 'supports auto-detecting source language' do - expect(service.supported?(nil, 'de')).to be true - end - - it 'does not support auto-detecting for unsupported target language' do - expect(service.supported?(nil, 'pt')).to be false - end - end - describe '#languages' do - subject(:languages) { service.send(:languages) } + subject(:languages) { service.languages } - it 'includes supported source languages' do + it 'returns source languages' do expect(languages.keys).to eq ['en', 'da', nil] end - it 'includes supported target languages for source language' do + it 'returns target languages for each source language' do expect(languages['en']).to eq %w(de es) + expect(languages['da']).to eq %w(en pt) end - it 'includes supported target languages for auto-detected language' do - expect(languages[nil]).to eq %w(de es en) + it 'returns target languages for auto-detected language' do + expect(languages[nil]).to eq %w(de en es pt) end end diff --git a/spec/models/status_spec.rb b/spec/models/status_spec.rb index 1f6cfc796..1e58c6d0d 100644 --- a/spec/models/status_spec.rb +++ b/spec/models/status_spec.rb @@ -114,85 +114,6 @@ RSpec.describe Status, type: :model do end end - describe '#translatable?' do - before do - allow(TranslationService).to receive(:configured?).and_return(true) - allow(TranslationService).to receive(:configured).and_return(TranslationService.new) - allow(TranslationService.configured).to receive(:supported?).with('es', 'en').and_return(true) - - subject.language = 'es' - subject.visibility = :public - end - - context 'all conditions are satisfied' do - it 'returns true' do - expect(subject.translatable?).to be true - end - end - - context 'translation service is not configured' do - it 'returns false' do - allow(TranslationService).to receive(:configured?).and_return(false) - allow(TranslationService).to receive(:configured).and_raise(TranslationService::NotConfiguredError) - expect(subject.translatable?).to be false - end - end - - context 'status language is nil' do - it 'returns true' do - subject.language = nil - allow(TranslationService.configured).to receive(:supported?).with(nil, 'en').and_return(true) - expect(subject.translatable?).to be true - end - end - - context 'status language is same as default locale' do - it 'returns false' do - subject.language = I18n.locale - expect(subject.translatable?).to be false - end - end - - context 'status language is unsupported' do - it 'returns false' do - subject.language = 'af' - allow(TranslationService.configured).to receive(:supported?).with('af', 'en').and_return(false) - expect(subject.translatable?).to be false - end - end - - context 'default locale is unsupported' do - it 'returns false' do - allow(TranslationService.configured).to receive(:supported?).with('es', 'af').and_return(false) - I18n.with_locale('af') do - expect(subject.translatable?).to be false - end - end - end - - context 'default locale has region' do - it 'returns true' do - I18n.with_locale('en-GB') do - expect(subject.translatable?).to be true - end - end - end - - context 'status text is blank' do - it 'returns false' do - subject.text = ' ' - expect(subject.translatable?).to be false - end - end - - context 'status visiblity is hidden' do - it 'returns false' do - subject.visibility = 'limited' - expect(subject.translatable?).to be false - end - end - end - describe '#content' do it 'returns the text of the status if it is not a reblog' do expect(subject.content).to eql subject.text diff --git a/spec/presenters/instance_presenter_spec.rb b/spec/presenters/instance_presenter_spec.rb index 795abd8b4..2a1d668ce 100644 --- a/spec/presenters/instance_presenter_spec.rb +++ b/spec/presenters/instance_presenter_spec.rb @@ -3,7 +3,7 @@ require 'rails_helper' describe InstancePresenter do - let(:instance_presenter) { InstancePresenter.new } + let(:instance_presenter) { described_class.new } describe '#description' do around do |example| |