From bd047acc356671727c112336bb237f979bba517d Mon Sep 17 00:00:00 2001 From: Christian Schmidt Date: Thu, 16 Mar 2023 11:07:24 +0100 Subject: Replace `Status#translatable?` with language matrix in separate endpoint (#24037) --- .../instances/translation_languages_controller.rb | 23 +++++++++++++++++ app/javascript/mastodon/actions/server.js | 27 +++++++++++++++++++ .../mastodon/components/status_content.jsx | 13 ++++++++-- app/javascript/mastodon/features/ui/index.jsx | 3 ++- app/javascript/mastodon/reducers/server.js | 9 +++++++ app/lib/translation_service.rb | 4 +-- app/lib/translation_service/deepl.rb | 30 +++++++++++++--------- app/lib/translation_service/libre_translate.rb | 18 +++++-------- app/models/status.rb | 10 -------- app/serializers/rest/status_serializer.rb | 6 +---- app/services/translate_status_service.rb | 16 +++++++++--- 11 files changed, 113 insertions(+), 46 deletions(-) create mode 100644 app/controllers/api/v1/instances/translation_languages_controller.rb (limited to 'app') diff --git a/app/controllers/api/v1/instances/translation_languages_controller.rb b/app/controllers/api/v1/instances/translation_languages_controller.rb new file mode 100644 index 000000000..3910a499e --- /dev/null +++ b/app/controllers/api/v1/instances/translation_languages_controller.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +class Api::V1::Instances::TranslationLanguagesController < Api::BaseController + skip_before_action :require_authenticated_user!, unless: :whitelist_mode? + + before_action :set_languages + + def show + expires_in 1.day, public: true + render json: @languages + end + + private + + def set_languages + if TranslationService.configured? + @languages = Rails.cache.fetch('translation_service/languages', expires_in: 7.days, race_condition_ttl: 1.hour) { TranslationService.configured.languages } + @languages['und'] = @languages.delete(nil) if @languages.key?(nil) + else + @languages = {} + end + end +end diff --git a/app/javascript/mastodon/actions/server.js b/app/javascript/mastodon/actions/server.js index 31d4aea10..091af0f0f 100644 --- a/app/javascript/mastodon/actions/server.js +++ b/app/javascript/mastodon/actions/server.js @@ -5,6 +5,10 @@ export const SERVER_FETCH_REQUEST = 'Server_FETCH_REQUEST'; export const SERVER_FETCH_SUCCESS = 'Server_FETCH_SUCCESS'; export const SERVER_FETCH_FAIL = 'Server_FETCH_FAIL'; +export const SERVER_TRANSLATION_LANGUAGES_FETCH_REQUEST = 'SERVER_TRANSLATION_LANGUAGES_FETCH_REQUEST'; +export const SERVER_TRANSLATION_LANGUAGES_FETCH_SUCCESS = 'SERVER_TRANSLATION_LANGUAGES_FETCH_SUCCESS'; +export const SERVER_TRANSLATION_LANGUAGES_FETCH_FAIL = 'SERVER_TRANSLATION_LANGUAGES_FETCH_FAIL'; + export const EXTENDED_DESCRIPTION_REQUEST = 'EXTENDED_DESCRIPTION_REQUEST'; export const EXTENDED_DESCRIPTION_SUCCESS = 'EXTENDED_DESCRIPTION_SUCCESS'; export const EXTENDED_DESCRIPTION_FAIL = 'EXTENDED_DESCRIPTION_FAIL'; @@ -37,6 +41,29 @@ const fetchServerFail = error => ({ error, }); +export const fetchServerTranslationLanguages = () => (dispatch, getState) => { + dispatch(fetchServerTranslationLanguagesRequest()); + + api(getState) + .get('/api/v1/instance/translation_languages').then(({ data }) => { + dispatch(fetchServerTranslationLanguagesSuccess(data)); + }).catch(err => dispatch(fetchServerTranslationLanguagesFail(err))); +}; + +const fetchServerTranslationLanguagesRequest = () => ({ + type: SERVER_TRANSLATION_LANGUAGES_FETCH_REQUEST, +}); + +const fetchServerTranslationLanguagesSuccess = translationLanguages => ({ + type: SERVER_TRANSLATION_LANGUAGES_FETCH_SUCCESS, + translationLanguages, +}); + +const fetchServerTranslationLanguagesFail = error => ({ + type: SERVER_TRANSLATION_LANGUAGES_FETCH_FAIL, + error, +}); + export const fetchExtendedDescription = () => (dispatch, getState) => { dispatch(fetchExtendedDescriptionRequest()); diff --git a/app/javascript/mastodon/components/status_content.jsx b/app/javascript/mastodon/components/status_content.jsx index f9c9fe079..67a487b00 100644 --- a/app/javascript/mastodon/components/status_content.jsx +++ b/app/javascript/mastodon/components/status_content.jsx @@ -3,6 +3,7 @@ import ImmutablePropTypes from 'react-immutable-proptypes'; import PropTypes from 'prop-types'; import { FormattedMessage, injectIntl } from 'react-intl'; import { Link } from 'react-router-dom'; +import { connect } from 'react-redux'; import classnames from 'classnames'; import PollContainer from 'mastodon/containers/poll_container'; import Icon from 'mastodon/components/icon'; @@ -47,7 +48,12 @@ class TranslateButton extends React.PureComponent { } -export default @injectIntl +const mapStateToProps = state => ({ + languages: state.getIn(['server', 'translationLanguages', 'items']), +}); + +export default @connect(mapStateToProps) +@injectIntl class StatusContent extends React.PureComponent { static contextTypes = { @@ -63,6 +69,7 @@ class StatusContent extends React.PureComponent { onClick: PropTypes.func, collapsable: PropTypes.bool, onCollapsedToggle: PropTypes.func, + languages: ImmutablePropTypes.map, intl: PropTypes.object, }; @@ -220,7 +227,9 @@ class StatusContent extends React.PureComponent { const hidden = this.props.onExpandedToggle ? !this.props.expanded : this.state.hidden; const renderReadMore = this.props.onClick && status.get('collapsed'); - const renderTranslate = this.props.onTranslate && status.get('translatable'); + const contentLocale = intl.locale.replace(/[_-].*/, ''); + const targetLanguages = this.props.languages?.get(status.get('language') || 'und'); + const renderTranslate = this.props.onTranslate && this.context.identity.signedIn && ['public', 'unlisted'].includes(status.get('visibility')) && status.get('contentHtml').length > 0 && targetLanguages?.includes(contentLocale); const content = { __html: status.get('translation') ? status.getIn(['translation', 'content']) : status.get('contentHtml') }; const spoilerContent = { __html: status.get('spoilerHtml') }; diff --git a/app/javascript/mastodon/features/ui/index.jsx b/app/javascript/mastodon/features/ui/index.jsx index 2dd59f95d..083707220 100644 --- a/app/javascript/mastodon/features/ui/index.jsx +++ b/app/javascript/mastodon/features/ui/index.jsx @@ -13,7 +13,7 @@ import { debounce } from 'lodash'; import { uploadCompose, resetCompose, changeComposeSpoilerness } from '../../actions/compose'; import { expandHomeTimeline } from '../../actions/timelines'; import { expandNotifications } from '../../actions/notifications'; -import { fetchServer } from '../../actions/server'; +import { fetchServer, fetchServerTranslationLanguages } from '../../actions/server'; import { clearHeight } from '../../actions/height_cache'; import { focusApp, unfocusApp, changeLayout } from 'mastodon/actions/app'; import { synchronouslySubmitMarkers, submitMarkers, fetchMarkers } from 'mastodon/actions/markers'; @@ -399,6 +399,7 @@ class UI extends React.PureComponent { this.props.dispatch(fetchMarkers()); this.props.dispatch(expandHomeTimeline()); this.props.dispatch(expandNotifications()); + this.props.dispatch(fetchServerTranslationLanguages()); setTimeout(() => this.props.dispatch(fetchServer()), 3000); } diff --git a/app/javascript/mastodon/reducers/server.js b/app/javascript/mastodon/reducers/server.js index db9f2b5e6..909ab2a66 100644 --- a/app/javascript/mastodon/reducers/server.js +++ b/app/javascript/mastodon/reducers/server.js @@ -2,6 +2,9 @@ import { SERVER_FETCH_REQUEST, SERVER_FETCH_SUCCESS, SERVER_FETCH_FAIL, + SERVER_TRANSLATION_LANGUAGES_FETCH_REQUEST, + SERVER_TRANSLATION_LANGUAGES_FETCH_SUCCESS, + SERVER_TRANSLATION_LANGUAGES_FETCH_FAIL, EXTENDED_DESCRIPTION_REQUEST, EXTENDED_DESCRIPTION_SUCCESS, EXTENDED_DESCRIPTION_FAIL, @@ -35,6 +38,12 @@ export default function server(state = initialState, action) { return state.set('server', fromJS(action.server)).setIn(['server', 'isLoading'], false); case SERVER_FETCH_FAIL: return state.setIn(['server', 'isLoading'], false); + case SERVER_TRANSLATION_LANGUAGES_FETCH_REQUEST: + return state.setIn(['translationLanguages', 'isLoading'], true); + case SERVER_TRANSLATION_LANGUAGES_FETCH_SUCCESS: + return state.setIn(['translationLanguages', 'items'], fromJS(action.translationLanguages)).setIn(['translationLanguages', 'isLoading'], false); + case SERVER_TRANSLATION_LANGUAGES_FETCH_FAIL: + return state.setIn(['translationLanguages', 'isLoading'], false); case EXTENDED_DESCRIPTION_REQUEST: return state.setIn(['extendedDescription', 'isLoading'], true); case EXTENDED_DESCRIPTION_SUCCESS: diff --git a/app/lib/translation_service.rb b/app/lib/translation_service.rb index 5ff93674a..bfe5de44f 100644 --- a/app/lib/translation_service.rb +++ b/app/lib/translation_service.rb @@ -21,8 +21,8 @@ class TranslationService ENV['DEEPL_API_KEY'].present? || ENV['LIBRE_TRANSLATE_ENDPOINT'].present? end - def supported?(_source_language, _target_language) - false + def languages + {} end def translate(_text, _source_language, _target_language) diff --git a/app/lib/translation_service/deepl.rb b/app/lib/translation_service/deepl.rb index deff95a1d..afcb7ecb2 100644 --- a/app/lib/translation_service/deepl.rb +++ b/app/lib/translation_service/deepl.rb @@ -17,25 +17,31 @@ class TranslationService::DeepL < TranslationService end end - def supported?(source_language, target_language) - source_language.in?(languages('source')) && target_language.in?(languages('target')) + def languages + source_languages = [nil] + fetch_languages('source') + + # In DeepL, EN and PT are deprecated in favor of EN-GB/EN-US and PT-BR/PT-PT, so + # they are supported but not returned by the API. + target_languages = %w(en pt) + fetch_languages('target') + + source_languages.index_with { |language| target_languages.without(nil, language) } end private - def languages(type) - Rails.cache.fetch("translation_service/deepl/languages/#{type}", expires_in: 7.days, race_condition_ttl: 1.minute) do - request(:get, "/v2/languages?type=#{type}") do |res| - # In DeepL, EN and PT are deprecated in favor of EN-GB/EN-US and PT-BR/PT-PT, so - # they are supported but not returned by the API. - extra = type == 'source' ? [nil] : %w(en pt) - languages = Oj.load(res.body_with_limit).map { |language| language['language'].downcase } - - languages + extra - end + def fetch_languages(type) + request(:get, "/v2/languages?type=#{type}") do |res| + Oj.load(res.body_with_limit).map { |language| normalize_language(language['language']) } end end + def normalize_language(language) + subtags = language.split(/[_-]/) + subtags[0].downcase! + subtags[1]&.upcase! + subtags.join('-') + end + def request(verb, path, **options) req = Request.new(verb, "#{base_url}#{path}", **options) req.add_headers(Authorization: "DeepL-Auth-Key #{@api_key}") diff --git a/app/lib/translation_service/libre_translate.rb b/app/lib/translation_service/libre_translate.rb index 743e4d77f..8bb194a9c 100644 --- a/app/lib/translation_service/libre_translate.rb +++ b/app/lib/translation_service/libre_translate.rb @@ -15,22 +15,18 @@ class TranslationService::LibreTranslate < TranslationService end end - def supported?(source_language, target_language) - languages.key?(source_language) && languages[source_language].include?(target_language) - end - - private - def languages - Rails.cache.fetch('translation_service/libre_translate/languages', expires_in: 7.days, race_condition_ttl: 1.minute) do - request(:get, '/languages') do |res| - languages = Oj.load(res.body_with_limit).to_h { |language| [language['code'], language['targets']] } - languages[nil] = languages.values.flatten.uniq - languages + request(:get, '/languages') do |res| + languages = Oj.load(res.body_with_limit).to_h do |language| + [language['code'], language['targets'].without(language['code'])] end + languages[nil] = languages.values.flatten.uniq.sort + languages end end + private + def request(verb, path, **options) req = Request.new(verb, "#{@base_url}#{path}", allow_local: true, **options) req.add_headers('Content-Type': 'application/json') diff --git a/app/models/status.rb b/app/models/status.rb index dd7ac2edb..e7ea191a8 100644 --- a/app/models/status.rb +++ b/app/models/status.rb @@ -232,16 +232,6 @@ class Status < ApplicationRecord public_visibility? || unlisted_visibility? end - def translatable? - translate_target_locale = I18n.locale.to_s.split(/[_-]/).first - - distributable? && - content.present? && - language != translate_target_locale && - TranslationService.configured? && - TranslationService.configured.supported?(language, translate_target_locale) - end - alias sign? distributable? def with_media? diff --git a/app/serializers/rest/status_serializer.rb b/app/serializers/rest/status_serializer.rb index a422f5b25..e0b8f32a6 100644 --- a/app/serializers/rest/status_serializer.rb +++ b/app/serializers/rest/status_serializer.rb @@ -4,7 +4,7 @@ class REST::StatusSerializer < ActiveModel::Serializer include FormattingHelper attributes :id, :created_at, :in_reply_to_id, :in_reply_to_account_id, - :sensitive, :spoiler_text, :visibility, :language, :translatable, + :sensitive, :spoiler_text, :visibility, :language, :uri, :url, :replies_count, :reblogs_count, :favourites_count, :edited_at @@ -50,10 +50,6 @@ class REST::StatusSerializer < ActiveModel::Serializer object.account.user_shows_application? || (current_user? && current_user.account_id == object.account_id) end - def translatable - current_user? && object.translatable? - end - def visibility # This visibility is masked behind "private" # to avoid API changes because there are no diff --git a/app/services/translate_status_service.rb b/app/services/translate_status_service.rb index 92d8b62a0..796f13a0d 100644 --- a/app/services/translate_status_service.rb +++ b/app/services/translate_status_service.rb @@ -6,19 +6,29 @@ class TranslateStatusService < BaseService include FormattingHelper def call(status, target_language) - raise Mastodon::NotPermittedError unless status.translatable? - @status = status @content = status_content_format(@status) @target_language = target_language + raise Mastodon::NotPermittedError unless permitted? + Rails.cache.fetch("translations/#{@status.language}/#{@target_language}/#{content_hash}", expires_in: CACHE_TTL) { translation_backend.translate(@content, @status.language, @target_language) } end private def translation_backend - TranslationService.configured + @translation_backend ||= TranslationService.configured + end + + def permitted? + return false unless @status.distributable? && @status.content.present? && TranslationService.configured? + + languages[@status.language]&.include?(@target_language) + end + + def languages + Rails.cache.fetch('translation_service/languages', expires_in: 7.days, race_condition_ttl: 1.hour) { TranslationService.configured.languages } end def content_hash -- cgit