From 0d6b878808a02aa4a544e894f06419c0f612c163 Mon Sep 17 00:00:00 2001 From: Eugen Rochko Date: Fri, 23 Sep 2022 23:00:12 +0200 Subject: Add user content translations with configurable backends (#19218) --- app/lib/translation_service/deepl.rb | 53 ++++++++++++++++++++++++++ app/lib/translation_service/libre_translate.rb | 43 +++++++++++++++++++++ app/lib/translation_service/translation.rb | 5 +++ 3 files changed, 101 insertions(+) create mode 100644 app/lib/translation_service/deepl.rb create mode 100644 app/lib/translation_service/libre_translate.rb create mode 100644 app/lib/translation_service/translation.rb (limited to 'app/lib/translation_service') diff --git a/app/lib/translation_service/deepl.rb b/app/lib/translation_service/deepl.rb new file mode 100644 index 000000000..89ccf01e5 --- /dev/null +++ b/app/lib/translation_service/deepl.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +class TranslationService::DeepL < TranslationService + include JsonLdHelper + + def initialize(plan, api_key) + super() + + @plan = plan + @api_key = api_key + end + + def translate(text, source_language, target_language) + request(text, source_language, target_language).perform do |res| + case res.code + when 429 + raise TooManyRequestsError + when 456 + raise QuotaExceededError + when 200...300 + transform_response(res.body_with_limit) + else + raise UnexpectedResponseError + end + end + end + + private + + def request(text, source_language, target_language) + req = Request.new(:post, endpoint_url, form: { text: text, source_lang: source_language.upcase, target_lang: target_language, tag_handling: 'html' }) + req.add_headers('Authorization': "DeepL-Auth-Key #{@api_key}") + req + end + + def endpoint_url + if @plan == 'free' + 'https://api-free.deepl.com/v2/translate' + else + 'https://api.deepl.com/v2/translate' + end + end + + def transform_response(str) + json = Oj.load(str, mode: :strict) + + raise UnexpectedResponseError unless json.is_a?(Hash) + + Translation.new(text: json.dig('translations', 0, 'text'), detected_source_language: json.dig('translations', 0, 'detected_source_language')&.downcase) + rescue Oj::ParseError + raise UnexpectedResponseError + end +end diff --git a/app/lib/translation_service/libre_translate.rb b/app/lib/translation_service/libre_translate.rb new file mode 100644 index 000000000..66acdeed7 --- /dev/null +++ b/app/lib/translation_service/libre_translate.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +class TranslationService::LibreTranslate < TranslationService + def initialize(base_url, api_key) + super() + + @base_url = base_url + @api_key = api_key + end + + def translate(text, source_language, target_language) + request(text, source_language, target_language).perform do |res| + case res.code + when 429 + raise TooManyRequestsError + when 403 + raise QuotaExceededError + when 200...300 + transform_response(res.body_with_limit, source_language) + else + raise UnexpectedResponseError + end + end + end + + private + + def request(text, source_language, target_language) + req = Request.new(:post, "#{@base_url}/translate", body: Oj.dump(q: text, source: source_language, target: target_language, format: 'html', api_key: @api_key)) + req.add_headers('Content-Type': 'application/json') + req + end + + def transform_response(str, source_language) + json = Oj.load(str, mode: :strict) + + raise UnexpectedResponseError unless json.is_a?(Hash) + + Translation.new(text: json['translatedText'], detected_source_language: source_language) + rescue Oj::ParseError + raise UnexpectedResponseError + end +end diff --git a/app/lib/translation_service/translation.rb b/app/lib/translation_service/translation.rb new file mode 100644 index 000000000..a55b82574 --- /dev/null +++ b/app/lib/translation_service/translation.rb @@ -0,0 +1,5 @@ +# frozen_string_literal: true + +class TranslationService::Translation < ActiveModelSerializers::Model + attributes :text, :detected_source_language +end -- cgit From d86dd067ceb1dc05e67417fa1d7f1e84519b2fca Mon Sep 17 00:00:00 2001 From: Yamagishi Kazutoshi Date: Wed, 28 Sep 2022 06:33:56 +0900 Subject: Fix auto detect language for translate service (#19244) --- .devcontainer/devcontainer.json | 2 +- .devcontainer/docker-compose.yml | 7 +++++++ app/lib/translation_service/deepl.rb | 2 +- app/lib/translation_service/libre_translate.rb | 3 ++- 4 files changed, 11 insertions(+), 3 deletions(-) (limited to 'app/lib/translation_service') diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 9d9e54d4f..47497794f 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -20,7 +20,7 @@ "forwardPorts": [3000, 4000], // Use 'postCreateCommand' to run commands after the container is created. - "postCreateCommand": "bundle install --path vendor/bundle && yarn install && ./bin/rails db:setup", + "postCreateCommand": "bundle install --path vendor/bundle && yarn install && git checkout -- Gemfile.lock && ./bin/rails db:setup", // Comment out to connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. "remoteUser": "vscode" diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index 538f6cccd..46f42c454 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -27,6 +27,7 @@ services: ES_ENABLED: 'true' ES_HOST: es ES_PORT: '9200' + LIBRE_TRANSLATE_ENDPOINT: http://libretranslate:5000 # Overrides default command so things don't shut down after the process ends. command: sleep infinity networks: @@ -72,6 +73,12 @@ services: soft: -1 hard: -1 + libretranslate: + image: libretranslate/libretranslate:v1.2.9 + restart: unless-stopped + networks: + - internal_network + volumes: postgres-data: redis-data: diff --git a/app/lib/translation_service/deepl.rb b/app/lib/translation_service/deepl.rb index 89ccf01e5..b75b604a8 100644 --- a/app/lib/translation_service/deepl.rb +++ b/app/lib/translation_service/deepl.rb @@ -28,7 +28,7 @@ class TranslationService::DeepL < TranslationService private def request(text, source_language, target_language) - req = Request.new(:post, endpoint_url, form: { text: text, source_lang: source_language.upcase, target_lang: target_language, tag_handling: 'html' }) + req = Request.new(:post, endpoint_url, form: { text: text, source_lang: source_language&.upcase, target_lang: target_language, tag_handling: 'html' }) req.add_headers('Authorization': "DeepL-Auth-Key #{@api_key}") req end diff --git a/app/lib/translation_service/libre_translate.rb b/app/lib/translation_service/libre_translate.rb index 66acdeed7..8cf26f868 100644 --- a/app/lib/translation_service/libre_translate.rb +++ b/app/lib/translation_service/libre_translate.rb @@ -26,7 +26,8 @@ class TranslationService::LibreTranslate < TranslationService private def request(text, source_language, target_language) - req = Request.new(:post, "#{@base_url}/translate", body: Oj.dump(q: text, source: source_language, target: target_language, format: 'html', api_key: @api_key)) + body = Oj.dump(q: text, source: source_language.presence || 'auto', target: target_language, format: 'html', api_key: @api_key) + req = Request.new(:post, "#{@base_url}/translate", body: body) req.add_headers('Content-Type': 'application/json') req end -- cgit From 30453fab80d55fc10766f0e067c31d96753ccfda Mon Sep 17 00:00:00 2001 From: Claire Date: Mon, 24 Oct 2022 18:37:57 +0200 Subject: Add mention of the translation provider when translating a post (#19433) --- app/javascript/mastodon/components/status_content.js | 2 +- app/lib/translation_service/deepl.rb | 2 +- app/lib/translation_service/libre_translate.rb | 2 +- app/lib/translation_service/translation.rb | 2 +- app/serializers/rest/translation_serializer.rb | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) (limited to 'app/lib/translation_service') diff --git a/app/javascript/mastodon/components/status_content.js b/app/javascript/mastodon/components/status_content.js index 0ed79e658..56e8432b6 100644 --- a/app/javascript/mastodon/components/status_content.js +++ b/app/javascript/mastodon/components/status_content.js @@ -208,7 +208,7 @@ class StatusContent extends React.PureComponent { const translateButton = ( ); diff --git a/app/lib/translation_service/deepl.rb b/app/lib/translation_service/deepl.rb index b75b604a8..537fd24c0 100644 --- a/app/lib/translation_service/deepl.rb +++ b/app/lib/translation_service/deepl.rb @@ -46,7 +46,7 @@ class TranslationService::DeepL < TranslationService raise UnexpectedResponseError unless json.is_a?(Hash) - Translation.new(text: json.dig('translations', 0, 'text'), detected_source_language: json.dig('translations', 0, 'detected_source_language')&.downcase) + Translation.new(text: json.dig('translations', 0, 'text'), detected_source_language: json.dig('translations', 0, 'detected_source_language')&.downcase, provider: 'DeepL.com') rescue Oj::ParseError raise UnexpectedResponseError end diff --git a/app/lib/translation_service/libre_translate.rb b/app/lib/translation_service/libre_translate.rb index 8cf26f868..43576e306 100644 --- a/app/lib/translation_service/libre_translate.rb +++ b/app/lib/translation_service/libre_translate.rb @@ -37,7 +37,7 @@ class TranslationService::LibreTranslate < TranslationService raise UnexpectedResponseError unless json.is_a?(Hash) - Translation.new(text: json['translatedText'], detected_source_language: source_language) + Translation.new(text: json['translatedText'], detected_source_language: source_language, provider: 'LibreTranslate') rescue Oj::ParseError raise UnexpectedResponseError end diff --git a/app/lib/translation_service/translation.rb b/app/lib/translation_service/translation.rb index a55b82574..19318c7e9 100644 --- a/app/lib/translation_service/translation.rb +++ b/app/lib/translation_service/translation.rb @@ -1,5 +1,5 @@ # frozen_string_literal: true class TranslationService::Translation < ActiveModelSerializers::Model - attributes :text, :detected_source_language + attributes :text, :detected_source_language, :provider end diff --git a/app/serializers/rest/translation_serializer.rb b/app/serializers/rest/translation_serializer.rb index a06f23f32..05ededc95 100644 --- a/app/serializers/rest/translation_serializer.rb +++ b/app/serializers/rest/translation_serializer.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true class REST::TranslationSerializer < ActiveModel::Serializer - attributes :content, :detected_source_language + attributes :content, :detected_source_language, :provider def content object.text -- cgit