Only offer translation for supported languages (#23879)

This commit is contained in:
Christian Schmidt 2023-03-03 21:06:31 +01:00 committed by GitHub
parent 0872f3e3d7
commit 5a8c651e8f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 336 additions and 31 deletions

View File

@ -97,6 +97,10 @@ Rails/Exit:
- 'lib/mastodon/cli_helper.rb'
- 'lib/cli.rb'
RSpec/FilePath:
CustomTransform:
DeepL: deepl
RSpec/NotToNot:
EnforcedStyle: to_not

View File

@ -6,7 +6,7 @@ import { Link } from 'react-router-dom';
import classnames from 'classnames';
import PollContainer from 'mastodon/containers/poll_container';
import Icon from 'mastodon/components/icon';
import { autoPlayGif, languages as preloadedLanguages, translationEnabled } from 'mastodon/initial_state';
import { autoPlayGif, languages as preloadedLanguages } from 'mastodon/initial_state';
const MAX_HEIGHT = 706; // 22px * 32 (+ 2px padding at the top)
@ -220,7 +220,7 @@ class StatusContent extends React.PureComponent {
const hidden = this.props.onExpandedToggle ? !this.props.expanded : this.state.hidden;
const renderReadMore = this.props.onClick && status.get('collapsed');
const renderTranslate = translationEnabled && this.context.identity.signedIn && this.props.onTranslate && ['public', 'unlisted'].includes(status.get('visibility')) && status.get('contentHtml').length > 0 && status.get('language') !== null && intl.locale !== status.get('language');
const renderTranslate = this.props.onTranslate && status.get('translatable');
const content = { __html: status.get('translation') ? status.getIn(['translation', 'content']) : status.get('contentHtml') };
const spoilerContent = { __html: status.get('spoilerHtml') };

View File

@ -80,7 +80,6 @@
* @property {boolean} use_blurhash
* @property {boolean=} use_pending_items
* @property {string} version
* @property {boolean} translation_enabled
*/
/**
@ -132,7 +131,6 @@ export const unfollowModal = getMeta('unfollow_modal');
export const useBlurhash = getMeta('use_blurhash');
export const usePendingItems = getMeta('use_pending_items');
export const version = getMeta('version');
export const translationEnabled = getMeta('translation_enabled');
export const languages = initialState?.languages;
export const statusPageUrl = getMeta('status_page_url');

View File

@ -21,6 +21,10 @@ class TranslationService
ENV['DEEPL_API_KEY'].present? || ENV['LIBRE_TRANSLATE_ENDPOINT'].present?
end
def supported?(_source_language, _target_language)
false
end
def translate(_text, _source_language, _target_language)
raise NotImplementedError
end

View File

@ -11,33 +11,53 @@ class TranslationService::DeepL < TranslationService
end
def translate(text, source_language, target_language)
request(text, source_language, target_language).perform do |res|
form = { text: text, source_lang: source_language&.upcase, target_lang: target_language, tag_handling: 'html' }
request(:post, '/v2/translate', form: form) do |res|
transform_response(res.body_with_limit)
end
end
def supported?(source_language, target_language)
source_language.in?(languages('source')) && target_language.in?(languages('target'))
end
private
def languages(type)
Rails.cache.fetch("translation_service/deepl/languages/#{type}", expires_in: 7.days, race_condition_ttl: 1.minute) do
request(:get, "/v2/languages?type=#{type}") do |res|
# In DeepL, EN and PT are deprecated in favor of EN-GB/EN-US and PT-BR/PT-PT, so
# they are supported but not returned by the API.
extra = type == 'source' ? [nil] : %w(en pt)
languages = Oj.load(res.body_with_limit).map { |language| language['language'].downcase }
languages + extra
end
end
end
def request(verb, path, **options)
req = Request.new(verb, "#{base_url}#{path}", **options)
req.add_headers(Authorization: "DeepL-Auth-Key #{@api_key}")
req.perform do |res|
case res.code
when 429
raise TooManyRequestsError
when 456
raise QuotaExceededError
when 200...300
transform_response(res.body_with_limit)
yield res
else
raise UnexpectedResponseError
end
end
end
private
def request(text, source_language, target_language)
req = Request.new(:post, endpoint_url, form: { text: text, source_lang: source_language&.upcase, target_lang: target_language, tag_handling: 'html' })
req.add_headers(Authorization: "DeepL-Auth-Key #{@api_key}")
req
end
def endpoint_url
def base_url
if @plan == 'free'
'https://api-free.deepl.com/v2/translate'
'https://api-free.deepl.com'
else
'https://api.deepl.com/v2/translate'
'https://api.deepl.com'
end
end

View File

@ -9,29 +9,45 @@ class TranslationService::LibreTranslate < TranslationService
end
def translate(text, source_language, target_language)
request(text, source_language, target_language).perform do |res|
body = Oj.dump(q: text, source: source_language.presence || 'auto', target: target_language, format: 'html', api_key: @api_key)
request(:post, '/translate', body: body) do |res|
transform_response(res.body_with_limit, source_language)
end
end
def supported?(source_language, target_language)
languages.key?(source_language) && languages[source_language].include?(target_language)
end
private
def languages
Rails.cache.fetch('translation_service/libre_translate/languages', expires_in: 7.days, race_condition_ttl: 1.minute) do
request(:get, '/languages') do |res|
languages = Oj.load(res.body_with_limit).to_h { |language| [language['code'], language['targets']] }
languages[nil] = languages.values.flatten.uniq
languages
end
end
end
def request(verb, path, **options)
req = Request.new(verb, "#{@base_url}#{path}", allow_local: true, **options)
req.add_headers('Content-Type': 'application/json')
req.perform do |res|
case res.code
when 429
raise TooManyRequestsError
when 403
raise QuotaExceededError
when 200...300
transform_response(res.body_with_limit, source_language)
yield res
else
raise UnexpectedResponseError
end
end
end
private
def request(text, source_language, target_language)
body = Oj.dump(q: text, source: source_language.presence || 'auto', target: target_language, format: 'html', api_key: @api_key)
req = Request.new(:post, "#{@base_url}/translate", body: body, allow_local: true)
req.add_headers('Content-Type': 'application/json')
req
end
def transform_response(str, source_language)
json = Oj.load(str, mode: :strict)

View File

@ -232,6 +232,16 @@ class Status < ApplicationRecord
public_visibility? || unlisted_visibility?
end
def translatable?
translate_target_locale = I18n.locale.to_s.split(/[_-]/).first
distributable? &&
content.present? &&
language != translate_target_locale &&
TranslationService.configured? &&
TranslationService.configured.supported?(language, translate_target_locale)
end
alias sign? distributable?
def with_media?

View File

@ -30,7 +30,6 @@ class InitialStateSerializer < ActiveModel::Serializer
timeline_preview: Setting.timeline_preview,
activity_api_enabled: Setting.activity_api_enabled,
single_user_mode: Rails.configuration.x.single_user_mode,
translation_enabled: TranslationService.configured?,
trends_as_landing_page: Setting.trends_as_landing_page,
status_page_url: Setting.status_page_url,
}

View File

@ -4,7 +4,7 @@ class REST::StatusSerializer < ActiveModel::Serializer
include FormattingHelper
attributes :id, :created_at, :in_reply_to_id, :in_reply_to_account_id,
:sensitive, :spoiler_text, :visibility, :language,
:sensitive, :spoiler_text, :visibility, :language, :translatable,
:uri, :url, :replies_count, :reblogs_count,
:favourites_count, :edited_at
@ -50,6 +50,10 @@ class REST::StatusSerializer < ActiveModel::Serializer
object.account.user_shows_application? || (current_user? && current_user.account_id == object.account_id)
end
def translatable
current_user? && object.translatable?
end
def visibility
# This visibility is masked behind "private"
# to avoid API changes because there are no

View File

@ -6,7 +6,7 @@ class TranslateStatusService < BaseService
include FormattingHelper
def call(status, target_language)
raise Mastodon::NotPermittedError unless status.public_visibility? || status.unlisted_visibility?
raise Mastodon::NotPermittedError unless status.translatable?
@status = status
@content = status_content_format(@status)

View File

@ -0,0 +1,100 @@
# frozen_string_literal: true
require 'rails_helper'
RSpec.describe TranslationService::DeepL do
subject(:service) { described_class.new(plan, 'my-api-key') }
let(:plan) { 'advanced' }
before do
stub_request(:get, 'https://api.deepl.com/v2/languages?type=source').to_return(
body: '[{"language":"EN","name":"English"},{"language":"UK","name":"Ukrainian"}]'
)
stub_request(:get, 'https://api.deepl.com/v2/languages?type=target').to_return(
body: '[{"language":"EN-GB","name":"English (British)"},{"language":"ZH","name":"Chinese"}]'
)
end
describe '#supported?' do
it 'supports included languages as source and target languages' do
expect(service.supported?('uk', 'en')).to be true
end
it 'supports auto-detecting source language' do
expect(service.supported?(nil, 'en')).to be true
end
it 'supports "en" and "pt" as target languages though not included in language list' do
expect(service.supported?('uk', 'en')).to be true
expect(service.supported?('uk', 'pt')).to be true
end
it 'does not support non-included language as target language' do
expect(service.supported?('uk', 'nl')).to be false
end
it 'does not support non-included language as source language' do
expect(service.supported?('da', 'en')).to be false
end
end
describe '#translate' do
it 'returns translation with specified source language' do
stub_request(:post, 'https://api.deepl.com/v2/translate')
.with(body: 'text=Hasta+la+vista&source_lang=ES&target_lang=en&tag_handling=html')
.to_return(body: '{"translations":[{"detected_source_language":"ES","text":"See you soon"}]}')
translation = service.translate('Hasta la vista', 'es', 'en')
expect(translation.detected_source_language).to eq 'es'
expect(translation.provider).to eq 'DeepL.com'
expect(translation.text).to eq 'See you soon'
end
it 'returns translation with auto-detected source language' do
stub_request(:post, 'https://api.deepl.com/v2/translate')
.with(body: 'text=Guten+Tag&source_lang&target_lang=en&tag_handling=html')
.to_return(body: '{"translations":[{"detected_source_language":"DE","text":"Good Morning"}]}')
translation = service.translate('Guten Tag', nil, 'en')
expect(translation.detected_source_language).to eq 'de'
expect(translation.provider).to eq 'DeepL.com'
expect(translation.text).to eq 'Good Morning'
end
end
describe '#languages?' do
it 'returns source languages' do
expect(service.send(:languages, 'source')).to eq ['en', 'uk', nil]
end
it 'returns target languages' do
expect(service.send(:languages, 'target')).to eq %w(en-gb zh en pt)
end
end
describe '#request' do
before do
stub_request(:any, //)
# rubocop:disable Lint/EmptyBlock
service.send(:request, :get, '/v2/languages') { |res| }
# rubocop:enable Lint/EmptyBlock
end
it 'uses paid plan base URL' do
expect(a_request(:get, 'https://api.deepl.com/v2/languages')).to have_been_made.once
end
context 'with free plan' do
let(:plan) { 'free' }
it 'uses free plan base URL' do
expect(a_request(:get, 'https://api-free.deepl.com/v2/languages')).to have_been_made.once
end
end
it 'sends API key' do
expect(a_request(:get, 'https://api.deepl.com/v2/languages').with(headers: { Authorization: 'DeepL-Auth-Key my-api-key' })).to have_been_made.once
end
end
end

View File

@ -0,0 +1,71 @@
# frozen_string_literal: true
require 'rails_helper'
RSpec.describe TranslationService::LibreTranslate do
subject(:service) { described_class.new('https://libretranslate.example.com', 'my-api-key') }
before do
stub_request(:get, 'https://libretranslate.example.com/languages').to_return(
body: '[{"code": "en","name": "English","targets": ["de","es"]},{"code": "da","name": "Danish","targets": ["en","de"]}]'
)
end
describe '#supported?' do
it 'supports included language pair' do
expect(service.supported?('en', 'de')).to be true
end
it 'does not support reversed language pair' do
expect(service.supported?('de', 'en')).to be false
end
it 'supports auto-detecting source language' do
expect(service.supported?(nil, 'de')).to be true
end
it 'does not support auto-detecting for unsupported target language' do
expect(service.supported?(nil, 'pt')).to be false
end
end
describe '#languages' do
subject(:languages) { service.send(:languages) }
it 'includes supported source languages' do
expect(languages.keys).to eq ['en', 'da', nil]
end
it 'includes supported target languages for source language' do
expect(languages['en']).to eq %w(de es)
end
it 'includes supported target languages for auto-detected language' do
expect(languages[nil]).to eq %w(de es en)
end
end
describe '#translate' do
it 'returns translation with specified source language' do
stub_request(:post, 'https://libretranslate.example.com/translate')
.with(body: '{"q":"Hasta la vista","source":"es","target":"en","format":"html","api_key":"my-api-key"}')
.to_return(body: '{"translatedText": "See you"}')
translation = service.translate('Hasta la vista', 'es', 'en')
expect(translation.detected_source_language).to eq 'es'
expect(translation.provider).to eq 'LibreTranslate'
expect(translation.text).to eq 'See you'
end
it 'returns translation with auto-detected source language' do
stub_request(:post, 'https://libretranslate.example.com/translate')
.with(body: '{"q":"Guten Morgen","source":"auto","target":"en","format":"html","api_key":"my-api-key"}')
.to_return(body: '{"detectedLanguage":{"confidence":92,"language":"de"},"translatedText":"Good morning"}')
translation = service.translate('Guten Morgen', nil, 'en')
expect(translation.detected_source_language).to be_nil
expect(translation.provider).to eq 'LibreTranslate'
expect(translation.text).to eq 'Good morning'
end
end
end

View File

@ -114,6 +114,85 @@ RSpec.describe Status, type: :model do
end
end
describe '#translatable?' do
before do
allow(TranslationService).to receive(:configured?).and_return(true)
allow(TranslationService).to receive(:configured).and_return(TranslationService.new)
allow(TranslationService.configured).to receive(:supported?).with('es', 'en').and_return(true)
subject.language = 'es'
subject.visibility = :public
end
context 'all conditions are satisfied' do
it 'returns true' do
expect(subject.translatable?).to be true
end
end
context 'translation service is not configured' do
it 'returns false' do
allow(TranslationService).to receive(:configured?).and_return(false)
allow(TranslationService).to receive(:configured).and_raise(TranslationService::NotConfiguredError)
expect(subject.translatable?).to be false
end
end
context 'status language is nil' do
it 'returns true' do
subject.language = nil
allow(TranslationService.configured).to receive(:supported?).with(nil, 'en').and_return(true)
expect(subject.translatable?).to be true
end
end
context 'status language is same as default locale' do
it 'returns false' do
subject.language = I18n.locale
expect(subject.translatable?).to be false
end
end
context 'status language is unsupported' do
it 'returns false' do
subject.language = 'af'
allow(TranslationService.configured).to receive(:supported?).with('af', 'en').and_return(false)
expect(subject.translatable?).to be false
end
end
context 'default locale is unsupported' do
it 'returns false' do
allow(TranslationService.configured).to receive(:supported?).with('es', 'af').and_return(false)
I18n.with_locale('af') do
expect(subject.translatable?).to be false
end
end
end
context 'default locale has region' do
it 'returns true' do
I18n.with_locale('en-GB') do
expect(subject.translatable?).to be true
end
end
end
context 'status text is blank' do
it 'returns false' do
subject.text = ' '
expect(subject.translatable?).to be false
end
end
context 'status visiblity is hidden' do
it 'returns false' do
subject.visibility = 'limited'
expect(subject.translatable?).to be false
end
end
end
describe '#content' do
it 'returns the text of the status if it is not a reblog' do
expect(subject.content).to eql subject.text