diff --git a/app/controllers/api/v1/statuses/translations_controller.rb b/app/controllers/api/v1/statuses/translations_controller.rb index bd5cd9bb07..3822620061 100644 --- a/app/controllers/api/v1/statuses/translations_controller.rb +++ b/app/controllers/api/v1/statuses/translations_controller.rb @@ -23,6 +23,6 @@ class Api::V1::Statuses::TranslationsController < Api::V1::Statuses::BaseControl private def set_translation - @translation = TranslateStatusService.new.call(@status, I18n.locale.to_s) + @translation = TranslateStatusService.new.call(@status, params[:source_language], I18n.locale.to_s) end end diff --git a/app/javascript/mastodon/actions/importer/normalizer.js b/app/javascript/mastodon/actions/importer/normalizer.js index 2c583f86d4..bf9cfa5e7b 100644 --- a/app/javascript/mastodon/actions/importer/normalizer.js +++ b/app/javascript/mastodon/actions/importer/normalizer.js @@ -109,6 +109,7 @@ export function normalizeStatusTranslation(translation, status) { const emojiMap = makeEmojiMap(status.get('emojis').toJS()); const normalTranslation = { + source_language: translation.source_language, detected_source_language: translation.detected_source_language, language: translation.language, provider: translation.provider, diff --git a/app/javascript/mastodon/actions/statuses.js b/app/javascript/mastodon/actions/statuses.js index 42d0c1c0f1..4ae2ac43a6 100644 --- a/app/javascript/mastodon/actions/statuses.js +++ b/app/javascript/mastodon/actions/statuses.js @@ -290,10 +290,10 @@ export function toggleStatusCollapse(id, isCollapsed) { }; } -export const translateStatus = id => (dispatch) => { +export const translateStatus = (id, source_language) => (dispatch) => { dispatch(translateStatusRequest(id)); - api().post(`/api/v1/statuses/${id}/translate`).then(response => { + api().post(`/api/v1/statuses/${id}/translate`, { source_language }).then(response => { dispatch(translateStatusSuccess(id, response.data)); }).catch(error => { dispatch(translateStatusFail(id, error)); diff --git a/app/javascript/mastodon/components/status.jsx b/app/javascript/mastodon/components/status.jsx index 21d596a58c..ca4f04ece1 100644 --- a/app/javascript/mastodon/components/status.jsx +++ b/app/javascript/mastodon/components/status.jsx @@ -108,6 +108,7 @@ class Status extends ImmutablePureComponent { onToggleHidden: PropTypes.func, onToggleCollapsed: PropTypes.func, onTranslate: PropTypes.func, + onUndoStatusTranslation: PropTypes.func, onInteractionModal: PropTypes.func, muted: PropTypes.bool, hidden: PropTypes.bool, @@ -193,8 +194,12 @@ class Status extends ImmutablePureComponent { this.props.onToggleCollapsed(this._properStatus(), isCollapsed); }; - handleTranslate = () => { - this.props.onTranslate(this._properStatus()); + handleTranslate = (sourceLanguage) => { + this.props.onTranslate(this._properStatus(), sourceLanguage); + }; + + handleUndoStatusTranslation = () => { + this.props.onUndoStatusTranslation(this._properStatus()); }; getAttachmentAspectRatio () { @@ -572,6 +577,7 @@ class Status extends ImmutablePureComponent { status={status} onClick={this.handleClick} onTranslate={this.handleTranslate} + onUndoStatusTranslation={this.handleUndoStatusTranslation} collapsible onCollapsedToggle={this.handleCollapsedToggle} {...statusContentProps} diff --git a/app/javascript/mastodon/components/status_content.jsx b/app/javascript/mastodon/components/status_content.jsx index 275fea5f0e..f04e45d866 100644 --- a/app/javascript/mastodon/components/status_content.jsx +++ b/app/javascript/mastodon/components/status_content.jsx @@ -30,32 +30,47 @@ class TranslateButton extends PureComponent { static propTypes = { translation: ImmutablePropTypes.map, - onClick: PropTypes.func, + onTranslate: PropTypes.func, + onDetectLanguage: PropTypes.func, + onUndoStatusTranslation: PropTypes.func, }; render () { - const { translation, onClick } = this.props; + const { translation, onTranslate, onDetectLanguage, onUndoStatusTranslation } = this.props; if (translation) { const language = preloadedLanguages.find(lang => lang[0] === translation.get('detected_source_language')); const languageName = language ? language[1] : translation.get('detected_source_language'); const provider = translation.get('provider'); + const languageDetectionButton = onDetectLanguage && ( + <> + + ยท + + ); + return (
- +
+ {languageDetectionButton} + + +
); } return ( - ); @@ -73,6 +88,7 @@ class StatusContent extends PureComponent { status: ImmutablePropTypes.map.isRequired, statusContent: PropTypes.string, onTranslate: PropTypes.func, + onUndoStatusTranslation: PropTypes.func, onClick: PropTypes.func, collapsible: PropTypes.bool, onCollapsedToggle: PropTypes.func, @@ -216,6 +232,10 @@ class StatusContent extends PureComponent { this.props.onTranslate(); }; + handleDetectLanguage = () => { + this.props.onTranslate('und'); + }; + setRef = (c) => { this.node = c; }; @@ -225,9 +245,13 @@ class StatusContent extends PureComponent { const renderReadMore = this.props.onClick && status.get('collapsed'); const contentLocale = intl.locale.replace(/[_-].*/, ''); - const targetLanguages = this.props.languages?.get(status.get('language') || 'und'); + const sourceLanguage = status.get('language') || 'und'; + const targetLanguages = this.props.languages?.get(sourceLanguage); const renderTranslate = this.props.onTranslate && this.props.identity.signedIn && ['public', 'unlisted'].includes(status.get('visibility')) && status.get('search_index').trim().length > 0 && targetLanguages?.includes(contentLocale); + const languageDetectionTargetLanguages = this.props.languages?.get('und'); + const renderDetectLanguage = sourceLanguage !== 'und' && languageDetectionTargetLanguages?.includes(contentLocale) && status.get('translation')?.get('source_language') !== 'und'; + const content = { __html: statusContent ?? getStatusContent(status) }; const language = status.getIn(['translation', 'language']) || status.get('language'); const classNames = classnames('status__content', { @@ -242,7 +266,12 @@ class StatusContent extends PureComponent { ); const translateButton = renderTranslate && ( - + ); const poll = !!status.get('poll') && ( diff --git a/app/javascript/mastodon/containers/status_container.jsx b/app/javascript/mastodon/containers/status_container.jsx index 0fb5f25560..fb57864b3f 100644 --- a/app/javascript/mastodon/containers/status_container.jsx +++ b/app/javascript/mastodon/containers/status_container.jsx @@ -122,12 +122,12 @@ const mapDispatchToProps = (dispatch, { contextType }) => ({ }); }, - onTranslate (status) { - if (status.get('translation')) { - dispatch(undoStatusTranslation(status.get('id'), status.get('poll'))); - } else { - dispatch(translateStatus(status.get('id'))); - } + onTranslate (status, sourceLanguage) { + dispatch(translateStatus(status.get('id'), sourceLanguage)); + }, + + onUndoStatusTranslation (status) { + dispatch(undoStatusTranslation(status.get('id'), status.get('poll'))); }, onDirect (account) { diff --git a/app/javascript/mastodon/features/status/components/detailed_status.tsx b/app/javascript/mastodon/features/status/components/detailed_status.tsx index 75d995b1e0..003052b3c8 100644 --- a/app/javascript/mastodon/features/status/components/detailed_status.tsx +++ b/app/javascript/mastodon/features/status/components/detailed_status.tsx @@ -43,7 +43,8 @@ export const DetailedStatus: React.FC<{ status: any; onOpenMedia?: (status: any, index: number, lang: string) => void; onOpenVideo?: (status: any, lang: string, options: VideoModalOptions) => void; - onTranslate?: (status: any) => void; + onTranslate?: (status: any, sourceLanguage?: string) => void; + onUndoStatusTranslation?: (status: any) => void; measureHeight?: boolean; onHeightChange?: () => void; domain: string; @@ -58,6 +59,7 @@ export const DetailedStatus: React.FC<{ onOpenMedia, onOpenVideo, onTranslate, + onUndoStatusTranslation, measureHeight, onHeightChange, domain, @@ -115,9 +117,16 @@ export const DetailedStatus: React.FC<{ [_measureHeight], ); - const handleTranslate = useCallback(() => { - if (onTranslate) onTranslate(status); - }, [onTranslate, status]); + const handleTranslate = useCallback( + (sourceLanguage?: string) => { + if (onTranslate) onTranslate(status, sourceLanguage); + }, + [onTranslate, status], + ); + + const handleUndoStatusTranslation = useCallback(() => { + if (onUndoStatusTranslation) onUndoStatusTranslation(status); + }, [onUndoStatusTranslation, status]); if (!properStatus) { return null; @@ -368,6 +377,7 @@ export const DetailedStatus: React.FC<{ diff --git a/app/javascript/mastodon/features/status/index.jsx b/app/javascript/mastodon/features/status/index.jsx index 7da2df3742..1fccb3d2af 100644 --- a/app/javascript/mastodon/features/status/index.jsx +++ b/app/javascript/mastodon/features/status/index.jsx @@ -339,14 +339,16 @@ class Status extends ImmutablePureComponent { } }; - handleTranslate = status => { + handleTranslate = (status, sourceLanguage) => { const { dispatch } = this.props; - if (status.get('translation')) { - dispatch(undoStatusTranslation(status.get('id'), status.get('poll'))); - } else { - dispatch(translateStatus(status.get('id'))); - } + dispatch(translateStatus(status.get('id'), sourceLanguage)); + }; + + handleUndoStatusTranslation = status => { + const { dispatch } = this.props; + + dispatch(undoStatusTranslation(status.get('id'), status.get('poll'))); }; handleBlockClick = (status) => { @@ -627,6 +629,7 @@ class Status extends ImmutablePureComponent { onOpenMedia={this.handleOpenMedia} onToggleHidden={this.handleToggleHidden} onTranslate={this.handleTranslate} + onUndoStatusTranslation={this.handleUndoStatusTranslation} domain={domain} showMedia={this.state.showMedia} onToggleMediaVisibility={this.handleToggleMediaVisibility} diff --git a/app/javascript/mastodon/locales/en.json b/app/javascript/mastodon/locales/en.json index 3d6e166498..703b77f843 100644 --- a/app/javascript/mastodon/locales/en.json +++ b/app/javascript/mastodon/locales/en.json @@ -840,6 +840,7 @@ "status.copy": "Copy link to post", "status.delete": "Delete", "status.detailed_status": "Detailed conversation view", + "status.detect_language": "Detect language", "status.direct": "Privately mention @{name}", "status.direct_indicator": "Private mention", "status.edit": "Edit", diff --git a/app/javascript/styles/mastodon/components.scss b/app/javascript/styles/mastodon/components.scss index e22a9ed9c9..f5ff8a4cf6 100644 --- a/app/javascript/styles/mastodon/components.scss +++ b/app/javascript/styles/mastodon/components.scss @@ -1383,6 +1383,17 @@ body > [data-popper-placement] { display: flex; justify-content: space-between; color: $dark-text-color; + + .translate-button__buttons { + margin-right: -6px; + float: right; + white-space: nowrap; + + .link-button { + display: inline; + margin: 0 6px; + } + } } .status__wrapper--filtered { diff --git a/app/lib/translation_service/deepl.rb b/app/lib/translation_service/deepl.rb index 7761dbe626..f53342e4be 100644 --- a/app/lib/translation_service/deepl.rb +++ b/app/lib/translation_service/deepl.rb @@ -11,6 +11,7 @@ class TranslationService::DeepL < TranslationService end def translate(texts, source_language, target_language) + source_language = nil if source_language == 'und' form = { text: texts, source_lang: source_language&.upcase, target_lang: target_language, tag_handling: 'html' } request(:post, '/v2/translate', form: form) do |res| transform_response(res.body_with_limit) @@ -18,7 +19,7 @@ class TranslationService::DeepL < TranslationService end def languages - source_languages = [nil] + fetch_languages('source') + source_languages = ['und'] + fetch_languages('source') # In DeepL, EN and PT are deprecated in favor of EN-GB/EN-US and PT-BR/PT-PT, so # they are supported but not returned by the API. diff --git a/app/lib/translation_service/libre_translate.rb b/app/lib/translation_service/libre_translate.rb index 0df8590f87..aac0f1cc6b 100644 --- a/app/lib/translation_service/libre_translate.rb +++ b/app/lib/translation_service/libre_translate.rb @@ -9,7 +9,8 @@ class TranslationService::LibreTranslate < TranslationService end def translate(texts, source_language, target_language) - body = Oj.dump(q: texts, source: source_language.presence || 'auto', target: target_language, format: 'html', api_key: @api_key) + source_language = 'auto' if source_language.in? [nil, 'und'] + body = Oj.dump(q: texts, source: source_language, target: target_language, format: 'html', api_key: @api_key) request(:post, '/translate', body: body) do |res| transform_response(res.body_with_limit, source_language) end @@ -20,7 +21,7 @@ class TranslationService::LibreTranslate < TranslationService languages = Oj.load(res.body_with_limit).to_h do |language| [language['code'], language['targets'].without(language['code'])] end - languages[nil] = languages.values.flatten.uniq.sort + languages['und'] = languages.values.flatten.uniq.sort languages end end diff --git a/app/models/translation.rb b/app/models/translation.rb index 7f8469c86e..97ef7deece 100644 --- a/app/models/translation.rb +++ b/app/models/translation.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true class Translation < ActiveModelSerializers::Model - attributes :status, :detected_source_language, :language, :provider, + attributes :status, :source_language, :detected_source_language, :language, :provider, :content, :spoiler_text, :poll_options, :media_attachments class Option < ActiveModelSerializers::Model diff --git a/app/serializers/rest/translation_serializer.rb b/app/serializers/rest/translation_serializer.rb index 40e2d28fb7..653efe6a12 100644 --- a/app/serializers/rest/translation_serializer.rb +++ b/app/serializers/rest/translation_serializer.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true class REST::TranslationSerializer < ActiveModel::Serializer - attributes :detected_source_language, :language, :provider, :spoiler_text, :content + attributes :source_language, :detected_source_language, :language, :provider, :spoiler_text, :content class PollSerializer < ActiveModel::Serializer attribute :id diff --git a/app/services/translate_status_service.rb b/app/services/translate_status_service.rb index bcd4703beb..8635e07f5d 100644 --- a/app/services/translate_status_service.rb +++ b/app/services/translate_status_service.rb @@ -6,17 +6,18 @@ class TranslateStatusService < BaseService include ERB::Util include FormattingHelper - def call(status, target_language) + def call(status, source_language, target_language) @status = status @source_texts = source_texts + @source_language = source_language || @status.language.presence || 'und' target_language = target_language.split(/[_-]/).first unless target_languages.include?(target_language) @target_language = target_language raise Mastodon::NotPermittedError unless permitted? - status_translation = Rails.cache.fetch("v2:translations/#{@status.language}/#{@target_language}/#{content_hash}", expires_in: CACHE_TTL) do - translations = translation_backend.translate(@source_texts.values, @status.language, @target_language) + status_translation = Rails.cache.fetch("v3:translations/#{@source_language}/#{@target_language}/#{content_hash}", expires_in: CACHE_TTL) do + translations = translation_backend.translate(@source_texts.values, @source_language, @target_language) build_status_translation(translations) end @@ -38,7 +39,7 @@ class TranslateStatusService < BaseService end def languages - Rails.cache.fetch('translation_service/languages', expires_in: 7.days, race_condition_ttl: 1.hour) { translation_backend.languages } + Rails.cache.fetch('v2:translation_service/languages', expires_in: 7.days, race_condition_ttl: 1.hour) { translation_backend.languages } end def target_languages @@ -67,6 +68,7 @@ class TranslateStatusService < BaseService def build_status_translation(translations) status_translation = Translation.new( + source_language: @source_language, detected_source_language: translations.first&.detected_source_language, language: @target_language, provider: translations.first&.provider, diff --git a/spec/lib/translation_service/deepl_spec.rb b/spec/lib/translation_service/deepl_spec.rb index 4797a3dc63..e7156c6dfc 100644 --- a/spec/lib/translation_service/deepl_spec.rb +++ b/spec/lib/translation_service/deepl_spec.rb @@ -38,7 +38,7 @@ RSpec.describe TranslationService::DeepL do .with(body: 'text=Guten+Tag&source_lang&target_lang=en&tag_handling=html') .to_return(body: '{"translations":[{"detected_source_language":"DE","text":"Good morning"}]}') - translations = service.translate(['Guten Tag'], nil, 'en') + translations = service.translate(['Guten Tag'], 'und', 'en') expect(translations.size).to eq 1 translation = translations.first @@ -62,7 +62,7 @@ RSpec.describe TranslationService::DeepL do describe '#languages' do it 'returns source languages' do - expect(service.languages.keys).to eq [nil, 'en', 'uk'] + expect(service.languages.keys).to eq %w(und en uk) end it 'returns target languages for each source language' do @@ -71,7 +71,7 @@ RSpec.describe TranslationService::DeepL do end it 'returns target languages for auto-detection' do - expect(service.languages[nil]).to eq %w(en pt en-GB zh) + expect(service.languages['und']).to eq %w(en pt en-GB zh) end end diff --git a/spec/lib/translation_service/libre_translate_spec.rb b/spec/lib/translation_service/libre_translate_spec.rb index 90966a8ebf..08433e6ff0 100644 --- a/spec/lib/translation_service/libre_translate_spec.rb +++ b/spec/lib/translation_service/libre_translate_spec.rb @@ -15,7 +15,7 @@ RSpec.describe TranslationService::LibreTranslate do subject(:languages) { service.languages } it 'returns source languages' do - expect(languages.keys).to eq ['en', 'da', nil] + expect(languages.keys).to eq %w(en da und) end it 'returns target languages for each source language' do @@ -24,7 +24,7 @@ RSpec.describe TranslationService::LibreTranslate do end it 'returns target languages for auto-detected language' do - expect(languages[nil]).to eq %w(de en es pt) + expect(languages['und']).to eq %w(de en es pt) end end diff --git a/spec/requests/api/v1/statuses/translations_spec.rb b/spec/requests/api/v1/statuses/translations_spec.rb index e316bd451b..6eda3e884b 100644 --- a/spec/requests/api/v1/statuses/translations_spec.rb +++ b/spec/requests/api/v1/statuses/translations_spec.rb @@ -34,7 +34,7 @@ RSpec.describe 'API V1 Statuses Translations' do translation = TranslationService::Translation.new(text: 'Hello') service = instance_double(TranslationService::DeepL, translate: [translation]) allow(TranslationService).to receive_messages(configured?: true, configured: service) - Rails.cache.write('translation_service/languages', { 'es' => ['en'] }) + Rails.cache.write('v2:translation_service/languages', { 'es' => ['en'] }) post "/api/v1/statuses/#{status.id}/translate", headers: headers end diff --git a/spec/services/translate_status_service_spec.rb b/spec/services/translate_status_service_spec.rb index ac7a43ff2a..fcece41bdc 100644 --- a/spec/services/translate_status_service_spec.rb +++ b/spec/services/translate_status_service_spec.rb @@ -33,7 +33,7 @@ RSpec.describe TranslateStatusService do end it 'returns translated status content and source language and provider and original status' do - expect(service.call(status, 'es')) + expect(service.call(status, nil, 'es')) .to have_attributes( content: '

Hola

', detected_source_language: 'en', @@ -47,13 +47,13 @@ RSpec.describe TranslateStatusService do let(:text) { 'Hello & :highfive:' } it 'does not translate shortcode' do - expect(service.call(status, 'es').content).to eq '

Hola & :highfive:

' + expect(service.call(status, nil, 'es').content).to eq '

Hola & :highfive:

' end end describe 'status has no spoiler_text' do it 'returns an empty string' do - expect(service.call(status, 'es').spoiler_text).to eq '' + expect(service.call(status, nil, 'es').spoiler_text).to eq '' end end @@ -61,7 +61,7 @@ RSpec.describe TranslateStatusService do let(:spoiler_text) { 'Hello & Hello!' } it 'translates the spoiler text' do - expect(service.call(status, 'es').spoiler_text).to eq 'Hola & Hola!' + expect(service.call(status, nil, 'es').spoiler_text).to eq 'Hola & Hola!' end end @@ -69,7 +69,7 @@ RSpec.describe TranslateStatusService do let(:spoiler_text) { 'Hello :highfive:' } it 'does not translate shortcode' do - expect(service.call(status, 'es').spoiler_text).to eq 'Hola :highfive:' + expect(service.call(status, nil, 'es').spoiler_text).to eq 'Hola :highfive:' end end @@ -77,7 +77,7 @@ RSpec.describe TranslateStatusService do let(:spoiler_text) { 'Hello :Hello:' } it 'translates the invalid shortcode' do - expect(service.call(status, 'es').spoiler_text).to eq 'Hola :Hola:' + expect(service.call(status, nil, 'es').spoiler_text).to eq 'Hola :Hola:' end end @@ -85,7 +85,7 @@ RSpec.describe TranslateStatusService do let(:poll) { Fabricate(:poll, options: ['Hello 1', 'Hello 2']) } it 'translates the poll option title' do - status_translation = service.call(status, 'es') + status_translation = service.call(status, nil, 'es') expect(status_translation.poll_options.size).to eq 2 expect(status_translation.poll_options.first.title).to eq 'Hola 1' end @@ -95,7 +95,7 @@ RSpec.describe TranslateStatusService do let(:media_attachments) { [Fabricate(:media_attachment, description: 'Hello & :highfive:')] } it 'translates the media attachment description' do - status_translation = service.call(status, 'es') + status_translation = service.call(status, nil, 'es') media_attachment = status_translation.media_attachments.first expect(media_attachment.id).to eq media_attachments.first.id @@ -105,11 +105,11 @@ RSpec.describe TranslateStatusService do describe 'target language is regional' do it 'uses regional variant' do - expect(service.call(status, 'es-MX').language).to eq 'es-MX' + expect(service.call(status, nil, 'es-MX').language).to eq 'es-MX' end it 'uses parent locale for unsupported regional variant' do - expect(service.call(status, 'es-XX').language).to eq 'es' + expect(service.call(status, nil, 'es-XX').language).to eq 'es' end end end