Merge fcbe8982e5 into 6d62581da1

app/controllers/api/v1/statuses/translations_controller.rb

@@ -23,6 +23,6 @@ class Api::V1::Statuses::TranslationsController < Api::V1::Statuses::BaseControl
   private
 
   def set_translation
-    @translation = TranslateStatusService.new.call(@status, I18n.locale.to_s)
+    @translation = TranslateStatusService.new.call(@status, params[:source_language], I18n.locale.to_s)
   end
 end

app/javascript/mastodon/actions/importer/normalizer.js

@@ -101,6 +101,7 @@ export function normalizeStatusTranslation(translation, status) {
   const emojiMap = makeEmojiMap(status.get('emojis').toJS());
 
   const normalTranslation = {
+    source_language: translation.source_language,
     detected_source_language: translation.detected_source_language,
     language: translation.language,
     provider: translation.provider,

app/javascript/mastodon/actions/statuses.js

@@ -335,10 +335,10 @@ export function toggleStatusCollapse(id, isCollapsed) {
   };
 }
 
-export const translateStatus = id => (dispatch) => {
+export const translateStatus = (id, source_language) => (dispatch) => {
   dispatch(translateStatusRequest(id));
 
-  api().post(`/api/v1/statuses/${id}/translate`).then(response => {
+  api().post(`/api/v1/statuses/${id}/translate`, { source_language }).then(response => {
     dispatch(translateStatusSuccess(id, response.data));
   }).catch(error => {
     dispatch(translateStatusFail(id, error));

app/javascript/mastodon/components/status.jsx

@@ -108,6 +108,7 @@ class Status extends ImmutablePureComponent {
     onToggleHidden: PropTypes.func,
     onToggleCollapsed: PropTypes.func,
     onTranslate: PropTypes.func,
+    onUndoStatusTranslation: PropTypes.func,
     onInteractionModal: PropTypes.func,
     muted: PropTypes.bool,
     hidden: PropTypes.bool,
@@ -188,8 +189,12 @@ class Status extends ImmutablePureComponent {
     this.props.onToggleCollapsed(this._properStatus(), isCollapsed);
   };
 
-  handleTranslate = () => {
-    this.props.onTranslate(this._properStatus());
+  handleTranslate = (sourceLanguage) => {
+    this.props.onTranslate(this._properStatus(), sourceLanguage);
+  };
+
+  handleUndoStatusTranslation = () => {
+    this.props.onUndoStatusTranslation(this._properStatus());
   };
 
   getAttachmentAspectRatio () {
@@ -569,6 +574,7 @@ class Status extends ImmutablePureComponent {
                   status={status}
                   onClick={this.handleClick}
                   onTranslate={this.handleTranslate}
+                  onUndoStatusTranslation={this.handleUndoStatusTranslation}
                   collapsible
                   onCollapsedToggle={this.handleCollapsedToggle}
                   {...statusContentProps}

app/javascript/mastodon/components/status_content.jsx

@@ -30,32 +30,47 @@ class TranslateButton extends PureComponent {
 
   static propTypes = {
     translation: ImmutablePropTypes.map,
-    onClick: PropTypes.func,
+    onTranslate: PropTypes.func,
+    onDetectLanguage: PropTypes.func,
+    onUndoStatusTranslation: PropTypes.func,
   };
 
   render () {
-    const { translation, onClick } = this.props;
+    const { translation, onTranslate, onDetectLanguage, onUndoStatusTranslation } = this.props;
 
     if (translation) {
       const language     = preloadedLanguages.find(lang => lang[0] === translation.get('detected_source_language'));
       const languageName = language ? language[2] : translation.get('detected_source_language');
       const provider     = translation.get('provider');
 
+      const languageDetectionButton = onDetectLanguage && (
+        <>
+          <button className='link-button' onClick={onDetectLanguage}>
+            <FormattedMessage id='status.detect_language' defaultMessage='Detect language' />
+          </button>
+          ·
+        </>
+      );
+
       return (
         <div className='translate-button'>
           <div className='translate-button__meta'>
             <FormattedMessage id='status.translated_from_with' defaultMessage='Translated from {lang} using {provider}' values={{ lang: languageName, provider }} />
           </div>
 
-          <button className='link-button' onClick={onClick}>
-            <FormattedMessage id='status.show_original' defaultMessage='Show original' />
-          </button>
+          <div className='translate-button__buttons'>
+            {languageDetectionButton}
+
+            <button className='link-button' onClick={onUndoStatusTranslation}>
+              <FormattedMessage id='status.show_original' defaultMessage='Show original' />
+            </button>
+          </div>
         </div>
       );
     }
 
     return (
-      <button className='status__content__translate-button' onClick={onClick}>
+      <button className='status__content__translate-button' onClick={onTranslate}>
         <FormattedMessage id='status.translate' defaultMessage='Translate' />
       </button>
     );
@@ -73,6 +88,7 @@ class StatusContent extends PureComponent {
     status: ImmutablePropTypes.map.isRequired,
     statusContent: PropTypes.string,
     onTranslate: PropTypes.func,
+    onUndoStatusTranslation: PropTypes.func,
     onClick: PropTypes.func,
     collapsible: PropTypes.bool,
     onCollapsedToggle: PropTypes.func,
@@ -215,6 +231,10 @@ class StatusContent extends PureComponent {
     this.props.onTranslate();
   };
 
+  handleDetectLanguage = () => {
+    this.props.onTranslate('und');
+  };
+
   setRef = (c) => {
     this.node = c;
   };
@@ -224,9 +244,13 @@ class StatusContent extends PureComponent {
 
     const renderReadMore = this.props.onClick && status.get('collapsed');
     const contentLocale = intl.locale.replace(/[_-].*/, '');
-    const targetLanguages = this.props.languages?.get(status.get('language') || 'und');
+    const sourceLanguage = status.get('language') || 'und';
+    const targetLanguages = this.props.languages?.get(sourceLanguage);
     const renderTranslate = this.props.onTranslate && this.props.identity.signedIn && ['public', 'unlisted'].includes(status.get('visibility')) && status.get('search_index').trim().length > 0 && targetLanguages?.includes(contentLocale);
 
+    const languageDetectionTargetLanguages = this.props.languages?.get('und');
+    const renderDetectLanguage = sourceLanguage !== 'und' && languageDetectionTargetLanguages?.includes(contentLocale) && status.get('translation')?.get('source_language') !== 'und';
+
     const content = { __html: statusContent ?? getStatusContent(status) };
     const language = status.getIn(['translation', 'language']) || status.get('language');
     const classNames = classnames('status__content', {
@@ -241,7 +265,12 @@ class StatusContent extends PureComponent {
     );
 
     const translateButton = renderTranslate && (
-      <TranslateButton onClick={this.handleTranslate} translation={status.get('translation')} />
+      <TranslateButton
+        onTranslate={this.handleTranslate}
+        onDetectLanguage={renderDetectLanguage && this.handleDetectLanguage}
+        onUndoStatusTranslation={this.props.onUndoStatusTranslation}
+        translation={status.get('translation')}
+      />
     );
 
     const poll = !!status.get('poll') && (

app/javascript/mastodon/containers/status_container.jsx

@@ -122,12 +122,12 @@ const mapDispatchToProps = (dispatch, { contextType }) => ({
     });
   },
 
-  onTranslate (status) {
-    if (status.get('translation')) {
-      dispatch(undoStatusTranslation(status.get('id'), status.get('poll')));
-    } else {
-      dispatch(translateStatus(status.get('id')));
-    }
+  onTranslate (status, sourceLanguage) {
+    dispatch(translateStatus(status.get('id'), sourceLanguage));
+  },
+
+  onUndoStatusTranslation (status) {
+    dispatch(undoStatusTranslation(status.get('id'), status.get('poll')));
   },
 
   onDirect (account) {

app/javascript/mastodon/features/status/components/detailed_status.tsx

@@ -43,7 +43,8 @@ export const DetailedStatus: React.FC<{
   status: any;
   onOpenMedia?: (status: any, index: number, lang: string) => void;
   onOpenVideo?: (status: any, lang: string, options: VideoModalOptions) => void;
-  onTranslate?: (status: any) => void;
+  onTranslate?: (status: any, sourceLanguage?: string) => void;
+  onUndoStatusTranslation?: (status: any) => void;
   measureHeight?: boolean;
   onHeightChange?: () => void;
   domain: string;
@@ -58,6 +59,7 @@ export const DetailedStatus: React.FC<{
   onOpenMedia,
   onOpenVideo,
   onTranslate,
+  onUndoStatusTranslation,
   measureHeight,
   onHeightChange,
   domain,
@@ -110,9 +112,16 @@ export const DetailedStatus: React.FC<{
     [_measureHeight],
   );
 
-  const handleTranslate = useCallback(() => {
-    if (onTranslate) onTranslate(status);
-  }, [onTranslate, status]);
+  const handleTranslate = useCallback(
+    (sourceLanguage?: string) => {
+      if (onTranslate) onTranslate(status, sourceLanguage);
+    },
+    [onTranslate, status],
+  );
+
+  const handleUndoStatusTranslation = useCallback(() => {
+    if (onUndoStatusTranslation) onUndoStatusTranslation(status);
+  }, [onUndoStatusTranslation, status]);
 
   if (!properStatus) {
     return null;
@@ -350,6 +359,7 @@ export const DetailedStatus: React.FC<{
             <StatusContent
               status={status}
               onTranslate={handleTranslate}
+              onUndoStatusTranslation={handleUndoStatusTranslation}
               {...(statusContentProps as any)}
             />
 

app/javascript/mastodon/features/status/index.jsx

@@ -392,14 +392,16 @@ class Status extends ImmutablePureComponent {
     }
   };
 
-  handleTranslate = status => {
+  handleTranslate = (status, sourceLanguage) => {
     const { dispatch } = this.props;
 
-    if (status.get('translation')) {
-      dispatch(undoStatusTranslation(status.get('id'), status.get('poll')));
-    } else {
-      dispatch(translateStatus(status.get('id')));
-    }
+    dispatch(translateStatus(status.get('id'), sourceLanguage));
+  };
+
+  handleUndoStatusTranslation = status => {
+    const { dispatch } = this.props;
+
+    dispatch(undoStatusTranslation(status.get('id'), status.get('poll')));
   };
 
   handleBlockClick = (status) => {
@@ -675,6 +677,7 @@ class Status extends ImmutablePureComponent {
                   onOpenMedia={this.handleOpenMedia}
                   onToggleHidden={this.handleToggleHidden}
                   onTranslate={this.handleTranslate}
+                  onUndoStatusTranslation={this.handleUndoStatusTranslation}
                   domain={domain}
                   showMedia={this.state.showMedia}
                   onToggleMediaVisibility={this.handleToggleMediaVisibility}

app/javascript/mastodon/locales/en.json

@@ -824,6 +824,7 @@
   "status.copy": "Copy link to post",
   "status.delete": "Delete",
   "status.detailed_status": "Detailed conversation view",
+  "status.detect_language": "Detect language",
   "status.direct": "Privately mention @{name}",
   "status.direct_indicator": "Private mention",
   "status.edit": "Edit",

app/javascript/styles/mastodon/components.scss

@@ -1348,6 +1348,17 @@ body > [data-popper-placement] {
   display: flex;
   justify-content: space-between;
   color: $dark-text-color;
+
+  .translate-button__buttons {
+    margin-right: -6px;
+    float: right;
+    white-space: nowrap;
+
+    .link-button {
+      display: inline;
+      margin: 0 6px;
+    }
+  }
 }
 
 .status__content__spoiler-link {

app/lib/translation_service/deepl.rb

@@ -11,6 +11,7 @@ class TranslationService::DeepL < TranslationService
   end
 
   def translate(texts, source_language, target_language)
+    source_language = nil if source_language == 'und'
     form = { text: texts, source_lang: source_language&.upcase, target_lang: target_language, tag_handling: 'html' }
     request(:post, '/v2/translate', form: form) do |res|
       transform_response(res.body_with_limit)
@@ -18,7 +19,7 @@ class TranslationService::DeepL < TranslationService
   end
 
   def languages
-    source_languages = [nil] + fetch_languages('source')
+    source_languages = ['und'] + fetch_languages('source')
 
     # In DeepL, EN and PT are deprecated in favor of EN-GB/EN-US and PT-BR/PT-PT, so
     # they are supported but not returned by the API.

app/lib/translation_service/libre_translate.rb

@@ -9,7 +9,8 @@ class TranslationService::LibreTranslate < TranslationService
   end
 
   def translate(texts, source_language, target_language)
-    body = Oj.dump(q: texts, source: source_language.presence || 'auto', target: target_language, format: 'html', api_key: @api_key)
+    source_language = 'auto' if source_language.in? [nil, 'und']
+    body = Oj.dump(q: texts, source: source_language, target: target_language, format: 'html', api_key: @api_key)
     request(:post, '/translate', body: body) do |res|
       transform_response(res.body_with_limit, source_language)
     end
@@ -20,7 +21,7 @@ class TranslationService::LibreTranslate < TranslationService
       languages = Oj.load(res.body_with_limit).to_h do |language|
         [language['code'], language['targets'].without(language['code'])]
       end
-      languages[nil] = languages.values.flatten.uniq.sort
+      languages['und'] = languages.values.flatten.uniq.sort
       languages
     end
   end

app/models/translation.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 class Translation < ActiveModelSerializers::Model
-  attributes :status, :detected_source_language, :language, :provider,
+  attributes :status, :source_language, :detected_source_language, :language, :provider,
              :content, :spoiler_text, :poll_options, :media_attachments
 
   class Option < ActiveModelSerializers::Model

app/serializers/rest/translation_serializer.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 class REST::TranslationSerializer < ActiveModel::Serializer
-  attributes :detected_source_language, :language, :provider, :spoiler_text, :content
+  attributes :source_language, :detected_source_language, :language, :provider, :spoiler_text, :content
 
   class PollSerializer < ActiveModel::Serializer
     attribute :id

app/services/translate_status_service.rb

@@ -6,17 +6,18 @@ class TranslateStatusService < BaseService
   include ERB::Util
   include FormattingHelper
 
-  def call(status, target_language)
+  def call(status, source_language, target_language)
     @status = status
     @source_texts = source_texts
+    @source_language = source_language || @status.language.presence || 'und'
 
     target_language = target_language.split(/[_-]/).first unless target_languages.include?(target_language)
     @target_language = target_language
 
     raise Mastodon::NotPermittedError unless permitted?
 
-    status_translation = Rails.cache.fetch("v2:translations/#{@status.language}/#{@target_language}/#{content_hash}", expires_in: CACHE_TTL) do
-      translations = translation_backend.translate(@source_texts.values, @status.language, @target_language)
+    status_translation = Rails.cache.fetch("v3:translations/#{@source_language}/#{@target_language}/#{content_hash}", expires_in: CACHE_TTL) do
+      translations = translation_backend.translate(@source_texts.values, @source_language, @target_language)
       build_status_translation(translations)
     end
 
@@ -38,7 +39,7 @@ class TranslateStatusService < BaseService
   end
 
   def languages
-    Rails.cache.fetch('translation_service/languages', expires_in: 7.days, race_condition_ttl: 1.hour) { translation_backend.languages }
+    Rails.cache.fetch('v2:translation_service/languages', expires_in: 7.days, race_condition_ttl: 1.hour) { translation_backend.languages }
   end
 
   def target_languages
@@ -67,6 +68,7 @@ class TranslateStatusService < BaseService
 
   def build_status_translation(translations)
     status_translation = Translation.new(
+      source_language: @source_language,
       detected_source_language: translations.first&.detected_source_language,
       language: @target_language,
       provider: translations.first&.provider,

spec/lib/translation_service/deepl_spec.rb

@@ -38,7 +38,7 @@ RSpec.describe TranslationService::DeepL do
         .with(body: 'text=Guten+Tag&source_lang&target_lang=en&tag_handling=html')
         .to_return(body: '{"translations":[{"detected_source_language":"DE","text":"Good morning"}]}')
 
-      translations = service.translate(['Guten Tag'], nil, 'en')
+      translations = service.translate(['Guten Tag'], 'und', 'en')
       expect(translations.size).to eq 1
 
       translation = translations.first
@@ -62,7 +62,7 @@ RSpec.describe TranslationService::DeepL do
 
   describe '#languages' do
     it 'returns source languages' do
-      expect(service.languages.keys).to eq [nil, 'en', 'uk']
+      expect(service.languages.keys).to eq %w(und en uk)
     end
 
     it 'returns target languages for each source language' do
@@ -71,7 +71,7 @@ RSpec.describe TranslationService::DeepL do
     end
 
     it 'returns target languages for auto-detection' do
-      expect(service.languages[nil]).to eq %w(en pt en-GB zh)
+      expect(service.languages['und']).to eq %w(en pt en-GB zh)
     end
   end
 

spec/lib/translation_service/libre_translate_spec.rb

@@ -15,7 +15,7 @@ RSpec.describe TranslationService::LibreTranslate do
     subject(:languages) { service.languages }
 
     it 'returns source languages' do
-      expect(languages.keys).to eq ['en', 'da', nil]
+      expect(languages.keys).to eq %w(en da und)
     end
 
     it 'returns target languages for each source language' do
@@ -24,7 +24,7 @@ RSpec.describe TranslationService::LibreTranslate do
     end
 
     it 'returns target languages for auto-detected language' do
-      expect(languages[nil]).to eq %w(de en es pt)
+      expect(languages['und']).to eq %w(de en es pt)
     end
   end
 

spec/requests/api/v1/statuses/translations_spec.rb

@@ -34,7 +34,7 @@ RSpec.describe 'API V1 Statuses Translations' do
         translation = TranslationService::Translation.new(text: 'Hello')
         service = instance_double(TranslationService::DeepL, translate: [translation])
         allow(TranslationService).to receive_messages(configured?: true, configured: service)
-        Rails.cache.write('translation_service/languages', { 'es' => ['en'] })
+        Rails.cache.write('v2:translation_service/languages', { 'es' => ['en'] })
         post "/api/v1/statuses/#{status.id}/translate", headers: headers
       end
 

spec/services/translate_status_service_spec.rb

@@ -33,7 +33,7 @@ RSpec.describe TranslateStatusService do
     end
 
     it 'returns translated status content and source language and provider and original status' do
-      expect(service.call(status, 'es'))
+      expect(service.call(status, nil, 'es'))
         .to have_attributes(
           content: '<p>Hola</p>',
           detected_source_language: 'en',
@@ -47,13 +47,13 @@ RSpec.describe TranslateStatusService do
       let(:text) { 'Hello & :highfive:' }
 
       it 'does not translate shortcode' do
-        expect(service.call(status, 'es').content).to eq '<p>Hola &amp; :highfive:</p>'
+        expect(service.call(status, nil, 'es').content).to eq '<p>Hola &amp; :highfive:</p>'
       end
     end
 
     describe 'status has no spoiler_text' do
       it 'returns an empty string' do
-        expect(service.call(status, 'es').spoiler_text).to eq ''
+        expect(service.call(status, nil, 'es').spoiler_text).to eq ''
       end
     end
 
@@ -61,7 +61,7 @@ RSpec.describe TranslateStatusService do
       let(:spoiler_text) { 'Hello & Hello!' }
 
       it 'translates the spoiler text' do
-        expect(service.call(status, 'es').spoiler_text).to eq 'Hola & Hola!'
+        expect(service.call(status, nil, 'es').spoiler_text).to eq 'Hola & Hola!'
       end
     end
 
@@ -69,7 +69,7 @@ RSpec.describe TranslateStatusService do
       let(:spoiler_text) { 'Hello :highfive:' }
 
       it 'does not translate shortcode' do
-        expect(service.call(status, 'es').spoiler_text).to eq 'Hola :highfive:'
+        expect(service.call(status, nil, 'es').spoiler_text).to eq 'Hola :highfive:'
       end
     end
 
@@ -77,7 +77,7 @@ RSpec.describe TranslateStatusService do
       let(:spoiler_text) { 'Hello :Hello:' }
 
       it 'translates the invalid shortcode' do
-        expect(service.call(status, 'es').spoiler_text).to eq 'Hola :Hola:'
+        expect(service.call(status, nil, 'es').spoiler_text).to eq 'Hola :Hola:'
       end
     end
 
@@ -85,7 +85,7 @@ RSpec.describe TranslateStatusService do
       let(:poll) { Fabricate(:poll, options: ['Hello 1', 'Hello 2']) }
 
       it 'translates the poll option title' do
-        status_translation = service.call(status, 'es')
+        status_translation = service.call(status, nil, 'es')
         expect(status_translation.poll_options.size).to eq 2
         expect(status_translation.poll_options.first.title).to eq 'Hola 1'
       end
@@ -95,7 +95,7 @@ RSpec.describe TranslateStatusService do
       let(:media_attachments) { [Fabricate(:media_attachment, description: 'Hello & :highfive:')] }
 
       it 'translates the media attachment description' do
-        status_translation = service.call(status, 'es')
+        status_translation = service.call(status, nil, 'es')
 
         media_attachment = status_translation.media_attachments.first
         expect(media_attachment.id).to eq media_attachments.first.id
@@ -105,11 +105,11 @@ RSpec.describe TranslateStatusService do
 
     describe 'target language is regional' do
       it 'uses regional variant' do
-        expect(service.call(status, 'es-MX').language).to eq 'es-MX'
+        expect(service.call(status, nil, 'es-MX').language).to eq 'es-MX'
       end
 
       it 'uses parent locale for unsupported regional variant' do
-        expect(service.call(status, 'es-XX').language).to eq 'es'
+        expect(service.call(status, nil, 'es-XX').language).to eq 'es'
       end
     end
   end