Offer language detection

app/controllers/api/v1/statuses/translations_controller.rb

@@ -23,6 +23,6 @@ class Api::V1::Statuses::TranslationsController < Api::V1::Statuses::BaseControl
   private
 
   def set_translation
-    @translation = TranslateStatusService.new.call(@status, content_locale)
+    @translation = TranslateStatusService.new.call(@status, params[:source_language], content_locale)
   end
 end

app/javascript/mastodon/actions/importer/normalizer.js

@@ -101,6 +101,7 @@ export function normalizeStatusTranslation(translation, status) {
   const emojiMap = makeEmojiMap(status.get('emojis').toJS());
 
   const normalTranslation = {
+    source_language: translation.source_language,
     detected_source_language: translation.detected_source_language,
     language: translation.language,
     provider: translation.provider,

app/javascript/mastodon/actions/statuses.js

@@ -333,10 +333,10 @@ export function toggleStatusCollapse(id, isCollapsed) {
   };
 }
 
-export const translateStatus = id => (dispatch) => {
+export const translateStatus = (id, source_language) => (dispatch) => {
   dispatch(translateStatusRequest(id));
 
-  api().post(`/api/v1/statuses/${id}/translate`).then(response => {
+  api().post(`/api/v1/statuses/${id}/translate`, { source_language }).then(response => {
     dispatch(translateStatusSuccess(id, response.data));
   }).catch(error => {
     dispatch(translateStatusFail(id, error));

app/javascript/mastodon/components/status.jsx

@@ -106,6 +106,7 @@ class Status extends ImmutablePureComponent {
     onToggleHidden: PropTypes.func,
     onToggleCollapsed: PropTypes.func,
     onTranslate: PropTypes.func,
+    onUndoStatusTranslation: PropTypes.func,
     onInteractionModal: PropTypes.func,
     muted: PropTypes.bool,
     hidden: PropTypes.bool,
@@ -200,8 +201,12 @@ class Status extends ImmutablePureComponent {
     this.props.onToggleCollapsed(this._properStatus(), isCollapsed);
   };
 
-  handleTranslate = () => {
+  handleTranslate = (sourceLanguage) => {
-    this.props.onTranslate(this._properStatus());
+    this.props.onTranslate(this._properStatus(), sourceLanguage);
+  };
+
+  handleUndoStatusTranslation = () => {
+    this.props.onUndoStatusTranslation(this._properStatus());
   };
 
   getAttachmentAspectRatio () {
@@ -579,6 +584,7 @@ class Status extends ImmutablePureComponent {
                   status={status}
                   onClick={this.handleClick}
                   onTranslate={this.handleTranslate}
+                  onUndoStatusTranslation={this.handleUndoStatusTranslation}
                   collapsible
                   onCollapsedToggle={this.handleCollapsedToggle}
                   {...statusContentProps}

app/javascript/mastodon/components/status_content.jsx

@@ -30,32 +30,47 @@ class TranslateButton extends PureComponent {
 
   static propTypes = {
     translation: ImmutablePropTypes.map,
-    onClick: PropTypes.func,
+    onTranslate: PropTypes.func,
+    onDetectLanguage: PropTypes.func,
+    onUndoStatusTranslation: PropTypes.func,
   };
 
   render () {
-    const { translation, onClick } = this.props;
+    const { translation, onTranslate, onDetectLanguage, onUndoStatusTranslation } = this.props;
 
     if (translation) {
       const language     = preloadedLanguages.find(lang => lang[0] === translation.get('detected_source_language'));
       const languageName = language ? language[2] : translation.get('detected_source_language');
       const provider     = translation.get('provider');
 
+      const languageDetectionButton = onDetectLanguage && (
+        <>
+          <button className='link-button' onClick={onDetectLanguage}>
+            <FormattedMessage id='status.detect_language' defaultMessage='Detect language' />
+          </button>
+          ·
+        </>
+      );
+
       return (
         <div className='translate-button'>
           <div className='translate-button__meta'>
             <FormattedMessage id='status.translated_from_with' defaultMessage='Translated from {lang} using {provider}' values={{ lang: languageName, provider }} />
           </div>
 
-          <button className='link-button' onClick={onClick}>
+          <div className='translate-button__buttons'>
-            <FormattedMessage id='status.show_original' defaultMessage='Show original' />
+            {languageDetectionButton}
-          </button>
+
+            <button className='link-button' onClick={onUndoStatusTranslation}>
+              <FormattedMessage id='status.show_original' defaultMessage='Show original' />
+            </button>
+          </div>
         </div>
       );
     }
 
     return (
-      <button className='status__content__translate-button' onClick={onClick}>
+      <button className='status__content__translate-button' onClick={onTranslate}>
         <FormattedMessage id='status.translate' defaultMessage='Translate' />
       </button>
     );
@@ -73,6 +88,7 @@ class StatusContent extends PureComponent {
     status: ImmutablePropTypes.map.isRequired,
     statusContent: PropTypes.string,
     onTranslate: PropTypes.func,
+    onUndoStatusTranslation: PropTypes.func,
     onClick: PropTypes.func,
     collapsible: PropTypes.bool,
     onCollapsedToggle: PropTypes.func,
@@ -215,6 +231,10 @@ class StatusContent extends PureComponent {
     this.props.onTranslate();
   };
 
+  handleDetectLanguage = () => {
+    this.props.onTranslate('und');
+  };
+
   setRef = (c) => {
     this.node = c;
   };
@@ -224,9 +244,13 @@ class StatusContent extends PureComponent {
 
     const renderReadMore = this.props.onClick && status.get('collapsed');
     const contentLocale = intl.locale.replace(/[_-].*/, '');
-    const targetLanguages = this.props.languages?.get(status.get('language') || 'und');
+    const sourceLanguage = status.get('language') || 'und';
+    const targetLanguages = this.props.languages?.get(sourceLanguage);
     const renderTranslate = this.props.onTranslate && this.props.identity.signedIn && ['public', 'unlisted'].includes(status.get('visibility')) && status.get('search_index').trim().length > 0 && targetLanguages?.includes(contentLocale);
 
+    const languageDetectionTargetLanguages = this.props.languages?.get('und');
+    const renderDetectLanguage = sourceLanguage !== 'und' && languageDetectionTargetLanguages?.includes(contentLocale) && status.get('translation')?.get('source_language') !== 'und';
+
     const content = { __html: statusContent ?? getStatusContent(status) };
     const language = status.getIn(['translation', 'language']) || status.get('language');
     const classNames = classnames('status__content', {
@@ -241,7 +265,12 @@ class StatusContent extends PureComponent {
     );
 
     const translateButton = renderTranslate && (
-      <TranslateButton onClick={this.handleTranslate} translation={status.get('translation')} />
+      <TranslateButton
+        onTranslate={this.handleTranslate}
+        onDetectLanguage={renderDetectLanguage && this.handleDetectLanguage}
+        onUndoStatusTranslation={this.props.onUndoStatusTranslation}
+        translation={status.get('translation')}
+      />
     );
 
     const poll = !!status.get('poll') && (

app/javascript/mastodon/containers/status_container.jsx

@@ -126,12 +126,12 @@ const mapDispatchToProps = (dispatch, { contextType }) => ({
     });
   },
 
-  onTranslate (status) {
+  onTranslate (status, sourceLanguage) {
-    if (status.get('translation')) {
+    dispatch(translateStatus(status.get('id'), sourceLanguage));
-      dispatch(undoStatusTranslation(status.get('id'), status.get('poll')));
+  },
-    } else {
+
-      dispatch(translateStatus(status.get('id')));
+  onUndoStatusTranslation (status) {
-    }
+    dispatch(undoStatusTranslation(status.get('id'), status.get('poll')));
   },
 
   onDirect (account) {

app/javascript/mastodon/features/status/components/detailed_status.jsx

@@ -36,6 +36,7 @@ class DetailedStatus extends ImmutablePureComponent {
     onOpenVideo: PropTypes.func.isRequired,
     onToggleHidden: PropTypes.func.isRequired,
     onTranslate: PropTypes.func.isRequired,
+    onUndoStatusTranslation: PropTypes.func.isRequired,
     measureHeight: PropTypes.bool,
     onHeightChange: PropTypes.func,
     domain: PropTypes.string.isRequired,
@@ -103,9 +104,14 @@ class DetailedStatus extends ImmutablePureComponent {
     window.open(href, 'mastodon-intent', 'width=445,height=600,resizable=no,menubar=no,status=no,scrollbars=yes');
   };
 
-  handleTranslate = () => {
+  handleTranslate = (sourceLanguage) => {
     const { onTranslate, status } = this.props;
-    onTranslate(status);
+    onTranslate(status, sourceLanguage);
+  };
+
+  handleUndoStatusTranslation = () => {
+    const { onUndoStatusTranslation, status } = this.props;
+    onUndoStatusTranslation(status);
   };
 
   _properStatus () {
@@ -285,6 +291,7 @@ class DetailedStatus extends ImmutablePureComponent {
               <StatusContent
                 status={status}
                 onTranslate={this.handleTranslate}
+                onUndoStatusTranslation={this.handleUndoStatusTranslation}
                 {...statusContentProps}
               />
 

app/javascript/mastodon/features/status/index.jsx

@@ -392,14 +392,16 @@ class Status extends ImmutablePureComponent {
     }
   };
 
-  handleTranslate = status => {
+  handleTranslate = (status, sourceLanguage) => {
     const { dispatch } = this.props;
 
-    if (status.get('translation')) {
+    dispatch(translateStatus(status.get('id'), sourceLanguage));
-      dispatch(undoStatusTranslation(status.get('id'), status.get('poll')));
+  };
-    } else {
+
-      dispatch(translateStatus(status.get('id')));
+  handleUndoStatusTranslation = status => {
-    }
+    const { dispatch } = this.props;
+
+    dispatch(undoStatusTranslation(status.get('id'), status.get('poll')));
   };
 
   handleBlockClick = (status) => {
@@ -675,6 +677,7 @@ class Status extends ImmutablePureComponent {
                   onOpenMedia={this.handleOpenMedia}
                   onToggleHidden={this.handleToggleHidden}
                   onTranslate={this.handleTranslate}
+                  onUndoStatusTranslation={this.handleUndoStatusTranslation}
                   domain={domain}
                   showMedia={this.state.showMedia}
                   onToggleMediaVisibility={this.handleToggleMediaVisibility}

app/javascript/mastodon/locales/en.json

@@ -779,6 +779,7 @@
   "status.copy": "Copy link to post",
   "status.delete": "Delete",
   "status.detailed_status": "Detailed conversation view",
+  "status.detect_language": "Detect language",
   "status.direct": "Privately mention @{name}",
   "status.direct_indicator": "Private mention",
   "status.edit": "Edit",

app/javascript/styles/mastodon/components.scss

@@ -1312,6 +1312,17 @@ body > [data-popper-placement] {
   display: flex;
   justify-content: space-between;
   color: $dark-text-color;
+
+  .translate-button__buttons {
+    margin-right: -6px;
+    float: right;
+    white-space: nowrap;
+
+    .link-button {
+      display: inline;
+      margin: 0 6px;
+    }
+  }
 }
 
 .status__content__spoiler-link {

app/lib/translation_service/deepl.rb

@@ -11,6 +11,7 @@ class TranslationService::DeepL < TranslationService
   end
 
   def translate(texts, source_language, target_language)
+    source_language = nil if source_language == 'und'
     form = { text: texts, source_lang: source_language&.upcase, target_lang: target_language, tag_handling: 'html' }
     request(:post, '/v2/translate', form: form) do |res|
       transform_response(res.body_with_limit)
@@ -18,7 +19,7 @@ class TranslationService::DeepL < TranslationService
   end
 
   def languages
-    source_languages = [nil] + fetch_languages('source')
+    source_languages = ['und'] + fetch_languages('source')
 
     # In DeepL, EN and PT are deprecated in favor of EN-GB/EN-US and PT-BR/PT-PT, so
     # they are supported but not returned by the API.

app/lib/translation_service/libre_translate.rb

@@ -9,7 +9,8 @@ class TranslationService::LibreTranslate < TranslationService
   end
 
   def translate(texts, source_language, target_language)
-    body = Oj.dump(q: texts, source: source_language.presence || 'auto', target: target_language, format: 'html', api_key: @api_key)
+    source_language = 'auto' if source_language.in? [nil, 'und']
+    body = Oj.dump(q: texts, source: source_language, target: target_language, format: 'html', api_key: @api_key)
     request(:post, '/translate', body: body) do |res|
       transform_response(res.body_with_limit, source_language)
     end
@@ -20,7 +21,7 @@ class TranslationService::LibreTranslate < TranslationService
       languages = Oj.load(res.body_with_limit).to_h do |language|
         [language['code'], language['targets'].without(language['code'])]
       end
-      languages[nil] = languages.values.flatten.uniq.sort
+      languages['und'] = languages.values.flatten.uniq.sort
       languages
     end
   end

app/models/translation.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 class Translation < ActiveModelSerializers::Model
-  attributes :status, :detected_source_language, :language, :provider,
+  attributes :status, :source_language, :detected_source_language, :language, :provider,
              :content, :spoiler_text, :poll_options, :media_attachments
 
   class Option < ActiveModelSerializers::Model

app/serializers/rest/translation_serializer.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 class REST::TranslationSerializer < ActiveModel::Serializer
-  attributes :detected_source_language, :language, :provider, :spoiler_text, :content
+  attributes :source_language, :detected_source_language, :language, :provider, :spoiler_text, :content
 
   class PollSerializer < ActiveModel::Serializer
     attribute :id

app/services/translate_status_service.rb

@@ -6,15 +6,16 @@ class TranslateStatusService < BaseService
   include ERB::Util
   include FormattingHelper
 
-  def call(status, target_language)
+  def call(status, source_language, target_language)
     @status = status
     @source_texts = source_texts
+    @source_language = source_language || @status.language.presence || 'und'
     @target_language = target_language
 
     raise Mastodon::NotPermittedError unless permitted?
 
-    status_translation = Rails.cache.fetch("v2:translations/#{@status.language}/#{@target_language}/#{content_hash}", expires_in: CACHE_TTL) do
+    status_translation = Rails.cache.fetch("v3:translations/#{@source_language}/#{@target_language}/#{content_hash}", expires_in: CACHE_TTL) do
-      translations = translation_backend.translate(@source_texts.values, @status.language, @target_language)
+      translations = translation_backend.translate(@source_texts.values, @source_language, @target_language)
       build_status_translation(translations)
     end
 
@@ -32,11 +33,11 @@ class TranslateStatusService < BaseService
   def permitted?
     return false unless @status.distributable? && TranslationService.configured?
 
-    languages[@status.language]&.include?(@target_language)
+    languages[@source_language]&.include?(@target_language)
   end
 
   def languages
-    Rails.cache.fetch('translation_service/languages', expires_in: 7.days, race_condition_ttl: 1.hour) { TranslationService.configured.languages }
+    Rails.cache.fetch('v2:translation_service/languages', expires_in: 7.days, race_condition_ttl: 1.hour) { TranslationService.configured.languages }
   end
 
   def content_hash
@@ -61,6 +62,7 @@ class TranslateStatusService < BaseService
 
   def build_status_translation(translations)
     status_translation = Translation.new(
+      source_language: @source_language,
       detected_source_language: translations.first&.detected_source_language,
       language: @target_language,
       provider: translations.first&.provider,