Merge 796bcc701d into 3b52dca405

app/javascript/mastodon/features/compose/components/compose_form.jsx

@@ -15,7 +15,6 @@ import { missingAltTextModal } from 'mastodon/initial_state';
 import AutosuggestInput from 'mastodon/components/autosuggest_input';
 import AutosuggestTextarea from 'mastodon/components/autosuggest_textarea';
 import { Button } from 'mastodon/components/button';
-import { LoadingIndicator } from 'mastodon/components/loading_indicator';
 import EmojiPickerDropdown from '../containers/emoji_picker_dropdown_container';
 import PollButtonContainer from '../containers/poll_button_container';
 import PrivacyDropdownContainer from '../containers/privacy_dropdown_container';
@@ -310,7 +309,7 @@ class ComposeForm extends ImmutablePureComponent {
                 >
                   {intl.formatMessage(
                     this.props.isEditing ?
-                      messages.saveChanges : 
+                      messages.saveChanges :
                       (this.props.isInReply ? messages.reply : messages.publish)
                   )}
                 </Button>

app/javascript/mastodon/features/compose/components/language_dropdown.tsx

@@ -20,7 +20,7 @@ import { languages as preloadedLanguages } from 'mastodon/initial_state';
 import type { RootState } from 'mastodon/store';
 import { useAppSelector, useAppDispatch } from 'mastodon/store';
 
-import { debouncedGuess } from '../util/language_detection';
+import { debouncedGuess, countLetters } from '../util/language_detection';
 
 const messages = defineMessages({
   changeLanguage: {
@@ -375,12 +375,25 @@ export const LanguageDropdown: React.FC = () => {
   );
 
   useEffect(() => {
-    if (text.length > 20) {
+    let canceled = false;
-      debouncedGuess(text, setGuess);
+
+    if (countLetters(text) >= 5) {
+      debouncedGuess(text)
+        .then((lang) => {
+          if (!canceled) {
+            setGuess(lang ?? '');
+          }
+        })
+        .catch(() => {
+          setGuess('');
+        });
     } else {
-      debouncedGuess.cancel();
       setGuess('');
     }
+
+    return () => {
+      canceled = true;
+    };
   }, [text, setGuess]);
 
   return (

app/javascript/mastodon/features/compose/util/language_detection.d.ts

@@ -0,0 +1,5 @@
+export declare const debouncedGuess: (
+  text: string,
+) => Promise<string | undefined>;
+
+export declare const countLetters: (text: string) => number;

app/javascript/mastodon/features/compose/util/language_detection.js

@@ -1,7 +1,5 @@
-import lande from 'lande';
+const languageDetectorInGlobalThis = 'LanguageDetector' in globalThis;
-import { debounce } from 'lodash';
+let languageDetectorSupportedAndReady = languageDetectorInGlobalThis && await globalThis.LanguageDetector.availability() === 'available';
-
-import { urlRegex } from './url_regex';
 
 const ISO_639_MAP = {
   afr: 'af', // Afrikaans
@@ -56,21 +54,23 @@ const ISO_639_MAP = {
   vie: 'vi', // Vietnamese
 };
 
-const guessLanguage = (text) => {
+const countLetters = (text) => {
-  text = text
+  const segmenter = new Intl.Segmenter('und', { granularity: 'grapheme' })
-    .replace(urlRegex, '')
+  const letters = [...segmenter.segment(text)]
-    .replace(/(^|[^/\w])@(([a-z0-9_]+)@[a-z0-9.-]+[a-z0-9]+)/ig, '');
+  return letters.length
-
-  if (text.length > 20) {
-    const [lang, confidence] = lande(text)[0];
-  
-    if (confidence > 0.8)
-      return ISO_639_MAP[lang];
-  }
-
-  return '';
 };
 
-export const debouncedGuess = debounce((text, setGuess) => {
+let module;
-  setGuess(guessLanguage(text));
+// If the API is supported, but the model not loaded yet…
-}, 500, { maxWait: 1500, leading: true, trailing: true });
+if (languageDetectorInGlobalThis) {
+  if (!languageDetectorSupportedAndReady) {
+    // …trigger the model download
+    globalThis.LanguageDetector.create();
+  }
+  module = await import('./language_detection_with_languagedetector');
+} else {
+  module = await import('./language_detection_with_laude');
+}
+const debouncedGuess = module.debouncedGuess;
+
+export { debouncedGuess, countLetters, ISO_639_MAP };

app/javascript/mastodon/features/compose/util/language_detection_with_languagedetector.js

@@ -0,0 +1,41 @@
+import { debounce } from 'lodash';
+
+import { urlRegex } from './url_regex';
+
+const guessLanguage = async (text) => {
+  text = text
+    .replace(urlRegex, '')
+    .replace(/(^|[^/\w])@(([a-z0-9_]+)@[a-z0-9.-]+[a-z0-9]+)/ig, '');
+
+  try {
+    const languageDetector = await globalThis.LanguageDetector.create();
+    let {detectedLanguage, confidence} = (await languageDetector.detect(text))[0];
+    if (confidence > 0.8) {
+      detectedLanguage = detectedLanguage.split('-')[0];
+      return detectedLanguage;
+    }
+  } catch {
+    return '';
+  }
+
+  return '';
+};
+
+const debouncedGuess = (() => {
+  let resolver = null;
+
+  const debounced = debounce((text) => {
+    const result = guessLanguage(text);
+    if (resolver) {
+      resolver(result);
+      resolver = null;
+    }
+  }, 500, { maxWait: 1500, leading: true, trailing: true });
+
+  return (text) => new Promise((resolve) => {
+    resolver = resolve;
+    debounced(text);
+  });
+})();
+
+export { debouncedGuess };

app/javascript/mastodon/features/compose/util/language_detection_with_laude.js

@@ -0,0 +1,38 @@
+import lande from 'lande';
+import { debounce } from 'lodash';
+
+import { countLetters, ISO_639_MAP } from './language_detection';
+import { urlRegex } from './url_regex';
+
+const guessLanguage = (text) => {
+  text = text
+    .replace(urlRegex, '')
+    .replace(/(^|[^/\w])@(([a-z0-9_]+)@[a-z0-9.-]+[a-z0-9]+)/ig, '');
+
+  if (countLetters(text) > 20) {
+    const [lang, confidence] = lande(text)[0];
+    if (confidence > 0.8)
+      return ISO_639_MAP[lang];
+  }
+
+  return '';
+};
+
+const debouncedGuess = (() => {
+  let resolver = null;
+
+  const debounced = debounce((text) => {
+    const result = guessLanguage(text);
+    if (resolver) {
+      resolver(result);
+      resolver = null;
+    }
+  }, 500, { maxWait: 1500, leading: true, trailing: true });
+
+  return (text) => new Promise((resolve) => {
+    resolver = resolve;
+    debounced(text);
+  });
+})();
+
+export { debouncedGuess };