diff --git a/app/javascript/mastodon/features/compose/components/language_dropdown.tsx b/app/javascript/mastodon/features/compose/components/language_dropdown.tsx index 72742153b1c..bf0069f353a 100644 --- a/app/javascript/mastodon/features/compose/components/language_dropdown.tsx +++ b/app/javascript/mastodon/features/compose/components/language_dropdown.tsx @@ -20,7 +20,7 @@ import { languages as preloadedLanguages } from 'mastodon/initial_state'; import type { RootState } from 'mastodon/store'; import { useAppSelector, useAppDispatch } from 'mastodon/store'; -import { debouncedGuess } from '../util/language_detection'; +import { debouncedGuess, countLetters } from '../util/language_detection'; const messages = defineMessages({ changeLanguage: { @@ -375,12 +375,25 @@ export const LanguageDropdown: React.FC = () => { ); useEffect(() => { - if (text.length > 20) { - debouncedGuess(text, setGuess); + let canceled = false; + + if (countLetters(text) >= 5) { + debouncedGuess(text) + .then((lang) => { + if (!canceled) { + setGuess(lang ?? ''); + } + }) + .catch(() => { + setGuess(''); + }); } else { - debouncedGuess.cancel(); setGuess(''); } + + return () => { + canceled = true; + }; }, [text, setGuess]); return ( diff --git a/app/javascript/mastodon/features/compose/util/language_detection.d.ts b/app/javascript/mastodon/features/compose/util/language_detection.d.ts new file mode 100644 index 00000000000..289c0476b85 --- /dev/null +++ b/app/javascript/mastodon/features/compose/util/language_detection.d.ts @@ -0,0 +1,5 @@ +export declare const debouncedGuess: ( + text: string, +) => Promise; + +export declare const countLetters: (text: string) => number; diff --git a/app/javascript/mastodon/features/compose/util/language_detection.js b/app/javascript/mastodon/features/compose/util/language_detection.js index ed22a2bd9ca..b82b60fa37e 100644 --- a/app/javascript/mastodon/features/compose/util/language_detection.js +++ b/app/javascript/mastodon/features/compose/util/language_detection.js @@ -1,7 +1,5 @@ -import lande from 'lande'; -import { debounce } from 'lodash'; - -import { urlRegex } from './url_regex'; +const languageDetectorInGlobalThis = 'LanguageDetector' in globalThis; +let languageDetectorSupportedAndReady = languageDetectorInGlobalThis && await globalThis.LanguageDetector.availability() === 'available'; const ISO_639_MAP = { afr: 'af', // Afrikaans @@ -56,21 +54,23 @@ const ISO_639_MAP = { vie: 'vi', // Vietnamese }; -const guessLanguage = (text) => { - text = text - .replace(urlRegex, '') - .replace(/(^|[^/\w])@(([a-z0-9_]+)@[a-z0-9.-]+[a-z0-9]+)/ig, ''); - - if (text.length > 20) { - const [lang, confidence] = lande(text)[0]; - - if (confidence > 0.8) - return ISO_639_MAP[lang]; - } - - return ''; +const countLetters = (text) => { + const segmenter = new Intl.Segmenter('und', { granularity: 'grapheme' }) + const letters = [...segmenter.segment(text)] + return letters.length }; -export const debouncedGuess = debounce((text, setGuess) => { - setGuess(guessLanguage(text)); -}, 500, { maxWait: 1500, leading: true, trailing: true }); +let module; +// If the API is supported, but the model not loaded yet… +if (languageDetectorInGlobalThis) { + if (!languageDetectorSupportedAndReady) { + // …trigger the model download + globalThis.LanguageDetector.create(); + } + module = await import('./language_detection_with_languagedetector'); +} else { + module = await import('./language_detection_with_lande'); +} +const debouncedGuess = module.debouncedGuess; + +export { debouncedGuess, countLetters, ISO_639_MAP }; diff --git a/app/javascript/mastodon/features/compose/util/language_detection_with_lande.js b/app/javascript/mastodon/features/compose/util/language_detection_with_lande.js new file mode 100644 index 00000000000..aa6e768a2c5 --- /dev/null +++ b/app/javascript/mastodon/features/compose/util/language_detection_with_lande.js @@ -0,0 +1,38 @@ +import lande from 'lande'; +import { debounce } from 'lodash'; + +import { countLetters, ISO_639_MAP } from './language_detection'; +import { urlRegex } from './url_regex'; + +const guessLanguage = (text) => { + text = text + .replace(urlRegex, '') + .replace(/(^|[^/\w])@(([a-z0-9_]+)@[a-z0-9.-]+[a-z0-9]+)/ig, ''); + + if (countLetters(text) > 20) { + const [lang, confidence] = lande(text)[0]; + if (confidence > 0.8) + return ISO_639_MAP[lang]; + } + + return ''; +}; + +const debouncedGuess = (() => { + let resolver = null; + + const debounced = debounce((text) => { + const result = guessLanguage(text); + if (resolver) { + resolver(result); + resolver = null; + } + }, 500, { maxWait: 1500, leading: true, trailing: true }); + + return (text) => new Promise((resolve) => { + resolver = resolve; + debounced(text); + }); +})(); + +export { debouncedGuess }; diff --git a/app/javascript/mastodon/features/compose/util/language_detection_with_languagedetector.js b/app/javascript/mastodon/features/compose/util/language_detection_with_languagedetector.js new file mode 100644 index 00000000000..de5df6e0921 --- /dev/null +++ b/app/javascript/mastodon/features/compose/util/language_detection_with_languagedetector.js @@ -0,0 +1,41 @@ +import { debounce } from 'lodash'; + +import { urlRegex } from './url_regex'; + +const guessLanguage = async (text) => { + text = text + .replace(urlRegex, '') + .replace(/(^|[^/\w])@(([a-z0-9_]+)@[a-z0-9.-]+[a-z0-9]+)/ig, ''); + + try { + const languageDetector = await globalThis.LanguageDetector.create(); + let {detectedLanguage, confidence} = (await languageDetector.detect(text))[0]; + if (confidence > 0.8) { + detectedLanguage = detectedLanguage.split('-')[0]; + return detectedLanguage; + } + } catch { + return ''; + } + + return ''; +}; + +const debouncedGuess = (() => { + let resolver = null; + + const debounced = debounce((text) => { + const result = guessLanguage(text); + if (resolver) { + resolver(result); + resolver = null; + } + }, 500, { maxWait: 1500, leading: true, trailing: true }); + + return (text) => new Promise((resolve) => { + resolver = resolve; + debounced(text); + }); +})(); + +export { debouncedGuess };