From 6bca52453af3a4e76063e28e797f5e5b45ba74aa Mon Sep 17 00:00:00 2001 From: Echo Date: Thu, 31 Jul 2025 19:30:14 +0200 Subject: [PATCH] Emoji Rendering Efficiency (#35568) --- .../mastodon/features/emoji/constants.ts | 11 + .../mastodon/features/emoji/database.test.ts | 139 ++++++++++++ .../mastodon/features/emoji/database.ts | 166 ++++++++++---- .../mastodon/features/emoji/emoji_html.tsx | 84 ++----- .../mastodon/features/emoji/emoji_text.tsx | 45 ---- .../mastodon/features/emoji/hooks.ts | 65 +++++- .../mastodon/features/emoji/index.ts | 32 ++- .../mastodon/features/emoji/loader.ts | 10 +- .../mastodon/features/emoji/render.test.ts | 214 +++++++++++++----- .../mastodon/features/emoji/render.ts | 198 ++++++++++------ .../mastodon/features/emoji/types.ts | 18 +- .../mastodon/features/emoji/utils.test.ts | 38 +++- .../mastodon/features/emoji/utils.ts | 32 ++- .../mastodon/features/emoji/worker.ts | 15 +- app/javascript/mastodon/main.tsx | 4 +- .../mastodon/utils/__tests__/cache.test.ts | 78 +++++++ app/javascript/mastodon/utils/cache.ts | 60 +++++ .../{performance.js => utils/performance.ts} | 6 +- app/javascript/testing/factories.ts | 27 +++ eslint.config.mjs | 6 + package.json | 4 +- vitest.config.mts | 1 + yarn.lock | 34 ++- 23 files changed, 954 insertions(+), 333 deletions(-) create mode 100644 app/javascript/mastodon/features/emoji/database.test.ts delete mode 100644 app/javascript/mastodon/features/emoji/emoji_text.tsx create mode 100644 app/javascript/mastodon/utils/__tests__/cache.test.ts create mode 100644 app/javascript/mastodon/utils/cache.ts rename app/javascript/mastodon/{performance.js => utils/performance.ts} (70%) diff --git a/app/javascript/mastodon/features/emoji/constants.ts b/app/javascript/mastodon/features/emoji/constants.ts index 09022371b22..a5ec9e6e2b4 100644 --- a/app/javascript/mastodon/features/emoji/constants.ts +++ b/app/javascript/mastodon/features/emoji/constants.ts @@ -15,6 +15,17 @@ export const SKIN_TONE_CODES = [ 0x1f3ff, // Dark skin tone ] as const; +// TODO: Test and create fallback for browsers that do not handle the /v flag. +export const UNICODE_EMOJI_REGEX = /\p{RGI_Emoji}/v; +// See: https://www.unicode.org/reports/tr51/#valid-emoji-tag-sequences +export const UNICODE_FLAG_EMOJI_REGEX = + /\p{RGI_Emoji_Flag_Sequence}|\p{RGI_Emoji_Tag_Sequence}/v; +export const CUSTOM_EMOJI_REGEX = /:([a-z0-9_]+):/i; +export const ANY_EMOJI_REGEX = new RegExp( + `(${UNICODE_EMOJI_REGEX.source}|${CUSTOM_EMOJI_REGEX.source})`, + 'gv', +); + // Emoji rendering modes. A mode is what we are using to render emojis, a style is what the user has selected. export const EMOJI_MODE_NATIVE = 'native'; export const EMOJI_MODE_NATIVE_WITH_FLAGS = 'native-flags'; diff --git a/app/javascript/mastodon/features/emoji/database.test.ts b/app/javascript/mastodon/features/emoji/database.test.ts new file mode 100644 index 00000000000..0689fd7c542 --- /dev/null +++ b/app/javascript/mastodon/features/emoji/database.test.ts @@ -0,0 +1,139 @@ +import { IDBFactory } from 'fake-indexeddb'; + +import { unicodeEmojiFactory } from '@/testing/factories'; + +import { + putEmojiData, + loadEmojiByHexcode, + searchEmojisByHexcodes, + searchEmojisByTag, + testClear, + testGet, +} from './database'; + +describe('emoji database', () => { + afterEach(() => { + testClear(); + indexedDB = new IDBFactory(); + }); + describe('putEmojiData', () => { + test('adds to loaded locales', async () => { + const { loadedLocales } = await testGet(); + expect(loadedLocales).toHaveLength(0); + await putEmojiData([], 'en'); + expect(loadedLocales).toContain('en'); + }); + + test('loads emoji into indexedDB', async () => { + await putEmojiData([unicodeEmojiFactory()], 'en'); + const { db } = await testGet(); + await expect(db.get('en', 'test')).resolves.toEqual( + unicodeEmojiFactory(), + ); + }); + }); + + describe('loadEmojiByHexcode', () => { + test('throws if the locale is not loaded', async () => { + await expect(loadEmojiByHexcode('en', 'test')).rejects.toThrowError( + 'Locale en', + ); + }); + + test('retrieves the emoji', async () => { + await putEmojiData([unicodeEmojiFactory()], 'en'); + await expect(loadEmojiByHexcode('test', 'en')).resolves.toEqual( + unicodeEmojiFactory(), + ); + }); + + test('returns undefined if not found', async () => { + await putEmojiData([], 'en'); + await expect(loadEmojiByHexcode('test', 'en')).resolves.toBeUndefined(); + }); + }); + + describe('searchEmojisByHexcodes', () => { + const data = [ + unicodeEmojiFactory({ hexcode: 'not a number' }), + unicodeEmojiFactory({ hexcode: '1' }), + unicodeEmojiFactory({ hexcode: '2' }), + unicodeEmojiFactory({ hexcode: '3' }), + unicodeEmojiFactory({ hexcode: 'another not a number' }), + ]; + beforeEach(async () => { + await putEmojiData(data, 'en'); + }); + test('finds emoji in consecutive range', async () => { + const actual = await searchEmojisByHexcodes(['1', '2', '3'], 'en'); + expect(actual).toHaveLength(3); + }); + + test('finds emoji in split range', async () => { + const actual = await searchEmojisByHexcodes(['1', '3'], 'en'); + expect(actual).toHaveLength(2); + expect(actual).toContainEqual(data.at(1)); + expect(actual).toContainEqual(data.at(3)); + }); + + test('finds emoji with non-numeric range', async () => { + const actual = await searchEmojisByHexcodes( + ['3', 'not a number', '1'], + 'en', + ); + expect(actual).toHaveLength(3); + expect(actual).toContainEqual(data.at(0)); + expect(actual).toContainEqual(data.at(1)); + expect(actual).toContainEqual(data.at(3)); + }); + + test('not found emoji are not returned', async () => { + const actual = await searchEmojisByHexcodes(['not found'], 'en'); + expect(actual).toHaveLength(0); + }); + + test('only found emojis are returned', async () => { + const actual = await searchEmojisByHexcodes( + ['another not a number', 'not found'], + 'en', + ); + expect(actual).toHaveLength(1); + expect(actual).toContainEqual(data.at(4)); + }); + }); + + describe('searchEmojisByTag', () => { + const data = [ + unicodeEmojiFactory({ hexcode: 'test1', tags: ['test 1'] }), + unicodeEmojiFactory({ + hexcode: 'test2', + tags: ['test 2', 'something else'], + }), + unicodeEmojiFactory({ hexcode: 'test3', tags: ['completely different'] }), + ]; + beforeEach(async () => { + await putEmojiData(data, 'en'); + }); + test('finds emojis with tag', async () => { + const actual = await searchEmojisByTag('test 1', 'en'); + expect(actual).toHaveLength(1); + expect(actual).toContainEqual(data.at(0)); + }); + + test('finds emojis starting with tag', async () => { + const actual = await searchEmojisByTag('test', 'en'); + expect(actual).toHaveLength(2); + expect(actual).not.toContainEqual(data.at(2)); + }); + + test('does not find emojis ending with tag', async () => { + const actual = await searchEmojisByTag('else', 'en'); + expect(actual).toHaveLength(0); + }); + + test('finds nothing with invalid tag', async () => { + const actual = await searchEmojisByTag('not found', 'en'); + expect(actual).toHaveLength(0); + }); + }); +}); diff --git a/app/javascript/mastodon/features/emoji/database.ts b/app/javascript/mastodon/features/emoji/database.ts index 0b8ddd34fbe..0e8ada1d0e0 100644 --- a/app/javascript/mastodon/features/emoji/database.ts +++ b/app/javascript/mastodon/features/emoji/database.ts @@ -9,6 +9,7 @@ import type { UnicodeEmojiData, LocaleOrCustom, } from './types'; +import { emojiLogger } from './utils'; interface EmojiDB extends LocaleTables, DBSchema { custom: { @@ -36,40 +37,63 @@ interface LocaleTable { } type LocaleTables = Record; +type Database = IDBPDatabase; + const SCHEMA_VERSION = 1; -let db: IDBPDatabase | null = null; +const loadedLocales = new Set(); -async function loadDB() { - if (db) { - return db; - } - db = await openDB('mastodon-emoji', SCHEMA_VERSION, { - upgrade(database) { - const customTable = database.createObjectStore('custom', { - keyPath: 'shortcode', - autoIncrement: false, - }); - customTable.createIndex('category', 'category'); +const log = emojiLogger('database'); - database.createObjectStore('etags'); +// Loads the database in a way that ensures it's only loaded once. +const loadDB = (() => { + let dbPromise: Promise | null = null; - for (const locale of SUPPORTED_LOCALES) { - const localeTable = database.createObjectStore(locale, { - keyPath: 'hexcode', + // Actually load the DB. + async function initDB() { + const db = await openDB('mastodon-emoji', SCHEMA_VERSION, { + upgrade(database) { + const customTable = database.createObjectStore('custom', { + keyPath: 'shortcode', autoIncrement: false, }); - localeTable.createIndex('group', 'group'); - localeTable.createIndex('label', 'label'); - localeTable.createIndex('order', 'order'); - localeTable.createIndex('tags', 'tags', { multiEntry: true }); - } - }, - }); - return db; -} + customTable.createIndex('category', 'category'); + + database.createObjectStore('etags'); + + for (const locale of SUPPORTED_LOCALES) { + const localeTable = database.createObjectStore(locale, { + keyPath: 'hexcode', + autoIncrement: false, + }); + localeTable.createIndex('group', 'group'); + localeTable.createIndex('label', 'label'); + localeTable.createIndex('order', 'order'); + localeTable.createIndex('tags', 'tags', { multiEntry: true }); + } + }, + }); + await syncLocales(db); + return db; + } + + // Loads the database, or returns the existing promise if it hasn't resolved yet. + const loadPromise = async (): Promise => { + if (dbPromise) { + return dbPromise; + } + dbPromise = initDB(); + return dbPromise; + }; + // Special way to reset the database, used for unit testing. + loadPromise.reset = () => { + dbPromise = null; + }; + return loadPromise; +})(); export async function putEmojiData(emojis: UnicodeEmojiData[], locale: Locale) { + loadedLocales.add(locale); const db = await loadDB(); const trx = db.transaction(locale, 'readwrite'); await Promise.all(emojis.map((emoji) => trx.store.put(emoji))); @@ -86,15 +110,15 @@ export async function putCustomEmojiData(emojis: CustomEmojiData[]) { export async function putLatestEtag(etag: string, localeString: string) { const locale = toSupportedLocaleOrCustom(localeString); const db = await loadDB(); - return db.put('etags', etag, locale); + await db.put('etags', etag, locale); } -export async function searchEmojiByHexcode( +export async function loadEmojiByHexcode( hexcode: string, localeString: string, ) { - const locale = toSupportedLocale(localeString); const db = await loadDB(); + const locale = toLoadedLocale(localeString); return db.get(locale, hexcode); } @@ -102,45 +126,39 @@ export async function searchEmojisByHexcodes( hexcodes: string[], localeString: string, ) { - const locale = toSupportedLocale(localeString); const db = await loadDB(); - return db.getAll( + const locale = toLoadedLocale(localeString); + const sortedCodes = hexcodes.toSorted(); + const results = await db.getAll( locale, - IDBKeyRange.bound(hexcodes[0], hexcodes[hexcodes.length - 1]), + IDBKeyRange.bound(sortedCodes.at(0), sortedCodes.at(-1)), ); + return results.filter((emoji) => hexcodes.includes(emoji.hexcode)); } -export async function searchEmojiByTag(tag: string, localeString: string) { - const locale = toSupportedLocale(localeString); - const range = IDBKeyRange.only(tag.toLowerCase()); +export async function searchEmojisByTag(tag: string, localeString: string) { const db = await loadDB(); + const locale = toLoadedLocale(localeString); + const range = IDBKeyRange.bound( + tag.toLowerCase(), + `${tag.toLowerCase()}\uffff`, + ); return db.getAllFromIndex(locale, 'tags', range); } -export async function searchCustomEmojiByShortcode(shortcode: string) { +export async function loadCustomEmojiByShortcode(shortcode: string) { const db = await loadDB(); return db.get('custom', shortcode); } export async function searchCustomEmojisByShortcodes(shortcodes: string[]) { const db = await loadDB(); - return db.getAll( + const sortedCodes = shortcodes.toSorted(); + const results = await db.getAll( 'custom', - IDBKeyRange.bound(shortcodes[0], shortcodes[shortcodes.length - 1]), + IDBKeyRange.bound(sortedCodes.at(0), sortedCodes.at(-1)), ); -} - -export async function findMissingLocales(localeStrings: string[]) { - const locales = new Set(localeStrings.map(toSupportedLocale)); - const missingLocales: Locale[] = []; - const db = await loadDB(); - for (const locale of locales) { - const rowCount = await db.count(locale); - if (!rowCount) { - missingLocales.push(locale); - } - } - return missingLocales; + return results.filter((emoji) => shortcodes.includes(emoji.shortcode)); } export async function loadLatestEtag(localeString: string) { @@ -153,3 +171,51 @@ export async function loadLatestEtag(localeString: string) { const etag = await db.get('etags', locale); return etag ?? null; } + +// Private functions + +async function syncLocales(db: Database) { + const locales = await Promise.all( + SUPPORTED_LOCALES.map( + async (locale) => + [locale, await hasLocale(locale, db)] satisfies [Locale, boolean], + ), + ); + for (const [locale, loaded] of locales) { + if (loaded) { + loadedLocales.add(locale); + } else { + loadedLocales.delete(locale); + } + } + log('Loaded %d locales: %o', loadedLocales.size, loadedLocales); +} + +function toLoadedLocale(localeString: string) { + const locale = toSupportedLocale(localeString); + if (localeString !== locale) { + log(`Locale ${locale} is different from provided ${localeString}`); + } + if (!loadedLocales.has(locale)) { + throw new Error(`Locale ${locale} is not loaded in emoji database`); + } + return locale; +} + +async function hasLocale(locale: Locale, db: Database): Promise { + if (loadedLocales.has(locale)) { + return true; + } + const rowCount = await db.count(locale); + return !!rowCount; +} + +// Testing helpers +export async function testGet() { + const db = await loadDB(); + return { db, loadedLocales }; +} +export function testClear() { + loadedLocales.clear(); + loadDB.reset(); +} diff --git a/app/javascript/mastodon/features/emoji/emoji_html.tsx b/app/javascript/mastodon/features/emoji/emoji_html.tsx index 85628e6723d..fdda62a3e61 100644 --- a/app/javascript/mastodon/features/emoji/emoji_html.tsx +++ b/app/javascript/mastodon/features/emoji/emoji_html.tsx @@ -1,81 +1,31 @@ -import type { HTMLAttributes } from 'react'; -import { useEffect, useMemo, useState } from 'react'; +import type { ComponentPropsWithoutRef, ElementType } from 'react'; -import type { List as ImmutableList } from 'immutable'; -import { isList } from 'immutable'; +import { useEmojify } from './hooks'; +import type { CustomEmojiMapArg } from './types'; -import type { ApiCustomEmojiJSON } from '@/mastodon/api_types/custom_emoji'; -import type { CustomEmoji } from '@/mastodon/models/custom_emoji'; -import { isModernEmojiEnabled } from '@/mastodon/utils/environment'; - -import { useEmojiAppState } from './hooks'; -import { emojifyElement } from './render'; -import type { ExtraCustomEmojiMap } from './types'; - -type EmojiHTMLProps = Omit< - HTMLAttributes, +type EmojiHTMLProps = Omit< + ComponentPropsWithoutRef, 'dangerouslySetInnerHTML' > & { htmlString: string; - extraEmojis?: ExtraCustomEmojiMap | ImmutableList; + extraEmojis?: CustomEmojiMapArg; + as?: Element; }; -export const EmojiHTML: React.FC = ({ - htmlString, +export const EmojiHTML = ({ extraEmojis, + htmlString, + as: asElement, // Rename for syntax highlighting ...props -}) => { - if (isModernEmojiEnabled()) { - return ( - - ); - } - return
; -}; +}: EmojiHTMLProps) => { + const Wrapper = asElement ?? 'div'; + const emojifiedHtml = useEmojify(htmlString, extraEmojis); -const ModernEmojiHTML: React.FC = ({ - extraEmojis: rawEmojis, - htmlString: text, - ...props -}) => { - const appState = useEmojiAppState(); - const [innerHTML, setInnerHTML] = useState(''); - - const extraEmojis: ExtraCustomEmojiMap = useMemo(() => { - if (!rawEmojis) { - return {}; - } - if (isList(rawEmojis)) { - return ( - rawEmojis.toJS() as ApiCustomEmojiJSON[] - ).reduce( - (acc, emoji) => ({ ...acc, [emoji.shortcode]: emoji }), - {}, - ); - } - return rawEmojis; - }, [rawEmojis]); - - useEffect(() => { - if (!text) { - return; - } - const cb = async () => { - const div = document.createElement('div'); - div.innerHTML = text; - const ele = await emojifyElement(div, appState, extraEmojis); - setInnerHTML(ele.innerHTML); - }; - void cb(); - }, [text, appState, extraEmojis]); - - if (!innerHTML) { + if (emojifiedHtml === null) { return null; } - return
; + return ( + + ); }; diff --git a/app/javascript/mastodon/features/emoji/emoji_text.tsx b/app/javascript/mastodon/features/emoji/emoji_text.tsx deleted file mode 100644 index 253371391a4..00000000000 --- a/app/javascript/mastodon/features/emoji/emoji_text.tsx +++ /dev/null @@ -1,45 +0,0 @@ -import { useEffect, useState } from 'react'; - -import { useEmojiAppState } from './hooks'; -import { emojifyText } from './render'; - -interface EmojiTextProps { - text: string; -} - -export const EmojiText: React.FC = ({ text }) => { - const appState = useEmojiAppState(); - const [rendered, setRendered] = useState<(string | HTMLImageElement)[]>([]); - - useEffect(() => { - const cb = async () => { - const rendered = await emojifyText(text, appState); - setRendered(rendered ?? []); - }; - void cb(); - }, [text, appState]); - - if (rendered.length === 0) { - return null; - } - - return ( - <> - {rendered.map((fragment, index) => { - if (typeof fragment === 'string') { - return {fragment}; - } - return ( - {fragment.alt} - ); - })} - - ); -}; diff --git a/app/javascript/mastodon/features/emoji/hooks.ts b/app/javascript/mastodon/features/emoji/hooks.ts index fd38129a19b..47af37b3731 100644 --- a/app/javascript/mastodon/features/emoji/hooks.ts +++ b/app/javascript/mastodon/features/emoji/hooks.ts @@ -1,8 +1,64 @@ +import { useCallback, useLayoutEffect, useMemo, useState } from 'react'; + +import { isList } from 'immutable'; + +import type { ApiCustomEmojiJSON } from '@/mastodon/api_types/custom_emoji'; import { useAppSelector } from '@/mastodon/store'; +import { isModernEmojiEnabled } from '@/mastodon/utils/environment'; import { toSupportedLocale } from './locale'; import { determineEmojiMode } from './mode'; -import type { EmojiAppState } from './types'; +import { emojifyElement } from './render'; +import type { + CustomEmojiMapArg, + EmojiAppState, + ExtraCustomEmojiMap, +} from './types'; +import { stringHasAnyEmoji } from './utils'; + +export function useEmojify(text: string, extraEmojis?: CustomEmojiMapArg) { + const [emojifiedText, setEmojifiedText] = useState(null); + + const appState = useEmojiAppState(); + const extra: ExtraCustomEmojiMap = useMemo(() => { + if (!extraEmojis) { + return {}; + } + if (isList(extraEmojis)) { + return ( + extraEmojis.toJS() as ApiCustomEmojiJSON[] + ).reduce( + (acc, emoji) => ({ ...acc, [emoji.shortcode]: emoji }), + {}, + ); + } + return extraEmojis; + }, [extraEmojis]); + + const emojify = useCallback( + async (input: string) => { + const wrapper = document.createElement('div'); + wrapper.innerHTML = input; + const result = await emojifyElement(wrapper, appState, extra); + if (result) { + setEmojifiedText(result.innerHTML); + } else { + setEmojifiedText(input); + } + }, + [appState, extra], + ); + useLayoutEffect(() => { + if (isModernEmojiEnabled() && !!text.trim() && stringHasAnyEmoji(text)) { + void emojify(text); + } else { + // If no emoji or we don't want to render, fall back. + setEmojifiedText(text); + } + }, [emojify, text]); + + return emojifiedText; +} export function useEmojiAppState(): EmojiAppState { const locale = useAppSelector((state) => @@ -12,5 +68,10 @@ export function useEmojiAppState(): EmojiAppState { determineEmojiMode(state.meta.get('emoji_style') as string), ); - return { currentLocale: locale, locales: [locale], mode }; + return { + currentLocale: locale, + locales: [locale], + mode, + darkTheme: document.body.classList.contains('theme-default'), + }; } diff --git a/app/javascript/mastodon/features/emoji/index.ts b/app/javascript/mastodon/features/emoji/index.ts index 541cea9aa99..99c16fe361c 100644 --- a/app/javascript/mastodon/features/emoji/index.ts +++ b/app/javascript/mastodon/features/emoji/index.ts @@ -2,12 +2,16 @@ import initialState from '@/mastodon/initial_state'; import { loadWorker } from '@/mastodon/utils/workers'; import { toSupportedLocale } from './locale'; +import { emojiLogger } from './utils'; const userLocale = toSupportedLocale(initialState?.meta.locale ?? 'en'); let worker: Worker | null = null; -export async function initializeEmoji() { +const log = emojiLogger('index'); + +export function initializeEmoji() { + log('initializing emojis'); if (!worker && 'Worker' in window) { try { worker = loadWorker(new URL('./worker', import.meta.url), { @@ -21,9 +25,16 @@ export async function initializeEmoji() { if (worker) { // Assign worker to const to make TS happy inside the event listener. const thisWorker = worker; + const timeoutId = setTimeout(() => { + log('worker is not ready after timeout'); + worker = null; + void fallbackLoad(); + }, 500); thisWorker.addEventListener('message', (event: MessageEvent) => { const { data: message } = event; if (message === 'ready') { + log('worker ready, loading data'); + clearTimeout(timeoutId); thisWorker.postMessage('custom'); void loadEmojiLocale(userLocale); // Load English locale as well, because people are still used to @@ -31,15 +42,22 @@ export async function initializeEmoji() { if (userLocale !== 'en') { void loadEmojiLocale('en'); } + } else { + log('got worker message: %s', message); } }); } else { - const { importCustomEmojiData } = await import('./loader'); - await importCustomEmojiData(); - await loadEmojiLocale(userLocale); - if (userLocale !== 'en') { - await loadEmojiLocale('en'); - } + void fallbackLoad(); + } +} + +async function fallbackLoad() { + log('falling back to main thread for loading'); + const { importCustomEmojiData } = await import('./loader'); + await importCustomEmojiData(); + await loadEmojiLocale(userLocale); + if (userLocale !== 'en') { + await loadEmojiLocale('en'); } } diff --git a/app/javascript/mastodon/features/emoji/loader.ts b/app/javascript/mastodon/features/emoji/loader.ts index 454b8383f07..72f57b6f6c0 100644 --- a/app/javascript/mastodon/features/emoji/loader.ts +++ b/app/javascript/mastodon/features/emoji/loader.ts @@ -2,7 +2,6 @@ import { flattenEmojiData } from 'emojibase'; import type { CompactEmoji, FlatCompactEmoji } from 'emojibase'; import type { ApiCustomEmojiJSON } from '@/mastodon/api_types/custom_emoji'; -import { isDevelopment } from '@/mastodon/utils/environment'; import { putEmojiData, @@ -12,6 +11,9 @@ import { } from './database'; import { toSupportedLocale, toSupportedLocaleOrCustom } from './locale'; import type { LocaleOrCustom } from './types'; +import { emojiLogger } from './utils'; + +const log = emojiLogger('loader'); export async function importEmojiData(localeString: string) { const locale = toSupportedLocale(localeString); @@ -20,6 +22,7 @@ export async function importEmojiData(localeString: string) { return; } const flattenedEmojis: FlatCompactEmoji[] = flattenEmojiData(emojis); + log('loaded %d for %s locale', flattenedEmojis.length, locale); await putEmojiData(flattenedEmojis, locale); } @@ -28,6 +31,7 @@ export async function importCustomEmojiData() { if (!emojis) { return; } + log('loaded %d custom emojis', emojis.length); await putCustomEmojiData(emojis); } @@ -41,7 +45,9 @@ async function fetchAndCheckEtag( if (locale === 'custom') { url.pathname = '/api/v1/custom_emojis'; } else { - url.pathname = `/packs${isDevelopment() ? '-dev' : ''}/emoji/${locale}.json`; + // This doesn't use isDevelopment() as that module loads initial state + // which breaks workers, as they cannot access the DOM. + url.pathname = `/packs${import.meta.env.DEV ? '-dev' : ''}/emoji/${locale}.json`; } const oldEtag = await loadLatestEtag(locale); diff --git a/app/javascript/mastodon/features/emoji/render.test.ts b/app/javascript/mastodon/features/emoji/render.test.ts index 23f85c36b3e..e9609e15dc5 100644 --- a/app/javascript/mastodon/features/emoji/render.test.ts +++ b/app/javascript/mastodon/features/emoji/render.test.ts @@ -1,94 +1,184 @@ +import { customEmojiFactory, unicodeEmojiFactory } from '@/testing/factories'; + import { EMOJI_MODE_NATIVE, EMOJI_MODE_NATIVE_WITH_FLAGS, EMOJI_MODE_TWEMOJI, } from './constants'; -import { emojifyElement, tokenizeText } from './render'; -import type { CustomEmojiData, UnicodeEmojiData } from './types'; +import * as db from './database'; +import { + emojifyElement, + emojifyText, + testCacheClear, + tokenizeText, +} from './render'; +import type { EmojiAppState, ExtraCustomEmojiMap } from './types'; -vitest.mock('./database', () => ({ - searchCustomEmojisByShortcodes: vitest.fn( - () => - [ - { - shortcode: 'custom', - static_url: 'emoji/static', - url: 'emoji/custom', - category: 'test', - visible_in_picker: true, - }, - ] satisfies CustomEmojiData[], - ), - searchEmojisByHexcodes: vitest.fn( - () => - [ - { +function mockDatabase() { + return { + searchCustomEmojisByShortcodes: vi + .spyOn(db, 'searchCustomEmojisByShortcodes') + .mockResolvedValue([customEmojiFactory()]), + searchEmojisByHexcodes: vi + .spyOn(db, 'searchEmojisByHexcodes') + .mockResolvedValue([ + unicodeEmojiFactory({ hexcode: '1F60A', - group: 0, label: 'smiling face with smiling eyes', - order: 0, - tags: ['smile', 'happy'], unicode: '😊', - }, - { + }), + unicodeEmojiFactory({ hexcode: '1F1EA-1F1FA', - group: 0, label: 'flag-eu', - order: 0, - tags: ['flag', 'european union'], unicode: 'πŸ‡ͺπŸ‡Ί', - }, - ] satisfies UnicodeEmojiData[], - ), - findMissingLocales: vitest.fn(() => []), -})); + }), + ]), + }; +} + +const expectedSmileImage = + '😊'; +const expectedFlagImage = + 'πŸ‡ͺπŸ‡Ί'; +const expectedCustomEmojiImage = + ':custom:'; +const expectedRemoteCustomEmojiImage = + ':remote:'; + +const mockExtraCustom: ExtraCustomEmojiMap = { + remote: { + shortcode: 'remote', + static_url: 'remote.social/static', + url: 'remote.social/custom', + }, +}; + +function testAppState(state: Partial = {}) { + return { + locales: ['en'], + mode: EMOJI_MODE_TWEMOJI, + currentLocale: 'en', + darkTheme: false, + ...state, + } satisfies EmojiAppState; +} describe('emojifyElement', () => { - const testElement = document.createElement('div'); - testElement.innerHTML = '

Hello 😊πŸ‡ͺπŸ‡Ί!

:custom:

'; - - const expectedSmileImage = - '😊'; - const expectedFlagImage = - 'πŸ‡ͺπŸ‡Ί'; - const expectedCustomEmojiImage = - ':custom:'; - - function cloneTestElement() { - return testElement.cloneNode(true) as HTMLElement; + function testElement(text = '

Hello 😊πŸ‡ͺπŸ‡Ί!

:custom:

') { + const testElement = document.createElement('div'); + testElement.innerHTML = text; + return testElement; } + afterEach(() => { + testCacheClear(); + vi.restoreAllMocks(); + }); + + test('caches element rendering results', async () => { + const { searchCustomEmojisByShortcodes, searchEmojisByHexcodes } = + mockDatabase(); + await emojifyElement(testElement(), testAppState()); + await emojifyElement(testElement(), testAppState()); + await emojifyElement(testElement(), testAppState()); + expect(searchEmojisByHexcodes).toHaveBeenCalledExactlyOnceWith( + ['1F1EA-1F1FA', '1F60A'], + 'en', + ); + expect(searchCustomEmojisByShortcodes).toHaveBeenCalledExactlyOnceWith([ + 'custom', + ]); + }); + test('emojifies custom emoji in native mode', async () => { - const emojifiedElement = await emojifyElement(cloneTestElement(), { - locales: ['en'], - mode: EMOJI_MODE_NATIVE, - currentLocale: 'en', - }); - expect(emojifiedElement.innerHTML).toBe( + const { searchEmojisByHexcodes } = mockDatabase(); + const actual = await emojifyElement( + testElement(), + testAppState({ mode: EMOJI_MODE_NATIVE }), + ); + assert(actual); + expect(actual.innerHTML).toBe( `

Hello 😊πŸ‡ͺπŸ‡Ί!

${expectedCustomEmojiImage}

`, ); + expect(searchEmojisByHexcodes).not.toHaveBeenCalled(); }); test('emojifies flag emoji in native-with-flags mode', async () => { - const emojifiedElement = await emojifyElement(cloneTestElement(), { - locales: ['en'], - mode: EMOJI_MODE_NATIVE_WITH_FLAGS, - currentLocale: 'en', - }); - expect(emojifiedElement.innerHTML).toBe( + const { searchEmojisByHexcodes } = mockDatabase(); + const actual = await emojifyElement( + testElement(), + testAppState({ mode: EMOJI_MODE_NATIVE_WITH_FLAGS }), + ); + assert(actual); + expect(actual.innerHTML).toBe( `

Hello 😊${expectedFlagImage}!

${expectedCustomEmojiImage}

`, ); + expect(searchEmojisByHexcodes).toHaveBeenCalledOnce(); }); test('emojifies everything in twemoji mode', async () => { - const emojifiedElement = await emojifyElement(cloneTestElement(), { - locales: ['en'], - mode: EMOJI_MODE_TWEMOJI, - currentLocale: 'en', - }); - expect(emojifiedElement.innerHTML).toBe( + const { searchCustomEmojisByShortcodes, searchEmojisByHexcodes } = + mockDatabase(); + const actual = await emojifyElement(testElement(), testAppState()); + assert(actual); + expect(actual.innerHTML).toBe( `

Hello ${expectedSmileImage}${expectedFlagImage}!

${expectedCustomEmojiImage}

`, ); + expect(searchEmojisByHexcodes).toHaveBeenCalledOnce(); + expect(searchCustomEmojisByShortcodes).toHaveBeenCalledOnce(); + }); + + test('emojifies with provided custom emoji', async () => { + const { searchCustomEmojisByShortcodes, searchEmojisByHexcodes } = + mockDatabase(); + const actual = await emojifyElement( + testElement('

hi :remote:

'), + testAppState(), + mockExtraCustom, + ); + assert(actual); + expect(actual.innerHTML).toBe( + `

hi ${expectedRemoteCustomEmojiImage}

`, + ); + expect(searchEmojisByHexcodes).not.toHaveBeenCalled(); + expect(searchCustomEmojisByShortcodes).not.toHaveBeenCalled(); + }); + + test('returns null when no emoji are found', async () => { + mockDatabase(); + const actual = await emojifyElement( + testElement('

here is just text :)

'), + testAppState(), + ); + expect(actual).toBeNull(); + }); +}); + +describe('emojifyText', () => { + test('returns original input when no emoji are in string', async () => { + const actual = await emojifyText('nothing here', testAppState()); + expect(actual).toBe('nothing here'); + }); + + test('renders Unicode emojis to twemojis', async () => { + mockDatabase(); + const actual = await emojifyText('Hello 😊πŸ‡ͺπŸ‡Ί!', testAppState()); + expect(actual).toBe(`Hello ${expectedSmileImage}${expectedFlagImage}!`); + }); + + test('renders custom emojis', async () => { + mockDatabase(); + const actual = await emojifyText('Hello :custom:!', testAppState()); + expect(actual).toBe(`Hello ${expectedCustomEmojiImage}!`); + }); + + test('renders provided extra emojis', async () => { + const actual = await emojifyText( + 'remote emoji :remote:', + testAppState(), + mockExtraCustom, + ); + expect(actual).toBe(`remote emoji ${expectedRemoteCustomEmojiImage}`); }); }); diff --git a/app/javascript/mastodon/features/emoji/render.ts b/app/javascript/mastodon/features/emoji/render.ts index 6ef9492147c..6486e65a709 100644 --- a/app/javascript/mastodon/features/emoji/render.ts +++ b/app/javascript/mastodon/features/emoji/render.ts @@ -1,8 +1,7 @@ -import type { Locale } from 'emojibase'; -import EMOJI_REGEX from 'emojibase-regex/emoji-loose'; - import { autoPlayGif } from '@/mastodon/initial_state'; +import { createLimitedCache } from '@/mastodon/utils/cache'; import { assetHost } from '@/mastodon/utils/config'; +import * as perf from '@/mastodon/utils/performance'; import { EMOJI_MODE_NATIVE, @@ -10,13 +9,12 @@ import { EMOJI_TYPE_UNICODE, EMOJI_TYPE_CUSTOM, EMOJI_STATE_MISSING, + ANY_EMOJI_REGEX, } from './constants'; import { - findMissingLocales, searchCustomEmojisByShortcodes, searchEmojisByHexcodes, } from './database'; -import { loadEmojiLocale } from './index'; import { emojiToUnicodeHex, twemojiHasBorder, @@ -34,18 +32,33 @@ import type { LocaleOrCustom, UnicodeEmojiToken, } from './types'; -import { stringHasUnicodeFlags } from './utils'; +import { emojiLogger, stringHasAnyEmoji, stringHasUnicodeFlags } from './utils'; -const localeCacheMap = new Map([ - [EMOJI_TYPE_CUSTOM, new Map()], -]); +const log = emojiLogger('render'); -// Emojifies an element. This modifies the element in place, replacing text nodes with emojified versions. +/** + * Emojifies an element. This modifies the element in place, replacing text nodes with emojified versions. + */ export async function emojifyElement( element: Element, appState: EmojiAppState, extraEmojis: ExtraCustomEmojiMap = {}, -): Promise { +): Promise { + const cacheKey = createCacheKey(element, appState, extraEmojis); + const cached = getCached(cacheKey); + if (cached !== undefined) { + log('Cache hit on %s', element.outerHTML); + if (cached === null) { + return null; + } + element.innerHTML = cached; + return element; + } + if (!stringHasAnyEmoji(element.innerHTML)) { + updateCache(cacheKey, null); + return null; + } + perf.start('emojifyElement()'); const queue: (HTMLElement | Text)[] = [element]; while (queue.length > 0) { const current = queue.shift(); @@ -61,7 +74,7 @@ export async function emojifyElement( current.textContent && (current instanceof Text || !current.hasChildNodes()) ) { - const renderedContent = await emojifyText( + const renderedContent = await textToElementArray( current.textContent, appState, extraEmojis, @@ -70,7 +83,7 @@ export async function emojifyElement( if (!(current instanceof Text)) { current.textContent = null; // Clear the text content if it's not a Text node. } - current.replaceWith(renderedToHTMLFragment(renderedContent)); + current.replaceWith(renderedToHTML(renderedContent)); } continue; } @@ -81,6 +94,8 @@ export async function emojifyElement( } } } + updateCache(cacheKey, element.innerHTML); + perf.stop('emojifyElement()'); return element; } @@ -88,7 +103,54 @@ export async function emojifyText( text: string, appState: EmojiAppState, extraEmojis: ExtraCustomEmojiMap = {}, +): Promise { + const cacheKey = createCacheKey(text, appState, extraEmojis); + const cached = getCached(cacheKey); + if (cached !== undefined) { + log('Cache hit on %s', text); + return cached ?? text; + } + if (!stringHasAnyEmoji(text)) { + updateCache(cacheKey, null); + return text; + } + const eleArray = await textToElementArray(text, appState, extraEmojis); + if (!eleArray) { + updateCache(cacheKey, null); + return text; + } + const rendered = renderedToHTML(eleArray, document.createElement('div')); + updateCache(cacheKey, rendered.innerHTML); + return rendered.innerHTML; +} + +// Private functions + +const { + set: updateCache, + get: getCached, + clear: cacheClear, +} = createLimitedCache({ log: log.extend('cache') }); + +function createCacheKey( + input: HTMLElement | string, + appState: EmojiAppState, + extraEmojis: ExtraCustomEmojiMap, ) { + return JSON.stringify([ + input instanceof HTMLElement ? input.outerHTML : input, + appState, + extraEmojis, + ]); +} + +type EmojifiedTextArray = (string | HTMLImageElement)[]; + +async function textToElementArray( + text: string, + appState: EmojiAppState, + extraEmojis: ExtraCustomEmojiMap = {}, +): Promise { // Exit if no text to convert. if (!text.trim()) { return null; @@ -102,10 +164,9 @@ export async function emojifyText( } // Get all emoji from the state map, loading any missing ones. - await ensureLocalesAreLoaded(appState.locales); - await loadMissingEmojiIntoCache(tokens, appState.locales); + await loadMissingEmojiIntoCache(tokens, appState, extraEmojis); - const renderedFragments: (string | HTMLImageElement)[] = []; + const renderedFragments: EmojifiedTextArray = []; for (const token of tokens) { if (typeof token !== 'string' && shouldRenderImage(token, appState.mode)) { let state: EmojiState | undefined; @@ -125,7 +186,7 @@ export async function emojifyText( // If the state is valid, create an image element. Otherwise, just append as text. if (state && typeof state !== 'string') { - const image = stateToImage(state); + const image = stateToImage(state, appState); renderedFragments.push(image); continue; } @@ -137,21 +198,6 @@ export async function emojifyText( return renderedFragments; } -// Private functions - -async function ensureLocalesAreLoaded(locales: Locale[]) { - const missingLocales = await findMissingLocales(locales); - for (const locale of missingLocales) { - await loadEmojiLocale(locale); - } -} - -const CUSTOM_EMOJI_REGEX = /:([a-z0-9_]+):/i; -const TOKENIZE_REGEX = new RegExp( - `(${EMOJI_REGEX.source}|${CUSTOM_EMOJI_REGEX.source})`, - 'g', -); - type TokenizedText = (string | EmojiToken)[]; export function tokenizeText(text: string): TokenizedText { @@ -161,7 +207,7 @@ export function tokenizeText(text: string): TokenizedText { const tokens = []; let lastIndex = 0; - for (const match of text.matchAll(TOKENIZE_REGEX)) { + for (const match of text.matchAll(ANY_EMOJI_REGEX)) { if (match.index > lastIndex) { tokens.push(text.slice(lastIndex, match.index)); } @@ -189,8 +235,18 @@ export function tokenizeText(text: string): TokenizedText { return tokens; } +const localeCacheMap = new Map([ + [ + EMOJI_TYPE_CUSTOM, + createLimitedCache({ log: log.extend('custom') }), + ], +]); + function cacheForLocale(locale: LocaleOrCustom): EmojiStateMap { - return localeCacheMap.get(locale) ?? (new Map() as EmojiStateMap); + return ( + localeCacheMap.get(locale) ?? + createLimitedCache({ log: log.extend(locale) }) + ); } function emojiForLocale( @@ -203,7 +259,8 @@ function emojiForLocale( async function loadMissingEmojiIntoCache( tokens: TokenizedText, - locales: Locale[], + { mode, currentLocale }: EmojiAppState, + extraEmojis: ExtraCustomEmojiMap, ) { const missingUnicodeEmoji = new Set(); const missingCustomEmoji = new Set(); @@ -217,42 +274,41 @@ async function loadMissingEmojiIntoCache( // If this is a custom emoji, check it separately. if (token.type === EMOJI_TYPE_CUSTOM) { const code = token.code; + if (code in extraEmojis) { + continue; // We don't care about extra emoji. + } const emojiState = emojiForLocale(code, EMOJI_TYPE_CUSTOM); if (!emojiState) { missingCustomEmoji.add(code); } // Otherwise this is a unicode emoji, so check it against all locales. - } else { + } else if (shouldRenderImage(token, mode)) { const code = emojiToUnicodeHex(token.code); if (missingUnicodeEmoji.has(code)) { continue; // Already marked as missing. } - for (const locale of locales) { - const emojiState = emojiForLocale(code, locale); - if (!emojiState) { - // If it's missing in one locale, we consider it missing for all. - missingUnicodeEmoji.add(code); - } + const emojiState = emojiForLocale(code, currentLocale); + if (!emojiState) { + // If it's missing in one locale, we consider it missing for all. + missingUnicodeEmoji.add(code); } } } if (missingUnicodeEmoji.size > 0) { const missingEmojis = Array.from(missingUnicodeEmoji).toSorted(); - for (const locale of locales) { - const emojis = await searchEmojisByHexcodes(missingEmojis, locale); - const cache = cacheForLocale(locale); - for (const emoji of emojis) { - cache.set(emoji.hexcode, { type: EMOJI_TYPE_UNICODE, data: emoji }); - } - const notFoundEmojis = missingEmojis.filter((code) => - emojis.every((emoji) => emoji.hexcode !== code), - ); - for (const code of notFoundEmojis) { - cache.set(code, EMOJI_STATE_MISSING); // Mark as missing if not found, as it's probably not a valid emoji. - } - localeCacheMap.set(locale, cache); + const emojis = await searchEmojisByHexcodes(missingEmojis, currentLocale); + const cache = cacheForLocale(currentLocale); + for (const emoji of emojis) { + cache.set(emoji.hexcode, { type: EMOJI_TYPE_UNICODE, data: emoji }); } + const notFoundEmojis = missingEmojis.filter((code) => + emojis.every((emoji) => emoji.hexcode !== code), + ); + for (const code of notFoundEmojis) { + cache.set(code, EMOJI_STATE_MISSING); // Mark as missing if not found, as it's probably not a valid emoji. + } + localeCacheMap.set(currentLocale, cache); } if (missingCustomEmoji.size > 0) { @@ -288,22 +344,24 @@ function shouldRenderImage(token: EmojiToken, mode: EmojiMode): boolean { return true; } -function stateToImage(state: EmojiLoadedState) { +function stateToImage(state: EmojiLoadedState, appState: EmojiAppState) { const image = document.createElement('img'); image.draggable = false; image.classList.add('emojione'); if (state.type === EMOJI_TYPE_UNICODE) { const emojiInfo = twemojiHasBorder(unicodeToTwemojiHex(state.data.hexcode)); - if (emojiInfo.hasLightBorder) { - image.dataset.lightCode = `${emojiInfo.hexCode}_BORDER`; - } else if (emojiInfo.hasDarkBorder) { - image.dataset.darkCode = `${emojiInfo.hexCode}_BORDER`; + let fileName = emojiInfo.hexCode; + if ( + (appState.darkTheme && emojiInfo.hasDarkBorder) || + (!appState.darkTheme && emojiInfo.hasLightBorder) + ) { + fileName = `${emojiInfo.hexCode}_border`; } image.alt = state.data.unicode; image.title = state.data.label; - image.src = `${assetHost}/emoji/${emojiInfo.hexCode}.svg`; + image.src = `${assetHost}/emoji/${fileName}.svg`; } else { // Custom emoji const shortCode = `:${state.data.shortcode}:`; @@ -318,8 +376,16 @@ function stateToImage(state: EmojiLoadedState) { return image; } -function renderedToHTMLFragment(renderedArray: (string | HTMLImageElement)[]) { - const fragment = document.createDocumentFragment(); +function renderedToHTML(renderedArray: EmojifiedTextArray): DocumentFragment; +function renderedToHTML( + renderedArray: EmojifiedTextArray, + parent: ParentType, +): ParentType; +function renderedToHTML( + renderedArray: EmojifiedTextArray, + parent: ParentNode | null = null, +) { + const fragment = parent ?? document.createDocumentFragment(); for (const fragmentItem of renderedArray) { if (typeof fragmentItem === 'string') { fragment.appendChild(document.createTextNode(fragmentItem)); @@ -329,3 +395,9 @@ function renderedToHTMLFragment(renderedArray: (string | HTMLImageElement)[]) { } return fragment; } + +// Testing helpers +export const testCacheClear = () => { + cacheClear(); + localeCacheMap.clear(); +}; diff --git a/app/javascript/mastodon/features/emoji/types.ts b/app/javascript/mastodon/features/emoji/types.ts index f5932ed97fd..85bbe6d1a56 100644 --- a/app/javascript/mastodon/features/emoji/types.ts +++ b/app/javascript/mastodon/features/emoji/types.ts @@ -1,6 +1,10 @@ +import type { List as ImmutableList } from 'immutable'; + import type { FlatCompactEmoji, Locale } from 'emojibase'; import type { ApiCustomEmojiJSON } from '@/mastodon/api_types/custom_emoji'; +import type { CustomEmoji } from '@/mastodon/models/custom_emoji'; +import type { LimitedCache } from '@/mastodon/utils/cache'; import type { EMOJI_MODE_NATIVE, @@ -22,6 +26,7 @@ export interface EmojiAppState { locales: Locale[]; currentLocale: Locale; mode: EmojiMode; + darkTheme: boolean; } export interface UnicodeEmojiToken { @@ -45,7 +50,7 @@ export interface EmojiStateUnicode { } export interface EmojiStateCustom { type: typeof EMOJI_TYPE_CUSTOM; - data: CustomEmojiData; + data: CustomEmojiRenderFields; } export type EmojiState = | EmojiStateMissing @@ -53,9 +58,16 @@ export type EmojiState = | EmojiStateCustom; export type EmojiLoadedState = EmojiStateUnicode | EmojiStateCustom; -export type EmojiStateMap = Map; +export type EmojiStateMap = LimitedCache; -export type ExtraCustomEmojiMap = Record; +export type CustomEmojiMapArg = + | ExtraCustomEmojiMap + | ImmutableList; +export type CustomEmojiRenderFields = Pick< + CustomEmojiData, + 'shortcode' | 'static_url' | 'url' +>; +export type ExtraCustomEmojiMap = Record; export interface TwemojiBorderInfo { hexCode: string; diff --git a/app/javascript/mastodon/features/emoji/utils.test.ts b/app/javascript/mastodon/features/emoji/utils.test.ts index 75cac8c5b4c..b9062294c47 100644 --- a/app/javascript/mastodon/features/emoji/utils.test.ts +++ b/app/javascript/mastodon/features/emoji/utils.test.ts @@ -1,8 +1,14 @@ -import { stringHasUnicodeEmoji, stringHasUnicodeFlags } from './utils'; +import { + stringHasAnyEmoji, + stringHasCustomEmoji, + stringHasUnicodeEmoji, + stringHasUnicodeFlags, +} from './utils'; -describe('stringHasEmoji', () => { +describe('stringHasUnicodeEmoji', () => { test.concurrent.for([ ['only text', false], + ['text with non-emoji symbols β„’Β©', false], ['text with emoji πŸ˜€', true], ['multiple emojis πŸ˜€πŸ˜ƒπŸ˜„', true], ['emoji with skin tone πŸ‘πŸ½', true], @@ -19,14 +25,14 @@ describe('stringHasEmoji', () => { ['emoji with enclosing keycap #️⃣', true], ['emoji with no visible glyph \u200D', false], ] as const)( - 'stringHasEmoji has emojis in "%s": %o', + 'stringHasUnicodeEmoji has emojis in "%s": %o', ([text, expected], { expect }) => { expect(stringHasUnicodeEmoji(text)).toBe(expected); }, ); }); -describe('stringHasFlags', () => { +describe('stringHasUnicodeFlags', () => { test.concurrent.for([ ['EU πŸ‡ͺπŸ‡Ί', true], ['Germany πŸ‡©πŸ‡ͺ', true], @@ -45,3 +51,27 @@ describe('stringHasFlags', () => { }, ); }); + +describe('stringHasCustomEmoji', () => { + test('string with custom emoji returns true', () => { + expect(stringHasCustomEmoji(':custom: :test:')).toBeTruthy(); + }); + test('string without custom emoji returns false', () => { + expect(stringHasCustomEmoji('πŸ³οΈβ€πŸŒˆ :πŸ³οΈβ€πŸŒˆ: text β„’')).toBeFalsy(); + }); +}); + +describe('stringHasAnyEmoji', () => { + test('string without any emoji or characters', () => { + expect(stringHasAnyEmoji('normal text. 12356?!')).toBeFalsy(); + }); + test('string with non-emoji characters', () => { + expect(stringHasAnyEmoji('β„’Β©')).toBeFalsy(); + }); + test('has unicode emoji', () => { + expect(stringHasAnyEmoji('πŸ³οΈβ€πŸŒˆπŸ”₯πŸ‡ΈπŸ‡Ή πŸ‘©β€πŸ”¬')).toBeTruthy(); + }); + test('has custom emoji', () => { + expect(stringHasAnyEmoji(':test: :custom:')).toBeTruthy(); + }); +}); diff --git a/app/javascript/mastodon/features/emoji/utils.ts b/app/javascript/mastodon/features/emoji/utils.ts index d00accea8c5..89f8d926466 100644 --- a/app/javascript/mastodon/features/emoji/utils.ts +++ b/app/javascript/mastodon/features/emoji/utils.ts @@ -1,13 +1,27 @@ -import EMOJI_REGEX from 'emojibase-regex/emoji-loose'; +import debug from 'debug'; -export function stringHasUnicodeEmoji(text: string): boolean { - return EMOJI_REGEX.test(text); +import { + CUSTOM_EMOJI_REGEX, + UNICODE_EMOJI_REGEX, + UNICODE_FLAG_EMOJI_REGEX, +} from './constants'; + +export function emojiLogger(segment: string) { + return debug(`emojis:${segment}`); } -// From https://github.com/talkjs/country-flag-emoji-polyfill/blob/master/src/index.ts#L49-L50 -const EMOJIS_FLAGS_REGEX = - /[\u{1F1E6}-\u{1F1FF}|\u{E0062}-\u{E0063}|\u{E0065}|\u{E0067}|\u{E006C}|\u{E006E}|\u{E0073}-\u{E0074}|\u{E0077}|\u{E007F}]+/u; - -export function stringHasUnicodeFlags(text: string): boolean { - return EMOJIS_FLAGS_REGEX.test(text); +export function stringHasUnicodeEmoji(input: string): boolean { + return UNICODE_EMOJI_REGEX.test(input); +} + +export function stringHasUnicodeFlags(input: string): boolean { + return UNICODE_FLAG_EMOJI_REGEX.test(input); +} + +export function stringHasCustomEmoji(input: string) { + return CUSTOM_EMOJI_REGEX.test(input); +} + +export function stringHasAnyEmoji(input: string) { + return stringHasUnicodeEmoji(input) || stringHasCustomEmoji(input); } diff --git a/app/javascript/mastodon/features/emoji/worker.ts b/app/javascript/mastodon/features/emoji/worker.ts index 1c48a077730..6fb7d36e936 100644 --- a/app/javascript/mastodon/features/emoji/worker.ts +++ b/app/javascript/mastodon/features/emoji/worker.ts @@ -5,9 +5,14 @@ self.postMessage('ready'); // After the worker is ready, notify the main thread function handleMessage(event: MessageEvent) { const { data: locale } = event; - if (locale !== 'custom') { - void importEmojiData(locale); - } else { - void importCustomEmojiData(); - } + void loadData(locale); +} + +async function loadData(locale: string) { + if (locale !== 'custom') { + await importEmojiData(locale); + } else { + await importCustomEmojiData(); + } + self.postMessage(`loaded ${locale}`); } diff --git a/app/javascript/mastodon/main.tsx b/app/javascript/mastodon/main.tsx index dcc71bdb843..456cc21c318 100644 --- a/app/javascript/mastodon/main.tsx +++ b/app/javascript/mastodon/main.tsx @@ -2,10 +2,10 @@ import { createRoot } from 'react-dom/client'; import { Globals } from '@react-spring/web'; +import * as perf from '@/mastodon/utils/performance'; import { setupBrowserNotifications } from 'mastodon/actions/notifications'; import Mastodon from 'mastodon/containers/mastodon'; import { me, reduceMotion } from 'mastodon/initial_state'; -import * as perf from 'mastodon/performance'; import ready from 'mastodon/ready'; import { store } from 'mastodon/store'; @@ -35,7 +35,7 @@ function main() { if (isModernEmojiEnabled()) { const { initializeEmoji } = await import('@/mastodon/features/emoji'); - await initializeEmoji(); + initializeEmoji(); } const root = createRoot(mountNode); diff --git a/app/javascript/mastodon/utils/__tests__/cache.test.ts b/app/javascript/mastodon/utils/__tests__/cache.test.ts new file mode 100644 index 00000000000..340a51fdb4b --- /dev/null +++ b/app/javascript/mastodon/utils/__tests__/cache.test.ts @@ -0,0 +1,78 @@ +import { createLimitedCache } from '../cache'; + +describe('createCache', () => { + test('returns expected methods', () => { + const actual = createLimitedCache(); + expect(actual).toBeTypeOf('object'); + expect(actual).toHaveProperty('get'); + expect(actual).toHaveProperty('has'); + expect(actual).toHaveProperty('delete'); + expect(actual).toHaveProperty('set'); + }); + + test('caches values provided to it', () => { + const cache = createLimitedCache(); + cache.set('test', 'result'); + expect(cache.get('test')).toBe('result'); + }); + + test('has returns expected values', () => { + const cache = createLimitedCache(); + cache.set('test', 'result'); + expect(cache.has('test')).toBeTruthy(); + expect(cache.has('not found')).toBeFalsy(); + }); + + test('updates a value if keys are the same', () => { + const cache = createLimitedCache(); + cache.set('test1', 1); + cache.set('test1', 2); + expect(cache.get('test1')).toBe(2); + }); + + test('delete removes an item', () => { + const cache = createLimitedCache(); + cache.set('test', 'result'); + expect(cache.has('test')).toBeTruthy(); + cache.delete('test'); + expect(cache.has('test')).toBeFalsy(); + expect(cache.get('test')).toBeUndefined(); + }); + + test('removes oldest item cached if it exceeds a set size', () => { + const cache = createLimitedCache({ maxSize: 1 }); + cache.set('test1', 1); + cache.set('test2', 2); + expect(cache.get('test1')).toBeUndefined(); + expect(cache.get('test2')).toBe(2); + }); + + test('retrieving a value bumps up last access', () => { + const cache = createLimitedCache({ maxSize: 2 }); + cache.set('test1', 1); + cache.set('test2', 2); + expect(cache.get('test1')).toBe(1); + cache.set('test3', 3); + expect(cache.get('test1')).toBe(1); + expect(cache.get('test2')).toBeUndefined(); + expect(cache.get('test3')).toBe(3); + }); + + test('logs when cache is added to and removed', () => { + const log = vi.fn(); + const cache = createLimitedCache({ maxSize: 1, log }); + cache.set('test1', 1); + expect(log).toHaveBeenLastCalledWith( + 'Added %s to cache, now size %d', + 'test1', + 1, + ); + cache.set('test2', 1); + expect(log).toHaveBeenLastCalledWith( + 'Added %s and deleted %s from cache, now size %d', + 'test2', + 'test1', + 1, + ); + }); +}); diff --git a/app/javascript/mastodon/utils/cache.ts b/app/javascript/mastodon/utils/cache.ts new file mode 100644 index 00000000000..2e3d21bfed4 --- /dev/null +++ b/app/javascript/mastodon/utils/cache.ts @@ -0,0 +1,60 @@ +export interface LimitedCache { + has: (key: CacheKey) => boolean; + get: (key: CacheKey) => CacheValue | undefined; + delete: (key: CacheKey) => void; + set: (key: CacheKey, value: CacheValue) => void; + clear: () => void; +} + +interface LimitedCacheArguments { + maxSize?: number; + log?: (...args: unknown[]) => void; +} + +export function createLimitedCache({ + maxSize = 100, + log = () => null, +}: LimitedCacheArguments = {}): LimitedCache { + const cacheMap = new Map(); + const cacheKeys = new Set(); + + function touchKey(key: CacheKey) { + if (cacheKeys.has(key)) { + cacheKeys.delete(key); + } + cacheKeys.add(key); + } + + return { + has: (key) => cacheMap.has(key), + get: (key) => { + if (cacheMap.has(key)) { + touchKey(key); + } + return cacheMap.get(key); + }, + delete: (key) => cacheMap.delete(key) && cacheKeys.delete(key), + set: (key, value) => { + cacheMap.set(key, value); + touchKey(key); + + const lastKey = cacheKeys.values().toArray().shift(); + if (cacheMap.size > maxSize && lastKey) { + cacheMap.delete(lastKey); + cacheKeys.delete(lastKey); + log( + 'Added %s and deleted %s from cache, now size %d', + key, + lastKey, + cacheMap.size, + ); + } else { + log('Added %s to cache, now size %d', key, cacheMap.size); + } + }, + clear: () => { + cacheMap.clear(); + cacheKeys.clear(); + }, + }; +} diff --git a/app/javascript/mastodon/performance.js b/app/javascript/mastodon/utils/performance.ts similarity index 70% rename from app/javascript/mastodon/performance.js rename to app/javascript/mastodon/utils/performance.ts index 1b2092cfc4c..e503e1ef587 100644 --- a/app/javascript/mastodon/performance.js +++ b/app/javascript/mastodon/utils/performance.ts @@ -4,15 +4,15 @@ import * as marky from 'marky'; -import { isDevelopment } from './utils/environment'; +import { isDevelopment } from './environment'; -export function start(name) { +export function start(name: string) { if (isDevelopment()) { marky.mark(name); } } -export function stop(name) { +export function stop(name: string) { if (isDevelopment()) { marky.stop(name); } diff --git a/app/javascript/testing/factories.ts b/app/javascript/testing/factories.ts index 5b2fbfe594e..cd5f72a06f0 100644 --- a/app/javascript/testing/factories.ts +++ b/app/javascript/testing/factories.ts @@ -1,4 +1,8 @@ import type { ApiRelationshipJSON } from '@/mastodon/api_types/relationships'; +import type { + CustomEmojiData, + UnicodeEmojiData, +} from '@/mastodon/features/emoji/types'; import { createAccountFromServerJSON } from '@/mastodon/models/account'; import type { ApiAccountJSON } from 'mastodon/api_types/accounts'; @@ -68,3 +72,26 @@ export const relationshipsFactory: FactoryFunction = ({ showing_reblogs: true, ...data, }); + +export function unicodeEmojiFactory( + data: Partial = {}, +): UnicodeEmojiData { + return { + hexcode: 'test', + label: 'Test', + unicode: 'πŸ§ͺ', + ...data, + }; +} + +export function customEmojiFactory( + data: Partial = {}, +): CustomEmojiData { + return { + shortcode: 'custom', + static_url: 'emoji/custom/static', + url: 'emoji/custom', + visible_in_picker: true, + ...data, + }; +} diff --git a/eslint.config.mjs b/eslint.config.mjs index 3d00a4adce9..43aabc51100 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -406,6 +406,12 @@ export default tseslint.config([ globals: globals.vitest, }, }, + { + files: ['**/*.test.*'], + rules: { + 'no-global-assign': 'off', + }, + }, { files: ['**/*.stories.ts', '**/*.stories.tsx', '.storybook/*'], rules: { diff --git a/package.json b/package.json index efe5b45fefb..736f29fb814 100644 --- a/package.json +++ b/package.json @@ -64,11 +64,11 @@ "color-blend": "^4.0.0", "core-js": "^3.30.2", "cross-env": "^10.0.0", + "debug": "^4.4.1", "detect-passive-events": "^2.0.3", "emoji-mart": "npm:emoji-mart-lazyload@latest", "emojibase": "^16.0.0", "emojibase-data": "^16.0.3", - "emojibase-regex": "^16.0.0", "escape-html": "^1.0.3", "fast-glob": "^3.3.3", "fuzzysort": "^3.0.0", @@ -137,6 +137,7 @@ "@storybook/react-vite": "^9.0.4", "@testing-library/dom": "^10.2.0", "@testing-library/react": "^16.0.0", + "@types/debug": "^4", "@types/emoji-mart": "3.0.14", "@types/escape-html": "^1.0.2", "@types/hoist-non-react-statics": "^3.3.1", @@ -174,6 +175,7 @@ "eslint-plugin-react": "^7.37.4", "eslint-plugin-react-hooks": "^5.2.0", "eslint-plugin-storybook": "^9.0.4", + "fake-indexeddb": "^6.0.1", "globals": "^16.0.0", "husky": "^9.0.11", "lint-staged": "^16.0.0", diff --git a/vitest.config.mts b/vitest.config.mts index 7df462ed6db..b129c293f4c 100644 --- a/vitest.config.mts +++ b/vitest.config.mts @@ -49,6 +49,7 @@ const legacyTests: TestProjectInlineConfiguration = { 'tmp/**', ], globals: true, + setupFiles: ['fake-indexeddb/auto'], }, }; diff --git a/yarn.lock b/yarn.lock index 9cb29176d8b..377f2999c85 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2632,6 +2632,7 @@ __metadata: "@storybook/react-vite": "npm:^9.0.4" "@testing-library/dom": "npm:^10.2.0" "@testing-library/react": "npm:^16.0.0" + "@types/debug": "npm:^4" "@types/emoji-mart": "npm:3.0.14" "@types/escape-html": "npm:^1.0.2" "@types/hoist-non-react-statics": "npm:^3.3.1" @@ -2673,11 +2674,11 @@ __metadata: color-blend: "npm:^4.0.0" core-js: "npm:^3.30.2" cross-env: "npm:^10.0.0" + debug: "npm:^4.4.1" detect-passive-events: "npm:^2.0.3" emoji-mart: "npm:emoji-mart-lazyload@latest" emojibase: "npm:^16.0.0" emojibase-data: "npm:^16.0.3" - emojibase-regex: "npm:^16.0.0" escape-html: "npm:^1.0.3" eslint: "npm:^9.23.0" eslint-import-resolver-typescript: "npm:^4.2.5" @@ -2689,6 +2690,7 @@ __metadata: eslint-plugin-react: "npm:^7.37.4" eslint-plugin-react-hooks: "npm:^5.2.0" eslint-plugin-storybook: "npm:^9.0.4" + fake-indexeddb: "npm:^6.0.1" fast-glob: "npm:^3.3.3" fuzzysort: "npm:^3.0.0" globals: "npm:^16.0.0" @@ -3931,6 +3933,15 @@ __metadata: languageName: node linkType: hard +"@types/debug@npm:^4": + version: 4.1.12 + resolution: "@types/debug@npm:4.1.12" + dependencies: + "@types/ms": "npm:*" + checksum: 10c0/5dcd465edbb5a7f226e9a5efd1f399c6172407ef5840686b73e3608ce135eeca54ae8037dcd9f16bdb2768ac74925b820a8b9ecc588a58ca09eca6acabe33e2f + languageName: node + linkType: hard + "@types/deep-eql@npm:*": version: 4.0.2 resolution: "@types/deep-eql@npm:4.0.2" @@ -4112,6 +4123,13 @@ __metadata: languageName: node linkType: hard +"@types/ms@npm:*": + version: 2.1.0 + resolution: "@types/ms@npm:2.1.0" + checksum: 10c0/5ce692ffe1549e1b827d99ef8ff71187457e0eb44adbae38fdf7b9a74bae8d20642ee963c14516db1d35fa2652e65f47680fdf679dcbde52bbfadd021f497225 + languageName: node + linkType: hard + "@types/node@npm:*, @types/node@npm:^22.0.0": version: 22.13.14 resolution: "@types/node@npm:22.13.14" @@ -6599,13 +6617,6 @@ __metadata: languageName: node linkType: hard -"emojibase-regex@npm:^16.0.0": - version: 16.0.0 - resolution: "emojibase-regex@npm:16.0.0" - checksum: 10c0/8ee5ff798e51caa581434b1cb2f9737e50195093c4efa1739df21a50a5496f80517924787d865e8cf7d6a0b4c90dbedc04bdc506dcbcc582e14cdf0bb47af0f0 - languageName: node - linkType: hard - "emojibase@npm:^16.0.0": version: 16.0.0 resolution: "emojibase@npm:16.0.0" @@ -7370,6 +7381,13 @@ __metadata: languageName: node linkType: hard +"fake-indexeddb@npm:^6.0.1": + version: 6.0.1 + resolution: "fake-indexeddb@npm:6.0.1" + checksum: 10c0/60f4ccdfd5ecb37bb98019056c688366847840cce7146e0005c5ca54823238455403b0a8803b898a11cf80f6147b1bb553457c6af427a644a6e64566cdbe42ec + languageName: node + linkType: hard + "fast-copy@npm:^3.0.2": version: 3.0.2 resolution: "fast-copy@npm:3.0.2"