Emoji Rendering Efficiency (#35568)

app/javascript/mastodon/features/emoji/constants.ts

@@ -15,6 +15,17 @@ export const SKIN_TONE_CODES = [
   0x1f3ff, // Dark skin tone
 ] as const;
 
+// TODO: Test and create fallback for browsers that do not handle the /v flag.
+export const UNICODE_EMOJI_REGEX = /\p{RGI_Emoji}/v;
+// See: https://www.unicode.org/reports/tr51/#valid-emoji-tag-sequences
+export const UNICODE_FLAG_EMOJI_REGEX =
+  /\p{RGI_Emoji_Flag_Sequence}|\p{RGI_Emoji_Tag_Sequence}/v;
+export const CUSTOM_EMOJI_REGEX = /:([a-z0-9_]+):/i;
+export const ANY_EMOJI_REGEX = new RegExp(
+  `(${UNICODE_EMOJI_REGEX.source}|${CUSTOM_EMOJI_REGEX.source})`,
+  'gv',
+);
+
 // Emoji rendering modes. A mode is what we are using to render emojis, a style is what the user has selected.
 export const EMOJI_MODE_NATIVE = 'native';
 export const EMOJI_MODE_NATIVE_WITH_FLAGS = 'native-flags';

app/javascript/mastodon/features/emoji/database.test.ts

@@ -0,0 +1,139 @@
+import { IDBFactory } from 'fake-indexeddb';
+
+import { unicodeEmojiFactory } from '@/testing/factories';
+
+import {
+  putEmojiData,
+  loadEmojiByHexcode,
+  searchEmojisByHexcodes,
+  searchEmojisByTag,
+  testClear,
+  testGet,
+} from './database';
+
+describe('emoji database', () => {
+  afterEach(() => {
+    testClear();
+    indexedDB = new IDBFactory();
+  });
+  describe('putEmojiData', () => {
+    test('adds to loaded locales', async () => {
+      const { loadedLocales } = await testGet();
+      expect(loadedLocales).toHaveLength(0);
+      await putEmojiData([], 'en');
+      expect(loadedLocales).toContain('en');
+    });
+
+    test('loads emoji into indexedDB', async () => {
+      await putEmojiData([unicodeEmojiFactory()], 'en');
+      const { db } = await testGet();
+      await expect(db.get('en', 'test')).resolves.toEqual(
+        unicodeEmojiFactory(),
+      );
+    });
+  });
+
+  describe('loadEmojiByHexcode', () => {
+    test('throws if the locale is not loaded', async () => {
+      await expect(loadEmojiByHexcode('en', 'test')).rejects.toThrowError(
+        'Locale en',
+      );
+    });
+
+    test('retrieves the emoji', async () => {
+      await putEmojiData([unicodeEmojiFactory()], 'en');
+      await expect(loadEmojiByHexcode('test', 'en')).resolves.toEqual(
+        unicodeEmojiFactory(),
+      );
+    });
+
+    test('returns undefined if not found', async () => {
+      await putEmojiData([], 'en');
+      await expect(loadEmojiByHexcode('test', 'en')).resolves.toBeUndefined();
+    });
+  });
+
+  describe('searchEmojisByHexcodes', () => {
+    const data = [
+      unicodeEmojiFactory({ hexcode: 'not a number' }),
+      unicodeEmojiFactory({ hexcode: '1' }),
+      unicodeEmojiFactory({ hexcode: '2' }),
+      unicodeEmojiFactory({ hexcode: '3' }),
+      unicodeEmojiFactory({ hexcode: 'another not a number' }),
+    ];
+    beforeEach(async () => {
+      await putEmojiData(data, 'en');
+    });
+    test('finds emoji in consecutive range', async () => {
+      const actual = await searchEmojisByHexcodes(['1', '2', '3'], 'en');
+      expect(actual).toHaveLength(3);
+    });
+
+    test('finds emoji in split range', async () => {
+      const actual = await searchEmojisByHexcodes(['1', '3'], 'en');
+      expect(actual).toHaveLength(2);
+      expect(actual).toContainEqual(data.at(1));
+      expect(actual).toContainEqual(data.at(3));
+    });
+
+    test('finds emoji with non-numeric range', async () => {
+      const actual = await searchEmojisByHexcodes(
+        ['3', 'not a number', '1'],
+        'en',
+      );
+      expect(actual).toHaveLength(3);
+      expect(actual).toContainEqual(data.at(0));
+      expect(actual).toContainEqual(data.at(1));
+      expect(actual).toContainEqual(data.at(3));
+    });
+
+    test('not found emoji are not returned', async () => {
+      const actual = await searchEmojisByHexcodes(['not found'], 'en');
+      expect(actual).toHaveLength(0);
+    });
+
+    test('only found emojis are returned', async () => {
+      const actual = await searchEmojisByHexcodes(
+        ['another not a number', 'not found'],
+        'en',
+      );
+      expect(actual).toHaveLength(1);
+      expect(actual).toContainEqual(data.at(4));
+    });
+  });
+
+  describe('searchEmojisByTag', () => {
+    const data = [
+      unicodeEmojiFactory({ hexcode: 'test1', tags: ['test 1'] }),
+      unicodeEmojiFactory({
+        hexcode: 'test2',
+        tags: ['test 2', 'something else'],
+      }),
+      unicodeEmojiFactory({ hexcode: 'test3', tags: ['completely different'] }),
+    ];
+    beforeEach(async () => {
+      await putEmojiData(data, 'en');
+    });
+    test('finds emojis with tag', async () => {
+      const actual = await searchEmojisByTag('test 1', 'en');
+      expect(actual).toHaveLength(1);
+      expect(actual).toContainEqual(data.at(0));
+    });
+
+    test('finds emojis starting with tag', async () => {
+      const actual = await searchEmojisByTag('test', 'en');
+      expect(actual).toHaveLength(2);
+      expect(actual).not.toContainEqual(data.at(2));
+    });
+
+    test('does not find emojis ending with tag', async () => {
+      const actual = await searchEmojisByTag('else', 'en');
+      expect(actual).toHaveLength(0);
+    });
+
+    test('finds nothing with invalid tag', async () => {
+      const actual = await searchEmojisByTag('not found', 'en');
+      expect(actual).toHaveLength(0);
+    });
+  });
+});

app/javascript/mastodon/features/emoji/database.ts

@@ -9,6 +9,7 @@ import type {
   UnicodeEmojiData,
   LocaleOrCustom,
 } from './types';
+import { emojiLogger } from './utils';
 
 interface EmojiDB extends LocaleTables, DBSchema {
   custom: {
@@ -36,40 +37,63 @@ interface LocaleTable {
 }
 type LocaleTables = Record<Locale, LocaleTable>;
 
+type Database = IDBPDatabase<EmojiDB>;
+
 const SCHEMA_VERSION = 1;
 
-let db: IDBPDatabase<EmojiDB> | null = null;
+const loadedLocales = new Set<Locale>();
 
-async function loadDB() {
-  if (db) {
-    return db;
-  }
-  db = await openDB<EmojiDB>('mastodon-emoji', SCHEMA_VERSION, {
-    upgrade(database) {
-      const customTable = database.createObjectStore('custom', {
-        keyPath: 'shortcode',
-        autoIncrement: false,
-      });
-      customTable.createIndex('category', 'category');
+const log = emojiLogger('database');
 
-      database.createObjectStore('etags');
+// Loads the database in a way that ensures it's only loaded once.
+const loadDB = (() => {
+  let dbPromise: Promise<Database> | null = null;
 
-      for (const locale of SUPPORTED_LOCALES) {
-        const localeTable = database.createObjectStore(locale, {
-          keyPath: 'hexcode',
+  // Actually load the DB.
+  async function initDB() {
+    const db = await openDB<EmojiDB>('mastodon-emoji', SCHEMA_VERSION, {
+      upgrade(database) {
+        const customTable = database.createObjectStore('custom', {
+          keyPath: 'shortcode',
           autoIncrement: false,
         });
-        localeTable.createIndex('group', 'group');
-        localeTable.createIndex('label', 'label');
-        localeTable.createIndex('order', 'order');
-        localeTable.createIndex('tags', 'tags', { multiEntry: true });
-      }
-    },
-  });
-  return db;
-}
+        customTable.createIndex('category', 'category');
+
+        database.createObjectStore('etags');
+
+        for (const locale of SUPPORTED_LOCALES) {
+          const localeTable = database.createObjectStore(locale, {
+            keyPath: 'hexcode',
+            autoIncrement: false,
+          });
+          localeTable.createIndex('group', 'group');
+          localeTable.createIndex('label', 'label');
+          localeTable.createIndex('order', 'order');
+          localeTable.createIndex('tags', 'tags', { multiEntry: true });
+        }
+      },
+    });
+    await syncLocales(db);
+    return db;
+  }
+
+  // Loads the database, or returns the existing promise if it hasn't resolved yet.
+  const loadPromise = async (): Promise<Database> => {
+    if (dbPromise) {
+      return dbPromise;
+    }
+    dbPromise = initDB();
+    return dbPromise;
+  };
+  // Special way to reset the database, used for unit testing.
+  loadPromise.reset = () => {
+    dbPromise = null;
+  };
+  return loadPromise;
+})();
 
 export async function putEmojiData(emojis: UnicodeEmojiData[], locale: Locale) {
+  loadedLocales.add(locale);
   const db = await loadDB();
   const trx = db.transaction(locale, 'readwrite');
   await Promise.all(emojis.map((emoji) => trx.store.put(emoji)));
@@ -86,15 +110,15 @@ export async function putCustomEmojiData(emojis: CustomEmojiData[]) {
 export async function putLatestEtag(etag: string, localeString: string) {
   const locale = toSupportedLocaleOrCustom(localeString);
   const db = await loadDB();
-  return db.put('etags', etag, locale);
+  await db.put('etags', etag, locale);
 }
 
-export async function searchEmojiByHexcode(
+export async function loadEmojiByHexcode(
   hexcode: string,
   localeString: string,
 ) {
-  const locale = toSupportedLocale(localeString);
   const db = await loadDB();
+  const locale = toLoadedLocale(localeString);
   return db.get(locale, hexcode);
 }
 
@@ -102,45 +126,39 @@ export async function searchEmojisByHexcodes(
   hexcodes: string[],
   localeString: string,
 ) {
-  const locale = toSupportedLocale(localeString);
   const db = await loadDB();
-  return db.getAll(
+  const locale = toLoadedLocale(localeString);
+  const sortedCodes = hexcodes.toSorted();
+  const results = await db.getAll(
     locale,
-    IDBKeyRange.bound(hexcodes[0], hexcodes[hexcodes.length - 1]),
+    IDBKeyRange.bound(sortedCodes.at(0), sortedCodes.at(-1)),
   );
+  return results.filter((emoji) => hexcodes.includes(emoji.hexcode));
 }
 
-export async function searchEmojiByTag(tag: string, localeString: string) {
-  const locale = toSupportedLocale(localeString);
-  const range = IDBKeyRange.only(tag.toLowerCase());
+export async function searchEmojisByTag(tag: string, localeString: string) {
   const db = await loadDB();
+  const locale = toLoadedLocale(localeString);
+  const range = IDBKeyRange.bound(
+    tag.toLowerCase(),
+    `${tag.toLowerCase()}\uffff`,
+  );
   return db.getAllFromIndex(locale, 'tags', range);
 }
 
-export async function searchCustomEmojiByShortcode(shortcode: string) {
+export async function loadCustomEmojiByShortcode(shortcode: string) {
   const db = await loadDB();
   return db.get('custom', shortcode);
 }
 
 export async function searchCustomEmojisByShortcodes(shortcodes: string[]) {
   const db = await loadDB();
-  return db.getAll(
+  const sortedCodes = shortcodes.toSorted();
+  const results = await db.getAll(
     'custom',
-    IDBKeyRange.bound(shortcodes[0], shortcodes[shortcodes.length - 1]),
+    IDBKeyRange.bound(sortedCodes.at(0), sortedCodes.at(-1)),
   );
-}
-
-export async function findMissingLocales(localeStrings: string[]) {
-  const locales = new Set(localeStrings.map(toSupportedLocale));
-  const missingLocales: Locale[] = [];
-  const db = await loadDB();
-  for (const locale of locales) {
-    const rowCount = await db.count(locale);
-    if (!rowCount) {
-      missingLocales.push(locale);
-    }
-  }
-  return missingLocales;
+  return results.filter((emoji) => shortcodes.includes(emoji.shortcode));
 }
 
 export async function loadLatestEtag(localeString: string) {
@@ -153,3 +171,51 @@ export async function loadLatestEtag(localeString: string) {
   const etag = await db.get('etags', locale);
   return etag ?? null;
 }
+
+// Private functions
+
+async function syncLocales(db: Database) {
+  const locales = await Promise.all(
+    SUPPORTED_LOCALES.map(
+      async (locale) =>
+        [locale, await hasLocale(locale, db)] satisfies [Locale, boolean],
+    ),
+  );
+  for (const [locale, loaded] of locales) {
+    if (loaded) {
+      loadedLocales.add(locale);
+    } else {
+      loadedLocales.delete(locale);
+    }
+  }
+  log('Loaded %d locales: %o', loadedLocales.size, loadedLocales);
+}
+
+function toLoadedLocale(localeString: string) {
+  const locale = toSupportedLocale(localeString);
+  if (localeString !== locale) {
+    log(`Locale ${locale} is different from provided ${localeString}`);
+  }
+  if (!loadedLocales.has(locale)) {
+    throw new Error(`Locale ${locale} is not loaded in emoji database`);
+  }
+  return locale;
+}
+
+async function hasLocale(locale: Locale, db: Database): Promise<boolean> {
+  if (loadedLocales.has(locale)) {
+    return true;
+  }
+  const rowCount = await db.count(locale);
+  return !!rowCount;
+}
+
+// Testing helpers
+export async function testGet() {
+  const db = await loadDB();
+  return { db, loadedLocales };
+}
+export function testClear() {
+  loadedLocales.clear();
+  loadDB.reset();
+}

app/javascript/mastodon/features/emoji/emoji_html.tsx

@@ -1,81 +1,31 @@
-import type { HTMLAttributes } from 'react';
-import { useEffect, useMemo, useState } from 'react';
+import type { ComponentPropsWithoutRef, ElementType } from 'react';
 
-import type { List as ImmutableList } from 'immutable';
-import { isList } from 'immutable';
+import { useEmojify } from './hooks';
+import type { CustomEmojiMapArg } from './types';
 
-import type { ApiCustomEmojiJSON } from '@/mastodon/api_types/custom_emoji';
-import type { CustomEmoji } from '@/mastodon/models/custom_emoji';
-import { isModernEmojiEnabled } from '@/mastodon/utils/environment';
-
-import { useEmojiAppState } from './hooks';
-import { emojifyElement } from './render';
-import type { ExtraCustomEmojiMap } from './types';
-
-type EmojiHTMLProps = Omit<
-  HTMLAttributes<HTMLDivElement>,
+type EmojiHTMLProps<Element extends ElementType = 'div'> = Omit<
+  ComponentPropsWithoutRef<Element>,
   'dangerouslySetInnerHTML'
 > & {
   htmlString: string;
-  extraEmojis?: ExtraCustomEmojiMap | ImmutableList<CustomEmoji>;
+  extraEmojis?: CustomEmojiMapArg;
+  as?: Element;
 };
 
-export const EmojiHTML: React.FC<EmojiHTMLProps> = ({
-  htmlString,
+export const EmojiHTML = <Element extends ElementType>({
   extraEmojis,
+  htmlString,
+  as: asElement, // Rename for syntax highlighting
   ...props
-}) => {
-  if (isModernEmojiEnabled()) {
-    return (
-      <ModernEmojiHTML
-        htmlString={htmlString}
-        extraEmojis={extraEmojis}
-        {...props}
-      />
-    );
-  }
-  return <div dangerouslySetInnerHTML={{ __html: htmlString }} {...props} />;
-};
+}: EmojiHTMLProps<Element>) => {
+  const Wrapper = asElement ?? 'div';
+  const emojifiedHtml = useEmojify(htmlString, extraEmojis);
 
-const ModernEmojiHTML: React.FC<EmojiHTMLProps> = ({
-  extraEmojis: rawEmojis,
-  htmlString: text,
-  ...props
-}) => {
-  const appState = useEmojiAppState();
-  const [innerHTML, setInnerHTML] = useState('');
-
-  const extraEmojis: ExtraCustomEmojiMap = useMemo(() => {
-    if (!rawEmojis) {
-      return {};
-    }
-    if (isList(rawEmojis)) {
-      return (
-        rawEmojis.toJS() as ApiCustomEmojiJSON[]
-      ).reduce<ExtraCustomEmojiMap>(
-        (acc, emoji) => ({ ...acc, [emoji.shortcode]: emoji }),
-        {},
-      );
-    }
-    return rawEmojis;
-  }, [rawEmojis]);
-
-  useEffect(() => {
-    if (!text) {
-      return;
-    }
-    const cb = async () => {
-      const div = document.createElement('div');
-      div.innerHTML = text;
-      const ele = await emojifyElement(div, appState, extraEmojis);
-      setInnerHTML(ele.innerHTML);
-    };
-    void cb();
-  }, [text, appState, extraEmojis]);
-
-  if (!innerHTML) {
+  if (emojifiedHtml === null) {
     return null;
   }
 
-  return <div {...props} dangerouslySetInnerHTML={{ __html: innerHTML }} />;
+  return (
+    <Wrapper {...props} dangerouslySetInnerHTML={{ __html: emojifiedHtml }} />
+  );
 };

app/javascript/mastodon/features/emoji/emoji_text.tsx

@@ -1,45 +0,0 @@
-import { useEffect, useState } from 'react';
-
-import { useEmojiAppState } from './hooks';
-import { emojifyText } from './render';
-
-interface EmojiTextProps {
-  text: string;
-}
-
-export const EmojiText: React.FC<EmojiTextProps> = ({ text }) => {
-  const appState = useEmojiAppState();
-  const [rendered, setRendered] = useState<(string | HTMLImageElement)[]>([]);
-
-  useEffect(() => {
-    const cb = async () => {
-      const rendered = await emojifyText(text, appState);
-      setRendered(rendered ?? []);
-    };
-    void cb();
-  }, [text, appState]);
-
-  if (rendered.length === 0) {
-    return null;
-  }
-
-  return (
-    <>
-      {rendered.map((fragment, index) => {
-        if (typeof fragment === 'string') {
-          return <span key={index}>{fragment}</span>;
-        }
-        return (
-          <img
-            key={index}
-            draggable='false'
-            src={fragment.src}
-            alt={fragment.alt}
-            title={fragment.title}
-            className={fragment.className}
-          />
-        );
-      })}
-    </>
-  );
-};

app/javascript/mastodon/features/emoji/hooks.ts

@@ -1,8 +1,64 @@
+import { useCallback, useLayoutEffect, useMemo, useState } from 'react';
+
+import { isList } from 'immutable';
+
+import type { ApiCustomEmojiJSON } from '@/mastodon/api_types/custom_emoji';
 import { useAppSelector } from '@/mastodon/store';
+import { isModernEmojiEnabled } from '@/mastodon/utils/environment';
 
 import { toSupportedLocale } from './locale';
 import { determineEmojiMode } from './mode';
-import type { EmojiAppState } from './types';
+import { emojifyElement } from './render';
+import type {
+  CustomEmojiMapArg,
+  EmojiAppState,
+  ExtraCustomEmojiMap,
+} from './types';
+import { stringHasAnyEmoji } from './utils';
+
+export function useEmojify(text: string, extraEmojis?: CustomEmojiMapArg) {
+  const [emojifiedText, setEmojifiedText] = useState<string | null>(null);
+
+  const appState = useEmojiAppState();
+  const extra: ExtraCustomEmojiMap = useMemo(() => {
+    if (!extraEmojis) {
+      return {};
+    }
+    if (isList(extraEmojis)) {
+      return (
+        extraEmojis.toJS() as ApiCustomEmojiJSON[]
+      ).reduce<ExtraCustomEmojiMap>(
+        (acc, emoji) => ({ ...acc, [emoji.shortcode]: emoji }),
+        {},
+      );
+    }
+    return extraEmojis;
+  }, [extraEmojis]);
+
+  const emojify = useCallback(
+    async (input: string) => {
+      const wrapper = document.createElement('div');
+      wrapper.innerHTML = input;
+      const result = await emojifyElement(wrapper, appState, extra);
+      if (result) {
+        setEmojifiedText(result.innerHTML);
+      } else {
+        setEmojifiedText(input);
+      }
+    },
+    [appState, extra],
+  );
+  useLayoutEffect(() => {
+    if (isModernEmojiEnabled() && !!text.trim() && stringHasAnyEmoji(text)) {
+      void emojify(text);
+    } else {
+      // If no emoji or we don't want to render, fall back.
+      setEmojifiedText(text);
+    }
+  }, [emojify, text]);
+
+  return emojifiedText;
+}
 
 export function useEmojiAppState(): EmojiAppState {
   const locale = useAppSelector((state) =>
@@ -12,5 +68,10 @@ export function useEmojiAppState(): EmojiAppState {
     determineEmojiMode(state.meta.get('emoji_style') as string),
   );
 
-  return { currentLocale: locale, locales: [locale], mode };
+  return {
+    currentLocale: locale,
+    locales: [locale],
+    mode,
+    darkTheme: document.body.classList.contains('theme-default'),
+  };
 }

app/javascript/mastodon/features/emoji/index.ts

@@ -2,12 +2,16 @@ import initialState from '@/mastodon/initial_state';
 import { loadWorker } from '@/mastodon/utils/workers';
 
 import { toSupportedLocale } from './locale';
+import { emojiLogger } from './utils';
 
 const userLocale = toSupportedLocale(initialState?.meta.locale ?? 'en');
 
 let worker: Worker | null = null;
 
-export async function initializeEmoji() {
+const log = emojiLogger('index');
+
+export function initializeEmoji() {
+  log('initializing emojis');
   if (!worker && 'Worker' in window) {
     try {
       worker = loadWorker(new URL('./worker', import.meta.url), {
@@ -21,9 +25,16 @@ export async function initializeEmoji() {
   if (worker) {
     // Assign worker to const to make TS happy inside the event listener.
     const thisWorker = worker;
+    const timeoutId = setTimeout(() => {
+      log('worker is not ready after timeout');
+      worker = null;
+      void fallbackLoad();
+    }, 500);
     thisWorker.addEventListener('message', (event: MessageEvent<string>) => {
       const { data: message } = event;
       if (message === 'ready') {
+        log('worker ready, loading data');
+        clearTimeout(timeoutId);
         thisWorker.postMessage('custom');
         void loadEmojiLocale(userLocale);
         // Load English locale as well, because people are still used to
@@ -31,15 +42,22 @@ export async function initializeEmoji() {
         if (userLocale !== 'en') {
           void loadEmojiLocale('en');
         }
+      } else {
+        log('got worker message: %s', message);
       }
     });
   } else {
-    const { importCustomEmojiData } = await import('./loader');
-    await importCustomEmojiData();
-    await loadEmojiLocale(userLocale);
-    if (userLocale !== 'en') {
-      await loadEmojiLocale('en');
-    }
+    void fallbackLoad();
+  }
+}
+
+async function fallbackLoad() {
+  log('falling back to main thread for loading');
+  const { importCustomEmojiData } = await import('./loader');
+  await importCustomEmojiData();
+  await loadEmojiLocale(userLocale);
+  if (userLocale !== 'en') {
+    await loadEmojiLocale('en');
   }
 }
 

app/javascript/mastodon/features/emoji/loader.ts

@@ -2,7 +2,6 @@ import { flattenEmojiData } from 'emojibase';
 import type { CompactEmoji, FlatCompactEmoji } from 'emojibase';
 
 import type { ApiCustomEmojiJSON } from '@/mastodon/api_types/custom_emoji';
-import { isDevelopment } from '@/mastodon/utils/environment';
 
 import {
   putEmojiData,
@@ -12,6 +11,9 @@ import {
 } from './database';
 import { toSupportedLocale, toSupportedLocaleOrCustom } from './locale';
 import type { LocaleOrCustom } from './types';
+import { emojiLogger } from './utils';
+
+const log = emojiLogger('loader');
 
 export async function importEmojiData(localeString: string) {
   const locale = toSupportedLocale(localeString);
@@ -20,6 +22,7 @@ export async function importEmojiData(localeString: string) {
     return;
   }
   const flattenedEmojis: FlatCompactEmoji[] = flattenEmojiData(emojis);
+  log('loaded %d for %s locale', flattenedEmojis.length, locale);
   await putEmojiData(flattenedEmojis, locale);
 }
 
@@ -28,6 +31,7 @@ export async function importCustomEmojiData() {
   if (!emojis) {
     return;
   }
+  log('loaded %d custom emojis', emojis.length);
   await putCustomEmojiData(emojis);
 }
 
@@ -41,7 +45,9 @@ async function fetchAndCheckEtag<ResultType extends object[]>(
   if (locale === 'custom') {
     url.pathname = '/api/v1/custom_emojis';
   } else {
-    url.pathname = `/packs${isDevelopment() ? '-dev' : ''}/emoji/${locale}.json`;
+    // This doesn't use isDevelopment() as that module loads initial state
+    // which breaks workers, as they cannot access the DOM.
+    url.pathname = `/packs${import.meta.env.DEV ? '-dev' : ''}/emoji/${locale}.json`;
   }
 
   const oldEtag = await loadLatestEtag(locale);

app/javascript/mastodon/features/emoji/render.test.ts

@@ -1,94 +1,184 @@
+import { customEmojiFactory, unicodeEmojiFactory } from '@/testing/factories';
+
 import {
   EMOJI_MODE_NATIVE,
   EMOJI_MODE_NATIVE_WITH_FLAGS,
   EMOJI_MODE_TWEMOJI,
 } from './constants';
-import { emojifyElement, tokenizeText } from './render';
-import type { CustomEmojiData, UnicodeEmojiData } from './types';
+import * as db from './database';
+import {
+  emojifyElement,
+  emojifyText,
+  testCacheClear,
+  tokenizeText,
+} from './render';
+import type { EmojiAppState, ExtraCustomEmojiMap } from './types';
 
-vitest.mock('./database', () => ({
-  searchCustomEmojisByShortcodes: vitest.fn(
-    () =>
-      [
-        {
-          shortcode: 'custom',
-          static_url: 'emoji/static',
-          url: 'emoji/custom',
-          category: 'test',
-          visible_in_picker: true,
-        },
-      ] satisfies CustomEmojiData[],
-  ),
-  searchEmojisByHexcodes: vitest.fn(
-    () =>
-      [
-        {
+function mockDatabase() {
+  return {
+    searchCustomEmojisByShortcodes: vi
+      .spyOn(db, 'searchCustomEmojisByShortcodes')
+      .mockResolvedValue([customEmojiFactory()]),
+    searchEmojisByHexcodes: vi
+      .spyOn(db, 'searchEmojisByHexcodes')
+      .mockResolvedValue([
+        unicodeEmojiFactory({
           hexcode: '1F60A',
-          group: 0,
           label: 'smiling face with smiling eyes',
-          order: 0,
-          tags: ['smile', 'happy'],
           unicode: '😊',
-        },
-        {
+        }),
+        unicodeEmojiFactory({
           hexcode: '1F1EA-1F1FA',
-          group: 0,
           label: 'flag-eu',
-          order: 0,
-          tags: ['flag', 'european union'],
           unicode: '🇪🇺',
-        },
-      ] satisfies UnicodeEmojiData[],
-  ),
-  findMissingLocales: vitest.fn(() => []),
-}));
+        }),
+      ]),
+  };
+}
+
+const expectedSmileImage =
+  '<img draggable="false" class="emojione" alt="😊" title="smiling face with smiling eyes" src="/emoji/1f60a.svg">';
+const expectedFlagImage =
+  '<img draggable="false" class="emojione" alt="🇪🇺" title="flag-eu" src="/emoji/1f1ea-1f1fa.svg">';
+const expectedCustomEmojiImage =
+  '<img draggable="false" class="emojione custom-emoji" alt=":custom:" title=":custom:" src="emoji/custom/static" data-original="emoji/custom" data-static="emoji/custom/static">';
+const expectedRemoteCustomEmojiImage =
+  '<img draggable="false" class="emojione custom-emoji" alt=":remote:" title=":remote:" src="remote.social/static" data-original="remote.social/custom" data-static="remote.social/static">';
+
+const mockExtraCustom: ExtraCustomEmojiMap = {
+  remote: {
+    shortcode: 'remote',
+    static_url: 'remote.social/static',
+    url: 'remote.social/custom',
+  },
+};
+
+function testAppState(state: Partial<EmojiAppState> = {}) {
+  return {
+    locales: ['en'],
+    mode: EMOJI_MODE_TWEMOJI,
+    currentLocale: 'en',
+    darkTheme: false,
+    ...state,
+  } satisfies EmojiAppState;
+}
 
 describe('emojifyElement', () => {
-  const testElement = document.createElement('div');
-  testElement.innerHTML = '<p>Hello 😊🇪🇺!</p><p>:custom:</p>';
-
-  const expectedSmileImage =
-    '<img draggable="false" class="emojione" alt="😊" title="smiling face with smiling eyes" src="/emoji/1f60a.svg">';
-  const expectedFlagImage =
-    '<img draggable="false" class="emojione" alt="🇪🇺" title="flag-eu" src="/emoji/1f1ea-1f1fa.svg">';
-  const expectedCustomEmojiImage =
-    '<img draggable="false" class="emojione custom-emoji" alt=":custom:" title=":custom:" src="emoji/static" data-original="emoji/custom" data-static="emoji/static">';
-
-  function cloneTestElement() {
-    return testElement.cloneNode(true) as HTMLElement;
+  function testElement(text = '<p>Hello 😊🇪🇺!</p><p>:custom:</p>') {
+    const testElement = document.createElement('div');
+    testElement.innerHTML = text;
+    return testElement;
   }
 
+  afterEach(() => {
+    testCacheClear();
+    vi.restoreAllMocks();
+  });
+
+  test('caches element rendering results', async () => {
+    const { searchCustomEmojisByShortcodes, searchEmojisByHexcodes } =
+      mockDatabase();
+    await emojifyElement(testElement(), testAppState());
+    await emojifyElement(testElement(), testAppState());
+    await emojifyElement(testElement(), testAppState());
+    expect(searchEmojisByHexcodes).toHaveBeenCalledExactlyOnceWith(
+      ['1F1EA-1F1FA', '1F60A'],
+      'en',
+    );
+    expect(searchCustomEmojisByShortcodes).toHaveBeenCalledExactlyOnceWith([
+      'custom',
+    ]);
+  });
+
   test('emojifies custom emoji in native mode', async () => {
-    const emojifiedElement = await emojifyElement(cloneTestElement(), {
-      locales: ['en'],
-      mode: EMOJI_MODE_NATIVE,
-      currentLocale: 'en',
-    });
-    expect(emojifiedElement.innerHTML).toBe(
+    const { searchEmojisByHexcodes } = mockDatabase();
+    const actual = await emojifyElement(
+      testElement(),
+      testAppState({ mode: EMOJI_MODE_NATIVE }),
+    );
+    assert(actual);
+    expect(actual.innerHTML).toBe(
       `<p>Hello 😊🇪🇺!</p><p>${expectedCustomEmojiImage}</p>`,
     );
+    expect(searchEmojisByHexcodes).not.toHaveBeenCalled();
   });
 
   test('emojifies flag emoji in native-with-flags mode', async () => {
-    const emojifiedElement = await emojifyElement(cloneTestElement(), {
-      locales: ['en'],
-      mode: EMOJI_MODE_NATIVE_WITH_FLAGS,
-      currentLocale: 'en',
-    });
-    expect(emojifiedElement.innerHTML).toBe(
+    const { searchEmojisByHexcodes } = mockDatabase();
+    const actual = await emojifyElement(
+      testElement(),
+      testAppState({ mode: EMOJI_MODE_NATIVE_WITH_FLAGS }),
+    );
+    assert(actual);
+    expect(actual.innerHTML).toBe(
       `<p>Hello 😊${expectedFlagImage}!</p><p>${expectedCustomEmojiImage}</p>`,
     );
+    expect(searchEmojisByHexcodes).toHaveBeenCalledOnce();
   });
 
   test('emojifies everything in twemoji mode', async () => {
-    const emojifiedElement = await emojifyElement(cloneTestElement(), {
-      locales: ['en'],
-      mode: EMOJI_MODE_TWEMOJI,
-      currentLocale: 'en',
-    });
-    expect(emojifiedElement.innerHTML).toBe(
+    const { searchCustomEmojisByShortcodes, searchEmojisByHexcodes } =
+      mockDatabase();
+    const actual = await emojifyElement(testElement(), testAppState());
+    assert(actual);
+    expect(actual.innerHTML).toBe(
       `<p>Hello ${expectedSmileImage}${expectedFlagImage}!</p><p>${expectedCustomEmojiImage}</p>`,
     );
+    expect(searchEmojisByHexcodes).toHaveBeenCalledOnce();
+    expect(searchCustomEmojisByShortcodes).toHaveBeenCalledOnce();
+  });
+
+  test('emojifies with provided custom emoji', async () => {
+    const { searchCustomEmojisByShortcodes, searchEmojisByHexcodes } =
+      mockDatabase();
+    const actual = await emojifyElement(
+      testElement('<p>hi :remote:</p>'),
+      testAppState(),
+      mockExtraCustom,
+    );
+    assert(actual);
+    expect(actual.innerHTML).toBe(
+      `<p>hi ${expectedRemoteCustomEmojiImage}</p>`,
+    );
+    expect(searchEmojisByHexcodes).not.toHaveBeenCalled();
+    expect(searchCustomEmojisByShortcodes).not.toHaveBeenCalled();
+  });
+
+  test('returns null when no emoji are found', async () => {
+    mockDatabase();
+    const actual = await emojifyElement(
+      testElement('<p>here is just text :)</p>'),
+      testAppState(),
+    );
+    expect(actual).toBeNull();
+  });
+});
+
+describe('emojifyText', () => {
+  test('returns original input when no emoji are in string', async () => {
+    const actual = await emojifyText('nothing here', testAppState());
+    expect(actual).toBe('nothing here');
+  });
+
+  test('renders Unicode emojis to twemojis', async () => {
+    mockDatabase();
+    const actual = await emojifyText('Hello 😊🇪🇺!', testAppState());
+    expect(actual).toBe(`Hello ${expectedSmileImage}${expectedFlagImage}!`);
+  });
+
+  test('renders custom emojis', async () => {
+    mockDatabase();
+    const actual = await emojifyText('Hello :custom:!', testAppState());
+    expect(actual).toBe(`Hello ${expectedCustomEmojiImage}!`);
+  });
+
+  test('renders provided extra emojis', async () => {
+    const actual = await emojifyText(
+      'remote emoji :remote:',
+      testAppState(),
+      mockExtraCustom,
+    );
+    expect(actual).toBe(`remote emoji ${expectedRemoteCustomEmojiImage}`);
   });
 });
 

app/javascript/mastodon/features/emoji/render.ts

@@ -1,8 +1,7 @@
-import type { Locale } from 'emojibase';
-import EMOJI_REGEX from 'emojibase-regex/emoji-loose';
-
 import { autoPlayGif } from '@/mastodon/initial_state';
+import { createLimitedCache } from '@/mastodon/utils/cache';
 import { assetHost } from '@/mastodon/utils/config';
+import * as perf from '@/mastodon/utils/performance';
 
 import {
   EMOJI_MODE_NATIVE,
@@ -10,13 +9,12 @@ import {
   EMOJI_TYPE_UNICODE,
   EMOJI_TYPE_CUSTOM,
   EMOJI_STATE_MISSING,
+  ANY_EMOJI_REGEX,
 } from './constants';
 import {
-  findMissingLocales,
   searchCustomEmojisByShortcodes,
   searchEmojisByHexcodes,
 } from './database';
-import { loadEmojiLocale } from './index';
 import {
   emojiToUnicodeHex,
   twemojiHasBorder,
@@ -34,18 +32,33 @@ import type {
   LocaleOrCustom,
   UnicodeEmojiToken,
 } from './types';
-import { stringHasUnicodeFlags } from './utils';
+import { emojiLogger, stringHasAnyEmoji, stringHasUnicodeFlags } from './utils';
 
-const localeCacheMap = new Map<LocaleOrCustom, EmojiStateMap>([
-  [EMOJI_TYPE_CUSTOM, new Map()],
-]);
+const log = emojiLogger('render');
 
-// Emojifies an element. This modifies the element in place, replacing text nodes with emojified versions.
+/**
+ * Emojifies an element. This modifies the element in place, replacing text nodes with emojified versions.
+ */
 export async function emojifyElement<Element extends HTMLElement>(
   element: Element,
   appState: EmojiAppState,
   extraEmojis: ExtraCustomEmojiMap = {},
-): Promise<Element> {
+): Promise<Element | null> {
+  const cacheKey = createCacheKey(element, appState, extraEmojis);
+  const cached = getCached(cacheKey);
+  if (cached !== undefined) {
+    log('Cache hit on %s', element.outerHTML);
+    if (cached === null) {
+      return null;
+    }
+    element.innerHTML = cached;
+    return element;
+  }
+  if (!stringHasAnyEmoji(element.innerHTML)) {
+    updateCache(cacheKey, null);
+    return null;
+  }
+  perf.start('emojifyElement()');
   const queue: (HTMLElement | Text)[] = [element];
   while (queue.length > 0) {
     const current = queue.shift();
@@ -61,7 +74,7 @@ export async function emojifyElement<Element extends HTMLElement>(
       current.textContent &&
       (current instanceof Text || !current.hasChildNodes())
     ) {
-      const renderedContent = await emojifyText(
+      const renderedContent = await textToElementArray(
         current.textContent,
         appState,
         extraEmojis,
@@ -70,7 +83,7 @@ export async function emojifyElement<Element extends HTMLElement>(
         if (!(current instanceof Text)) {
           current.textContent = null; // Clear the text content if it's not a Text node.
         }
-        current.replaceWith(renderedToHTMLFragment(renderedContent));
+        current.replaceWith(renderedToHTML(renderedContent));
       }
       continue;
     }
@@ -81,6 +94,8 @@ export async function emojifyElement<Element extends HTMLElement>(
       }
     }
   }
+  updateCache(cacheKey, element.innerHTML);
+  perf.stop('emojifyElement()');
   return element;
 }
 
@@ -88,7 +103,54 @@ export async function emojifyText(
   text: string,
   appState: EmojiAppState,
   extraEmojis: ExtraCustomEmojiMap = {},
+): Promise<string | null> {
+  const cacheKey = createCacheKey(text, appState, extraEmojis);
+  const cached = getCached(cacheKey);
+  if (cached !== undefined) {
+    log('Cache hit on %s', text);
+    return cached ?? text;
+  }
+  if (!stringHasAnyEmoji(text)) {
+    updateCache(cacheKey, null);
+    return text;
+  }
+  const eleArray = await textToElementArray(text, appState, extraEmojis);
+  if (!eleArray) {
+    updateCache(cacheKey, null);
+    return text;
+  }
+  const rendered = renderedToHTML(eleArray, document.createElement('div'));
+  updateCache(cacheKey, rendered.innerHTML);
+  return rendered.innerHTML;
+}
+
+// Private functions
+
+const {
+  set: updateCache,
+  get: getCached,
+  clear: cacheClear,
+} = createLimitedCache<string | null>({ log: log.extend('cache') });
+
+function createCacheKey(
+  input: HTMLElement | string,
+  appState: EmojiAppState,
+  extraEmojis: ExtraCustomEmojiMap,
 ) {
+  return JSON.stringify([
+    input instanceof HTMLElement ? input.outerHTML : input,
+    appState,
+    extraEmojis,
+  ]);
+}
+
+type EmojifiedTextArray = (string | HTMLImageElement)[];
+
+async function textToElementArray(
+  text: string,
+  appState: EmojiAppState,
+  extraEmojis: ExtraCustomEmojiMap = {},
+): Promise<EmojifiedTextArray | null> {
   // Exit if no text to convert.
   if (!text.trim()) {
     return null;
@@ -102,10 +164,9 @@ export async function emojifyText(
   }
 
   // Get all emoji from the state map, loading any missing ones.
-  await ensureLocalesAreLoaded(appState.locales);
-  await loadMissingEmojiIntoCache(tokens, appState.locales);
+  await loadMissingEmojiIntoCache(tokens, appState, extraEmojis);
 
-  const renderedFragments: (string | HTMLImageElement)[] = [];
+  const renderedFragments: EmojifiedTextArray = [];
   for (const token of tokens) {
     if (typeof token !== 'string' && shouldRenderImage(token, appState.mode)) {
       let state: EmojiState | undefined;
@@ -125,7 +186,7 @@ export async function emojifyText(
 
       // If the state is valid, create an image element. Otherwise, just append as text.
       if (state && typeof state !== 'string') {
-        const image = stateToImage(state);
+        const image = stateToImage(state, appState);
         renderedFragments.push(image);
         continue;
       }
@@ -137,21 +198,6 @@ export async function emojifyText(
   return renderedFragments;
 }
 
-// Private functions
-
-async function ensureLocalesAreLoaded(locales: Locale[]) {
-  const missingLocales = await findMissingLocales(locales);
-  for (const locale of missingLocales) {
-    await loadEmojiLocale(locale);
-  }
-}
-
-const CUSTOM_EMOJI_REGEX = /:([a-z0-9_]+):/i;
-const TOKENIZE_REGEX = new RegExp(
-  `(${EMOJI_REGEX.source}|${CUSTOM_EMOJI_REGEX.source})`,
-  'g',
-);
-
 type TokenizedText = (string | EmojiToken)[];
 
 export function tokenizeText(text: string): TokenizedText {
@@ -161,7 +207,7 @@ export function tokenizeText(text: string): TokenizedText {
 
   const tokens = [];
   let lastIndex = 0;
-  for (const match of text.matchAll(TOKENIZE_REGEX)) {
+  for (const match of text.matchAll(ANY_EMOJI_REGEX)) {
     if (match.index > lastIndex) {
       tokens.push(text.slice(lastIndex, match.index));
     }
@@ -189,8 +235,18 @@ export function tokenizeText(text: string): TokenizedText {
   return tokens;
 }
 
+const localeCacheMap = new Map<LocaleOrCustom, EmojiStateMap>([
+  [
+    EMOJI_TYPE_CUSTOM,
+    createLimitedCache<EmojiState>({ log: log.extend('custom') }),
+  ],
+]);
+
 function cacheForLocale(locale: LocaleOrCustom): EmojiStateMap {
-  return localeCacheMap.get(locale) ?? (new Map() as EmojiStateMap);
+  return (
+    localeCacheMap.get(locale) ??
+    createLimitedCache<EmojiState>({ log: log.extend(locale) })
+  );
 }
 
 function emojiForLocale(
@@ -203,7 +259,8 @@ function emojiForLocale(
 
 async function loadMissingEmojiIntoCache(
   tokens: TokenizedText,
-  locales: Locale[],
+  { mode, currentLocale }: EmojiAppState,
+  extraEmojis: ExtraCustomEmojiMap,
 ) {
   const missingUnicodeEmoji = new Set<string>();
   const missingCustomEmoji = new Set<string>();
@@ -217,42 +274,41 @@ async function loadMissingEmojiIntoCache(
     // If this is a custom emoji, check it separately.
     if (token.type === EMOJI_TYPE_CUSTOM) {
       const code = token.code;
+      if (code in extraEmojis) {
+        continue; // We don't care about extra emoji.
+      }
       const emojiState = emojiForLocale(code, EMOJI_TYPE_CUSTOM);
       if (!emojiState) {
         missingCustomEmoji.add(code);
       }
       // Otherwise this is a unicode emoji, so check it against all locales.
-    } else {
+    } else if (shouldRenderImage(token, mode)) {
       const code = emojiToUnicodeHex(token.code);
       if (missingUnicodeEmoji.has(code)) {
         continue; // Already marked as missing.
       }
-      for (const locale of locales) {
-        const emojiState = emojiForLocale(code, locale);
-        if (!emojiState) {
-          // If it's missing in one locale, we consider it missing for all.
-          missingUnicodeEmoji.add(code);
-        }
+      const emojiState = emojiForLocale(code, currentLocale);
+      if (!emojiState) {
+        // If it's missing in one locale, we consider it missing for all.
+        missingUnicodeEmoji.add(code);
       }
     }
   }
 
   if (missingUnicodeEmoji.size > 0) {
     const missingEmojis = Array.from(missingUnicodeEmoji).toSorted();
-    for (const locale of locales) {
-      const emojis = await searchEmojisByHexcodes(missingEmojis, locale);
-      const cache = cacheForLocale(locale);
-      for (const emoji of emojis) {
-        cache.set(emoji.hexcode, { type: EMOJI_TYPE_UNICODE, data: emoji });
-      }
-      const notFoundEmojis = missingEmojis.filter((code) =>
-        emojis.every((emoji) => emoji.hexcode !== code),
-      );
-      for (const code of notFoundEmojis) {
-        cache.set(code, EMOJI_STATE_MISSING); // Mark as missing if not found, as it's probably not a valid emoji.
-      }
-      localeCacheMap.set(locale, cache);
+    const emojis = await searchEmojisByHexcodes(missingEmojis, currentLocale);
+    const cache = cacheForLocale(currentLocale);
+    for (const emoji of emojis) {
+      cache.set(emoji.hexcode, { type: EMOJI_TYPE_UNICODE, data: emoji });
     }
+    const notFoundEmojis = missingEmojis.filter((code) =>
+      emojis.every((emoji) => emoji.hexcode !== code),
+    );
+    for (const code of notFoundEmojis) {
+      cache.set(code, EMOJI_STATE_MISSING); // Mark as missing if not found, as it's probably not a valid emoji.
+    }
+    localeCacheMap.set(currentLocale, cache);
   }
 
   if (missingCustomEmoji.size > 0) {
@@ -288,22 +344,24 @@ function shouldRenderImage(token: EmojiToken, mode: EmojiMode): boolean {
   return true;
 }
 
-function stateToImage(state: EmojiLoadedState) {
+function stateToImage(state: EmojiLoadedState, appState: EmojiAppState) {
   const image = document.createElement('img');
   image.draggable = false;
   image.classList.add('emojione');
 
   if (state.type === EMOJI_TYPE_UNICODE) {
     const emojiInfo = twemojiHasBorder(unicodeToTwemojiHex(state.data.hexcode));
-    if (emojiInfo.hasLightBorder) {
-      image.dataset.lightCode = `${emojiInfo.hexCode}_BORDER`;
-    } else if (emojiInfo.hasDarkBorder) {
-      image.dataset.darkCode = `${emojiInfo.hexCode}_BORDER`;
+    let fileName = emojiInfo.hexCode;
+    if (
+      (appState.darkTheme && emojiInfo.hasDarkBorder) ||
+      (!appState.darkTheme && emojiInfo.hasLightBorder)
+    ) {
+      fileName = `${emojiInfo.hexCode}_border`;
     }
 
     image.alt = state.data.unicode;
     image.title = state.data.label;
-    image.src = `${assetHost}/emoji/${emojiInfo.hexCode}.svg`;
+    image.src = `${assetHost}/emoji/${fileName}.svg`;
   } else {
     // Custom emoji
     const shortCode = `:${state.data.shortcode}:`;
@@ -318,8 +376,16 @@ function stateToImage(state: EmojiLoadedState) {
   return image;
 }
 
-function renderedToHTMLFragment(renderedArray: (string | HTMLImageElement)[]) {
-  const fragment = document.createDocumentFragment();
+function renderedToHTML(renderedArray: EmojifiedTextArray): DocumentFragment;
+function renderedToHTML<ParentType extends ParentNode>(
+  renderedArray: EmojifiedTextArray,
+  parent: ParentType,
+): ParentType;
+function renderedToHTML(
+  renderedArray: EmojifiedTextArray,
+  parent: ParentNode | null = null,
+) {
+  const fragment = parent ?? document.createDocumentFragment();
   for (const fragmentItem of renderedArray) {
     if (typeof fragmentItem === 'string') {
       fragment.appendChild(document.createTextNode(fragmentItem));
@@ -329,3 +395,9 @@ function renderedToHTMLFragment(renderedArray: (string | HTMLImageElement)[]) {
   }
   return fragment;
 }
+
+// Testing helpers
+export const testCacheClear = () => {
+  cacheClear();
+  localeCacheMap.clear();
+};

app/javascript/mastodon/features/emoji/types.ts

@@ -1,6 +1,10 @@
+import type { List as ImmutableList } from 'immutable';
+
 import type { FlatCompactEmoji, Locale } from 'emojibase';
 
 import type { ApiCustomEmojiJSON } from '@/mastodon/api_types/custom_emoji';
+import type { CustomEmoji } from '@/mastodon/models/custom_emoji';
+import type { LimitedCache } from '@/mastodon/utils/cache';
 
 import type {
   EMOJI_MODE_NATIVE,
@@ -22,6 +26,7 @@ export interface EmojiAppState {
   locales: Locale[];
   currentLocale: Locale;
   mode: EmojiMode;
+  darkTheme: boolean;
 }
 
 export interface UnicodeEmojiToken {
@@ -45,7 +50,7 @@ export interface EmojiStateUnicode {
 }
 export interface EmojiStateCustom {
   type: typeof EMOJI_TYPE_CUSTOM;
-  data: CustomEmojiData;
+  data: CustomEmojiRenderFields;
 }
 export type EmojiState =
   | EmojiStateMissing
@@ -53,9 +58,16 @@ export type EmojiState =
   | EmojiStateCustom;
 export type EmojiLoadedState = EmojiStateUnicode | EmojiStateCustom;
 
-export type EmojiStateMap = Map<string, EmojiState>;
+export type EmojiStateMap = LimitedCache<string, EmojiState>;
 
-export type ExtraCustomEmojiMap = Record<string, ApiCustomEmojiJSON>;
+export type CustomEmojiMapArg =
+  | ExtraCustomEmojiMap
+  | ImmutableList<CustomEmoji>;
+export type CustomEmojiRenderFields = Pick<
+  CustomEmojiData,
+  'shortcode' | 'static_url' | 'url'
+>;
+export type ExtraCustomEmojiMap = Record<string, CustomEmojiRenderFields>;
 
 export interface TwemojiBorderInfo {
   hexCode: string;

app/javascript/mastodon/features/emoji/utils.test.ts

@@ -1,8 +1,14 @@
-import { stringHasUnicodeEmoji, stringHasUnicodeFlags } from './utils';
+import {
+  stringHasAnyEmoji,
+  stringHasCustomEmoji,
+  stringHasUnicodeEmoji,
+  stringHasUnicodeFlags,
+} from './utils';
 
-describe('stringHasEmoji', () => {
+describe('stringHasUnicodeEmoji', () => {
   test.concurrent.for([
     ['only text', false],
+    ['text with non-emoji symbols ™©', false],
     ['text with emoji 😀', true],
     ['multiple emojis 😀😃😄', true],
     ['emoji with skin tone 👍🏽', true],
@@ -19,14 +25,14 @@ describe('stringHasEmoji', () => {
     ['emoji with enclosing keycap #️⃣', true],
     ['emoji with no visible glyph \u200D', false],
   ] as const)(
-    'stringHasEmoji has emojis in "%s": %o',
+    'stringHasUnicodeEmoji has emojis in "%s": %o',
     ([text, expected], { expect }) => {
       expect(stringHasUnicodeEmoji(text)).toBe(expected);
     },
   );
 });
 
-describe('stringHasFlags', () => {
+describe('stringHasUnicodeFlags', () => {
   test.concurrent.for([
     ['EU 🇪🇺', true],
     ['Germany 🇩🇪', true],
@@ -45,3 +51,27 @@ describe('stringHasFlags', () => {
     },
   );
 });
+
+describe('stringHasCustomEmoji', () => {
+  test('string with custom emoji returns true', () => {
+    expect(stringHasCustomEmoji(':custom: :test:')).toBeTruthy();
+  });
+  test('string without custom emoji returns false', () => {
+    expect(stringHasCustomEmoji('🏳️‍🌈 :🏳️‍🌈: text ™')).toBeFalsy();
+  });
+});
+
+describe('stringHasAnyEmoji', () => {
+  test('string without any emoji or characters', () => {
+    expect(stringHasAnyEmoji('normal text. 12356?!')).toBeFalsy();
+  });
+  test('string with non-emoji characters', () => {
+    expect(stringHasAnyEmoji('™©')).toBeFalsy();
+  });
+  test('has unicode emoji', () => {
+    expect(stringHasAnyEmoji('🏳️‍🌈🔥🇸🇹 👩‍🔬')).toBeTruthy();
+  });
+  test('has custom emoji', () => {
+    expect(stringHasAnyEmoji(':test: :custom:')).toBeTruthy();
+  });
+});

app/javascript/mastodon/features/emoji/utils.ts

@@ -1,13 +1,27 @@
-import EMOJI_REGEX from 'emojibase-regex/emoji-loose';
+import debug from 'debug';
 
-export function stringHasUnicodeEmoji(text: string): boolean {
-  return EMOJI_REGEX.test(text);
+import {
+  CUSTOM_EMOJI_REGEX,
+  UNICODE_EMOJI_REGEX,
+  UNICODE_FLAG_EMOJI_REGEX,
+} from './constants';
+
+export function emojiLogger(segment: string) {
+  return debug(`emojis:${segment}`);
 }
 
-// From https://github.com/talkjs/country-flag-emoji-polyfill/blob/master/src/index.ts#L49-L50
-const EMOJIS_FLAGS_REGEX =
-  /[\u{1F1E6}-\u{1F1FF}|\u{E0062}-\u{E0063}|\u{E0065}|\u{E0067}|\u{E006C}|\u{E006E}|\u{E0073}-\u{E0074}|\u{E0077}|\u{E007F}]+/u;
-
-export function stringHasUnicodeFlags(text: string): boolean {
-  return EMOJIS_FLAGS_REGEX.test(text);
+export function stringHasUnicodeEmoji(input: string): boolean {
+  return UNICODE_EMOJI_REGEX.test(input);
+}
+
+export function stringHasUnicodeFlags(input: string): boolean {
+  return UNICODE_FLAG_EMOJI_REGEX.test(input);
+}
+
+export function stringHasCustomEmoji(input: string) {
+  return CUSTOM_EMOJI_REGEX.test(input);
+}
+
+export function stringHasAnyEmoji(input: string) {
+  return stringHasUnicodeEmoji(input) || stringHasCustomEmoji(input);
 }

app/javascript/mastodon/features/emoji/worker.ts

@@ -5,9 +5,14 @@ self.postMessage('ready'); // After the worker is ready, notify the main thread
 
 function handleMessage(event: MessageEvent<string>) {
   const { data: locale } = event;
-  if (locale !== 'custom') {
-    void importEmojiData(locale);
-  } else {
-    void importCustomEmojiData();
-  }
+  void loadData(locale);
+}
+
+async function loadData(locale: string) {
+  if (locale !== 'custom') {
+    await importEmojiData(locale);
+  } else {
+    await importCustomEmojiData();
+  }
+  self.postMessage(`loaded ${locale}`);
 }

app/javascript/mastodon/main.tsx

@@ -2,10 +2,10 @@ import { createRoot } from 'react-dom/client';
 
 import { Globals } from '@react-spring/web';
 
+import * as perf from '@/mastodon/utils/performance';
 import { setupBrowserNotifications } from 'mastodon/actions/notifications';
 import Mastodon from 'mastodon/containers/mastodon';
 import { me, reduceMotion } from 'mastodon/initial_state';
-import * as perf from 'mastodon/performance';
 import ready from 'mastodon/ready';
 import { store } from 'mastodon/store';
 
@@ -35,7 +35,7 @@ function main() {
 
     if (isModernEmojiEnabled()) {
       const { initializeEmoji } = await import('@/mastodon/features/emoji');
-      await initializeEmoji();
+      initializeEmoji();
     }
 
     const root = createRoot(mountNode);

app/javascript/mastodon/utils/__tests__/cache.test.ts

@@ -0,0 +1,78 @@
+import { createLimitedCache } from '../cache';
+
+describe('createCache', () => {
+  test('returns expected methods', () => {
+    const actual = createLimitedCache();
+    expect(actual).toBeTypeOf('object');
+    expect(actual).toHaveProperty('get');
+    expect(actual).toHaveProperty('has');
+    expect(actual).toHaveProperty('delete');
+    expect(actual).toHaveProperty('set');
+  });
+
+  test('caches values provided to it', () => {
+    const cache = createLimitedCache();
+    cache.set('test', 'result');
+    expect(cache.get('test')).toBe('result');
+  });
+
+  test('has returns expected values', () => {
+    const cache = createLimitedCache();
+    cache.set('test', 'result');
+    expect(cache.has('test')).toBeTruthy();
+    expect(cache.has('not found')).toBeFalsy();
+  });
+
+  test('updates a value if keys are the same', () => {
+    const cache = createLimitedCache();
+    cache.set('test1', 1);
+    cache.set('test1', 2);
+    expect(cache.get('test1')).toBe(2);
+  });
+
+  test('delete removes an item', () => {
+    const cache = createLimitedCache();
+    cache.set('test', 'result');
+    expect(cache.has('test')).toBeTruthy();
+    cache.delete('test');
+    expect(cache.has('test')).toBeFalsy();
+    expect(cache.get('test')).toBeUndefined();
+  });
+
+  test('removes oldest item cached if it exceeds a set size', () => {
+    const cache = createLimitedCache({ maxSize: 1 });
+    cache.set('test1', 1);
+    cache.set('test2', 2);
+    expect(cache.get('test1')).toBeUndefined();
+    expect(cache.get('test2')).toBe(2);
+  });
+
+  test('retrieving a value bumps up last access', () => {
+    const cache = createLimitedCache({ maxSize: 2 });
+    cache.set('test1', 1);
+    cache.set('test2', 2);
+    expect(cache.get('test1')).toBe(1);
+    cache.set('test3', 3);
+    expect(cache.get('test1')).toBe(1);
+    expect(cache.get('test2')).toBeUndefined();
+    expect(cache.get('test3')).toBe(3);
+  });
+
+  test('logs when cache is added to and removed', () => {
+    const log = vi.fn();
+    const cache = createLimitedCache({ maxSize: 1, log });
+    cache.set('test1', 1);
+    expect(log).toHaveBeenLastCalledWith(
+      'Added %s to cache, now size %d',
+      'test1',
+      1,
+    );
+    cache.set('test2', 1);
+    expect(log).toHaveBeenLastCalledWith(
+      'Added %s and deleted %s from cache, now size %d',
+      'test2',
+      'test1',
+      1,
+    );
+  });
+});

app/javascript/mastodon/utils/cache.ts

@@ -0,0 +1,60 @@
+export interface LimitedCache<CacheKey, CacheValue> {
+  has: (key: CacheKey) => boolean;
+  get: (key: CacheKey) => CacheValue | undefined;
+  delete: (key: CacheKey) => void;
+  set: (key: CacheKey, value: CacheValue) => void;
+  clear: () => void;
+}
+
+interface LimitedCacheArguments {
+  maxSize?: number;
+  log?: (...args: unknown[]) => void;
+}
+
+export function createLimitedCache<CacheValue, CacheKey = string>({
+  maxSize = 100,
+  log = () => null,
+}: LimitedCacheArguments = {}): LimitedCache<CacheKey, CacheValue> {
+  const cacheMap = new Map<CacheKey, CacheValue>();
+  const cacheKeys = new Set<CacheKey>();
+
+  function touchKey(key: CacheKey) {
+    if (cacheKeys.has(key)) {
+      cacheKeys.delete(key);
+    }
+    cacheKeys.add(key);
+  }
+
+  return {
+    has: (key) => cacheMap.has(key),
+    get: (key) => {
+      if (cacheMap.has(key)) {
+        touchKey(key);
+      }
+      return cacheMap.get(key);
+    },
+    delete: (key) => cacheMap.delete(key) && cacheKeys.delete(key),
+    set: (key, value) => {
+      cacheMap.set(key, value);
+      touchKey(key);
+
+      const lastKey = cacheKeys.values().toArray().shift();
+      if (cacheMap.size > maxSize && lastKey) {
+        cacheMap.delete(lastKey);
+        cacheKeys.delete(lastKey);
+        log(
+          'Added %s and deleted %s from cache, now size %d',
+          key,
+          lastKey,
+          cacheMap.size,
+        );
+      } else {
+        log('Added %s to cache, now size %d', key, cacheMap.size);
+      }
+    },
+    clear: () => {
+      cacheMap.clear();
+      cacheKeys.clear();
+    },
+  };
+}

app/javascript/mastodon/utils/performance.ts

@@ -4,15 +4,15 @@
 
 import * as marky from 'marky';
 
-import { isDevelopment } from './utils/environment';
+import { isDevelopment } from './environment';
 
-export function start(name) {
+export function start(name: string) {
   if (isDevelopment()) {
     marky.mark(name);
   }
 }
 
-export function stop(name) {
+export function stop(name: string) {
   if (isDevelopment()) {
     marky.stop(name);
   }

app/javascript/testing/factories.ts

@@ -1,4 +1,8 @@
 import type { ApiRelationshipJSON } from '@/mastodon/api_types/relationships';
+import type {
+  CustomEmojiData,
+  UnicodeEmojiData,
+} from '@/mastodon/features/emoji/types';
 import { createAccountFromServerJSON } from '@/mastodon/models/account';
 import type { ApiAccountJSON } from 'mastodon/api_types/accounts';
 
@@ -68,3 +72,26 @@ export const relationshipsFactory: FactoryFunction<ApiRelationshipJSON> = ({
   showing_reblogs: true,
   ...data,
 });
+
+export function unicodeEmojiFactory(
+  data: Partial<UnicodeEmojiData> = {},
+): UnicodeEmojiData {
+  return {
+    hexcode: 'test',
+    label: 'Test',
+    unicode: '🧪',
+    ...data,
+  };
+}
+
+export function customEmojiFactory(
+  data: Partial<CustomEmojiData> = {},
+): CustomEmojiData {
+  return {
+    shortcode: 'custom',
+    static_url: 'emoji/custom/static',
+    url: 'emoji/custom',
+    visible_in_picker: true,
+    ...data,
+  };
+}

eslint.config.mjs

@@ -406,6 +406,12 @@ export default tseslint.config([
       globals: globals.vitest,
     },
   },
+  {
+    files: ['**/*.test.*'],
+    rules: {
+      'no-global-assign': 'off',
+    },
+  },
   {
     files: ['**/*.stories.ts', '**/*.stories.tsx', '.storybook/*'],
     rules: {

package.json

@@ -64,11 +64,11 @@
     "color-blend": "^4.0.0",
     "core-js": "^3.30.2",
     "cross-env": "^10.0.0",
+    "debug": "^4.4.1",
     "detect-passive-events": "^2.0.3",
     "emoji-mart": "npm:emoji-mart-lazyload@latest",
     "emojibase": "^16.0.0",
     "emojibase-data": "^16.0.3",
-    "emojibase-regex": "^16.0.0",
     "escape-html": "^1.0.3",
     "fast-glob": "^3.3.3",
     "fuzzysort": "^3.0.0",
@@ -137,6 +137,7 @@
     "@storybook/react-vite": "^9.0.4",
     "@testing-library/dom": "^10.2.0",
     "@testing-library/react": "^16.0.0",
+    "@types/debug": "^4",
     "@types/emoji-mart": "3.0.14",
     "@types/escape-html": "^1.0.2",
     "@types/hoist-non-react-statics": "^3.3.1",
@@ -174,6 +175,7 @@
     "eslint-plugin-react": "^7.37.4",
     "eslint-plugin-react-hooks": "^5.2.0",
     "eslint-plugin-storybook": "^9.0.4",
+    "fake-indexeddb": "^6.0.1",
     "globals": "^16.0.0",
     "husky": "^9.0.11",
     "lint-staged": "^16.0.0",

vitest.config.mts

@@ -49,6 +49,7 @@ const legacyTests: TestProjectInlineConfiguration = {
       'tmp/**',
     ],
     globals: true,
+    setupFiles: ['fake-indexeddb/auto'],
   },
 };
 

yarn.lock

@@ -2632,6 +2632,7 @@ __metadata:
     "@storybook/react-vite": "npm:^9.0.4"
     "@testing-library/dom": "npm:^10.2.0"
     "@testing-library/react": "npm:^16.0.0"
+    "@types/debug": "npm:^4"
     "@types/emoji-mart": "npm:3.0.14"
     "@types/escape-html": "npm:^1.0.2"
     "@types/hoist-non-react-statics": "npm:^3.3.1"
@@ -2673,11 +2674,11 @@ __metadata:
     color-blend: "npm:^4.0.0"
     core-js: "npm:^3.30.2"
     cross-env: "npm:^10.0.0"
+    debug: "npm:^4.4.1"
     detect-passive-events: "npm:^2.0.3"
     emoji-mart: "npm:emoji-mart-lazyload@latest"
     emojibase: "npm:^16.0.0"
     emojibase-data: "npm:^16.0.3"
-    emojibase-regex: "npm:^16.0.0"
     escape-html: "npm:^1.0.3"
     eslint: "npm:^9.23.0"
     eslint-import-resolver-typescript: "npm:^4.2.5"
@@ -2689,6 +2690,7 @@ __metadata:
     eslint-plugin-react: "npm:^7.37.4"
     eslint-plugin-react-hooks: "npm:^5.2.0"
     eslint-plugin-storybook: "npm:^9.0.4"
+    fake-indexeddb: "npm:^6.0.1"
     fast-glob: "npm:^3.3.3"
     fuzzysort: "npm:^3.0.0"
     globals: "npm:^16.0.0"
@@ -3931,6 +3933,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@types/debug@npm:^4":
+  version: 4.1.12
+  resolution: "@types/debug@npm:4.1.12"
+  dependencies:
+    "@types/ms": "npm:*"
+  checksum: 10c0/5dcd465edbb5a7f226e9a5efd1f399c6172407ef5840686b73e3608ce135eeca54ae8037dcd9f16bdb2768ac74925b820a8b9ecc588a58ca09eca6acabe33e2f
+  languageName: node
+  linkType: hard
+
 "@types/deep-eql@npm:*":
   version: 4.0.2
   resolution: "@types/deep-eql@npm:4.0.2"
@@ -4112,6 +4123,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@types/ms@npm:*":
+  version: 2.1.0
+  resolution: "@types/ms@npm:2.1.0"
+  checksum: 10c0/5ce692ffe1549e1b827d99ef8ff71187457e0eb44adbae38fdf7b9a74bae8d20642ee963c14516db1d35fa2652e65f47680fdf679dcbde52bbfadd021f497225
+  languageName: node
+  linkType: hard
+
 "@types/node@npm:*, @types/node@npm:^22.0.0":
   version: 22.13.14
   resolution: "@types/node@npm:22.13.14"
@@ -6599,13 +6617,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"emojibase-regex@npm:^16.0.0":
-  version: 16.0.0
-  resolution: "emojibase-regex@npm:16.0.0"
-  checksum: 10c0/8ee5ff798e51caa581434b1cb2f9737e50195093c4efa1739df21a50a5496f80517924787d865e8cf7d6a0b4c90dbedc04bdc506dcbcc582e14cdf0bb47af0f0
-  languageName: node
-  linkType: hard
-
 "emojibase@npm:^16.0.0":
   version: 16.0.0
   resolution: "emojibase@npm:16.0.0"
@@ -7370,6 +7381,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"fake-indexeddb@npm:^6.0.1":
+  version: 6.0.1
+  resolution: "fake-indexeddb@npm:6.0.1"
+  checksum: 10c0/60f4ccdfd5ecb37bb98019056c688366847840cce7146e0005c5ca54823238455403b0a8803b898a11cf80f6147b1bb553457c6af427a644a6e64566cdbe42ec
+  languageName: node
+  linkType: hard
+
 "fast-copy@npm:^3.0.2":
   version: 3.0.2
   resolution: "fast-copy@npm:3.0.2"