Emoji: Refresh custom emoji on new (#37271)

app/javascript/mastodon/actions/importer/emoji.ts

@@ -0,0 +1,22 @@
+import type { ApiCustomEmojiJSON } from '@/mastodon/api_types/custom_emoji';
+import { loadCustomEmoji } from '@/mastodon/features/emoji';
+
+export async function importCustomEmoji(emojis: ApiCustomEmojiJSON[]) {
+  if (emojis.length === 0) {
+    return;
+  }
+
+  // First, check if we already have them all.
+  const { searchCustomEmojisByShortcodes, clearEtag } =
+    await import('@/mastodon/features/emoji/database');
+
+  const existingEmojis = await searchCustomEmojisByShortcodes(
+    emojis.map((emoji) => emoji.shortcode),
+  );
+
+  // If there's a mismatch, re-import all custom emojis.
+  if (existingEmojis.length < emojis.length) {
+    await clearEtag('custom');
+    await loadCustomEmoji();
+  }
+}

app/javascript/mastodon/actions/importer/index.js

@@ -1,6 +1,7 @@
 import { createPollFromServerJSON } from 'mastodon/models/poll';
 
 import { importAccounts } from './accounts';
+import { importCustomEmoji } from './emoji';
 import { normalizeStatus } from './normalizer';
 import { importPolls } from './polls';
 
@@ -39,6 +40,10 @@ export function importFetchedAccounts(accounts) {
     if (account.moved) {
       processAccount(account.moved);
     }
+
+    if (account.emojis && account.username === account.acct) {
+      importCustomEmoji(account.emojis);
+    }
   }
 
   accounts.forEach(processAccount);
@@ -80,6 +85,10 @@ export function importFetchedStatuses(statuses, options = {}) {
       if (status.card) {
         status.card.authors.forEach(author => author.account && pushUnique(accounts, author.account));
       }
+
+      if (status.emojis && status.account.username === status.account.acct) {
+        importCustomEmoji(status.emojis);
+      }
     }
 
     statuses.forEach(processStatus);

app/javascript/mastodon/actions/importer/normalizer.js

@@ -2,6 +2,8 @@ import escapeTextContentForBrowser from 'escape-html';
 
 import { expandSpoilers } from '../../initial_state';
 
+import { importCustomEmoji } from './emoji';
+
 const domParser = new DOMParser();
 
 export function searchTextFromRawStatus (status) {
@@ -151,5 +153,9 @@ export function normalizeAnnouncement(announcement) {
 
   normalAnnouncement.contentHtml = normalAnnouncement.content;
 
+  if (normalAnnouncement.emojis) {
+    importCustomEmoji(normalAnnouncement.emojis);
+  }
+
   return normalAnnouncement;
 }

app/javascript/mastodon/features/emoji/database.test.ts

@@ -43,11 +43,26 @@ describe('emoji database', () => {
   describe('putCustomEmojiData', () => {
     test('loads custom emoji into indexedDB', async () => {
       const { db } = await testGet();
-      await putCustomEmojiData([customEmojiFactory()]);
+      await putCustomEmojiData({ emojis: [customEmojiFactory()] });
       await expect(db.get('custom', 'custom')).resolves.toEqual(
         customEmojiFactory(),
       );
     });
+
+    test('clears existing custom emoji if specified', async () => {
+      const { db } = await testGet();
+      await putCustomEmojiData({
+        emojis: [customEmojiFactory({ shortcode: 'emoji1' })],
+      });
+      await putCustomEmojiData({
+        emojis: [customEmojiFactory({ shortcode: 'emoji2' })],
+        clear: true,
+      });
+      await expect(db.get('custom', 'emoji1')).resolves.toBeUndefined();
+      await expect(db.get('custom', 'emoji2')).resolves.toEqual(
+        customEmojiFactory({ shortcode: 'emoji2' }),
+      );
+    });
   });
 
   describe('putLegacyShortcodes', () => {

app/javascript/mastodon/features/emoji/database.ts

@@ -161,11 +161,26 @@ export async function putEmojiData(emojis: UnicodeEmojiData[], locale: Locale) {
   await trx.done;
 }
 
-export async function putCustomEmojiData(emojis: CustomEmojiData[]) {
+export async function putCustomEmojiData({
+  emojis,
+  clear = false,
+}: {
+  emojis: CustomEmojiData[];
+  clear?: boolean;
+}) {
   const db = await loadDB();
   const trx = db.transaction('custom', 'readwrite');
+
+  // When importing from the API, clear everything first.
+  if (clear) {
+    await trx.store.clear();
+    log('Cleared existing custom emojis in database');
+  }
+
   await Promise.all(emojis.map((emoji) => trx.store.put(emoji)));
   await trx.done;
+
+  log('Imported %d custom emojis into database', emojis.length);
 }
 
 export async function putLegacyShortcodes(shortcodes: ShortcodesDataset) {
@@ -188,6 +203,13 @@ export async function putLatestEtag(etag: string, localeString: string) {
   await db.put('etags', etag, locale);
 }
 
+export async function clearEtag(localeString: string) {
+  const locale = toSupportedLocaleOrCustom(localeString);
+  const db = await loadDB();
+  await db.delete('etags', locale);
+  log('Cleared etag for %s', locale);
+}
+
 export async function loadEmojiByHexcode(
   hexcode: string,
   localeString: string,

app/javascript/mastodon/features/emoji/index.ts

@@ -3,7 +3,6 @@ import type { Locale } from 'emojibase';
 import { initialState } from '@/mastodon/initial_state';
 
 import type { EMOJI_DB_NAME_SHORTCODES, EMOJI_TYPE_CUSTOM } from './constants';
-import { importLegacyShortcodes, localeToShortcodesPath } from './loader';
 import { toSupportedLocale } from './locale';
 import type { LocaleOrCustom } from './types';
 import { emojiLogger } from './utils';
@@ -16,48 +15,54 @@ let worker: Worker | null = null;
 
 const log = emojiLogger('index');
 
-const WORKER_TIMEOUT = 1_000; // 1 second
+// This is too short, but better to fallback quickly than wait.
+const WORKER_TIMEOUT = 1_000;
 
 export function initializeEmoji() {
   log('initializing emojis');
+
+  // Create a temp worker, and assign it to the module-level worker once we know it's ready.
+  let tempWorker: Worker | null = null;
   if (!worker && 'Worker' in window) {
     try {
-      worker = new EmojiWorker();
+      tempWorker = new EmojiWorker();
     } catch (err) {
       console.warn('Error creating web worker:', err);
     }
   }
 
-  if (worker) {
-    const timeoutId = setTimeout(() => {
-      log('worker is not ready after timeout');
-      worker = null;
-      void fallbackLoad();
-    }, WORKER_TIMEOUT);
-    worker.addEventListener('message', (event: MessageEvent<string>) => {
-      const { data: message } = event;
-      if (message === 'ready') {
-        log('worker ready, loading data');
-        clearTimeout(timeoutId);
-        messageWorker('custom');
-        messageWorker('shortcodes');
-        void loadEmojiLocale(userLocale);
-      } else {
-        log('got worker message: %s', message);
-      }
-    });
-  } else {
+  if (!tempWorker) {
     void fallbackLoad();
+    return;
   }
+
+  const timeoutId = setTimeout(() => {
+    log('worker is not ready after timeout');
+    void fallbackLoad();
+  }, WORKER_TIMEOUT);
+
+  tempWorker.addEventListener('message', (event: MessageEvent<string>) => {
+    const { data: message } = event;
+
+    worker ??= tempWorker;
+
+    if (message === 'ready') {
+      log('worker ready, loading data');
+      clearTimeout(timeoutId);
+      messageWorker('shortcodes');
+      void loadCustomEmoji();
+      void loadEmojiLocale(userLocale);
+    } else {
+      log('got worker message: %s', message);
+    }
+  });
 }
 
 async function fallbackLoad() {
   log('falling back to main thread for loading');
-  const { importCustomEmojiData } = await import('./loader');
-  const emojis = await importCustomEmojiData();
-  if (emojis) {
-    log('loaded %d custom emojis', emojis.length);
-  }
+
+  await loadCustomEmoji();
+  const { importLegacyShortcodes } = await import('./loader');
   const shortcodes = await importLegacyShortcodes();
   if (shortcodes.length) {
     log('loaded %d legacy shortcodes', shortcodes.length);
@@ -67,11 +72,11 @@ async function fallbackLoad() {
 
 async function loadEmojiLocale(localeString: string) {
   const locale = toSupportedLocale(localeString);
-  const { importEmojiData, localeToEmojiPath: localeToPath } =
+  const { importEmojiData, localeToEmojiPath, localeToShortcodesPath } =
     await import('./loader');
 
   if (worker) {
-    const path = await localeToPath(locale);
+    const path = await localeToEmojiPath(locale);
     const shortcodesPath = await localeToShortcodesPath(locale);
     log('asking worker to load locale %s from %s', locale, path);
     messageWorker(locale, path, shortcodesPath);
@@ -83,6 +88,18 @@ async function loadEmojiLocale(localeString: string) {
   }
 }
 
+export async function loadCustomEmoji() {
+  if (worker) {
+    messageWorker('custom');
+  } else {
+    const { importCustomEmojiData } = await import('./loader');
+    const emojis = await importCustomEmojiData();
+    if (emojis && emojis.length > 0) {
+      log('loaded %d custom emojis', emojis.length);
+    }
+  }
+}
+
 function messageWorker(
   locale: typeof EMOJI_TYPE_CUSTOM | typeof EMOJI_DB_NAME_SHORTCODES,
 ): void;

app/javascript/mastodon/features/emoji/loader.ts

@@ -76,7 +76,7 @@ export async function importCustomEmojiData() {
   if (!emojis) {
     return;
   }
-  await putCustomEmojiData(emojis);
+  await putCustomEmojiData({ emojis, clear: true });
   return emojis;
 }