adds script to generate locale data for emojis

app/javascript/mastodon/features/emoji/data.ts

@@ -1,135 +1,84 @@
-import type {
+import type { SkinVariation } from 'emoji-mart/dist-es/utils/data';
-  Category,
-  Data as EmojiMartData,
-  SkinVariation,
-} from 'emoji-mart/dist-es/utils/data';
 import { joinShortcodes } from 'emojibase';
-import type {
+import type { CompactEmoji, Locale, ShortcodesDataset } from 'emojibase';
-  CompactEmoji,
-  Locale,
-  MessagesDataset,
-  ShortcodesDataset,
-} from 'emojibase';
 
-async function fetchEmojibaseData(locale: Locale): Promise<CompactEmoji[]> {
+async function fetchEmojibaseData(
-  const [emojiData, shortcodes] = await Promise.all([
+  locale: Locale,
-    import(`emojibase-data/${locale}/compact.json`) as Promise<CompactEmoji[]>,
+  withShortcodes = true,
-    import(
+): Promise<CompactEmoji[]> {
+  const { default: emojiData } = (await import(
+    `emojibase-data/${locale}/compact.json`
+  )) as { default: CompactEmoji[] };
+  if (!withShortcodes) {
+    return emojiData;
+  }
+  const shortcodes = (await import(
     `emojibase-data/${locale}/shortcodes/cldr.json`
-    ) as Promise<ShortcodesDataset>,
+  )) as ShortcodesDataset;
-  ]);
 
   return joinShortcodes(emojiData, [shortcodes]);
 }
 
-function fetchEmojibaseMessages(locale: Locale): Promise<MessagesDataset> {
-  return import(`emojibase-data/${locale}/messages.json`);
-}
-
 export async function fetchEmojiMartData() {
-  const { default: data } = await import('./emoji_data.json');
+  const { default: data } = await import('emoji-datasource/emoji.json');
   return data;
 }
 
-/*
+export type UnicodeMapping = Record<string, string>;
-{
+
-	"man-rowing-boat":{
+export async function emojibaseToUnicodeMapping(
-		"a":"Man Rowing Boat",
+  locale: Locale,
-		"b":"1F6A3-200D-2642-FE0F",
+): Promise<UnicodeMapping> {
-		"f":true,
+  const emojibaseCompactEmojis = await fetchEmojibaseData(locale, false);
-		"k":[
+
-			36,
+  const mapping: UnicodeMapping = {};
-			36
+
-		],
+  for (const emoji of emojibaseCompactEmojis) {
-		"c":"1F6A3-200D-2642",
+    mapping[emoji.hexcode] = emoji.label;
-		"j":[
-			"person_rowing_boat",
-			"sport",
-			"move"
-		],
-		"skin_variations":{
-			"1F3FB":{
-				"unified":"1F6A3-1F3FB-200D-2642-FE0F",
-				"non_qualified":"1F6A3-1F3FB-200D-2642",
-				"image":"1f6a3-1f3fb-200d-2642-fe0f.png",
-				"sheet_x":36,
-				"sheet_y":37,
-				"added_in":"4.0",
-				"has_img_apple":true,
-				"has_img_google":true,
-				"has_img_twitter":true,
-				"has_img_facebook":true
-			},
   }
+
+  return mapping;
 }
+
+export interface EmojiMartCompactEmoji {
+  hexcode: string;
+  x: number;
+  y: number;
+  labels?: string[];
+  skins?: Omit<EmojiMartCompactEmoji, 'skins' | 'labels'>[];
 }
-*/
 
 export async function emojibaseToEmojiMart(
   locale: Locale,
-): Promise<EmojiMartData> {
+): Promise<EmojiMartCompactEmoji[]> {
-  const emojibaseCompactEmojis = await fetchEmojibaseData(locale);
+  const emojibaseCompactEmojis = await fetchEmojibaseData(locale, true);
-  const emojibaseMessages = await fetchEmojibaseMessages(locale);
+  const emojiMartData = await fetchEmojiMartData();
 
-  const groupNumToEmojiMap = new Map<number, Set<string>>();
+  const emojibaseMap = new Map<string, CompactEmoji>();
-  const emojis: EmojiMartData['emojis'] = {};
-  const aliases: EmojiMartData['aliases'] = {};
   for (const emoji of emojibaseCompactEmojis) {
-    const shortcode = emoji.shortcodes?.at(0);
+    emojibaseMap.set(emoji.hexcode, emoji);
-    if (!shortcode || !emoji.shortcodes) {
-      continue; // Skip emojis without shortcodes
   }
 
-    for (const alias of emoji.shortcodes.slice(1)) {
+  const emojis: EmojiMartCompactEmoji[] = [];
-      aliases[alias] = shortcode;
+  for (const emoji of emojiMartData) {
-    }
+    const emojibaseEmoji = emojibaseMap.get(emoji.unified);
-
+    let skins: EmojiMartCompactEmoji['skins'];
-    if (emoji.group) {
+    if (emoji.skin_variations) {
-      const existingEmojis =
+      skins = Object.values(emoji.skin_variations).map(
-        groupNumToEmojiMap.get(emoji.group) ?? new Set<string>();
+        (skin: SkinVariation) => ({
-      existingEmojis.add(shortcode);
+          hexcode: skin.unified,
-      groupNumToEmojiMap.set(emoji.group, existingEmojis);
+          x: skin.sheet_x,
-    }
+          y: skin.sheet_y,
-
-    emojis[shortcode] = {
-      name: emoji.label,
-      unified: emoji.hexcode,
-      has_img_twitter: true,
-      sheet_x: 0,
-      sheet_y: 0,
-      keywords: emoji.tags,
-      skin_variations: emoji.skins?.reduce(
-        (acc, skin) => ({
-          ...acc,
-          [skin.hexcode]: {
-            unified: skin.hexcode,
-            non_qualified: skin.hexcode,
-            image: skin.hexcode,
-            sheet_x: 0,
-            sheet_y: 0,
-            added_in: '1.0', // Placeholder, should be set based on actual version
-            has_img_apple: true,
-            has_img_google: true,
-            has_img_twitter: true,
-            has_img_facebook: true,
-          },
-        }),
-        {} as Record<string, SkinVariation>,
-      ),
-    };
-  }
-
-  const categories: Category[] = emojibaseMessages.groups.map(
-    (group, index) => ({
-      id: group.key,
-      name: group.message,
-      emojis: groupNumToEmojiMap.get(index)?.values().toArray() ?? [],
         }),
       );
-  return {
+    }
-    compressed: false,
+    emojis.push({
-    categories,
+      hexcode: emoji.unified,
-    emojis,
+      x: emoji.sheet_x,
-    aliases,
+      y: emoji.sheet_y,
-  };
+      labels: emojibaseEmoji?.shortcodes,
+      skins,
+    });
+  }
+  return emojis;
 }

app/javascript/mastodon/features/emoji/generate.ts

@@ -0,0 +1,67 @@
+/* eslint-disable no-console */
+import fs from 'node:fs/promises';
+import path from 'node:path';
+
+import type { Locale } from 'emojibase';
+import { SUPPORTED_LOCALES } from 'emojibase';
+
+import { emojibaseToEmojiMart, emojibaseToUnicodeMapping } from './data';
+
+const projectRoot = process.cwd();
+const emojiPath = path.resolve(projectRoot, 'public/emoji');
+
+async function calculateLocales() {
+  // Get a list of all locales we support.
+  const localeFilesPath = path.resolve(
+    projectRoot,
+    'app/javascript/mastodon/locales',
+  );
+  const localeDir = await fs.readdir(localeFilesPath, 'utf-8');
+  const appLocales: string[] = [];
+  for (const file of localeDir) {
+    if (file.endsWith('.json')) {
+      appLocales.push(path.basename(file, '.json').toLowerCase());
+    }
+  }
+  // Get the intersection of supported locales and app locales and generate emoji JSON files for those.
+  return SUPPORTED_LOCALES.filter((locale) =>
+    appLocales.includes(locale.toLowerCase()),
+  );
+}
+
+async function generateAllEmojiJsonFiles() {
+  const emojiLocalePath = path.resolve(emojiPath, 'locales');
+  const emojiLocales = await calculateLocales();
+  await fs.mkdir(emojiLocalePath, { recursive: true }); // Ensure the locales directory exists
+  await Promise.all(
+    emojiLocales.map((locale) => writeEmojiLocaleJson(locale, emojiLocalePath)),
+  );
+  console.log(
+    `Generated emoji JSON files for locales: ${emojiLocales.join(', ')}`,
+  );
+}
+
+async function writeEmojiLocaleJson(locale: Locale, dirPath: string) {
+  const mapping = await emojibaseToUnicodeMapping(locale);
+  const filePath = path.join(dirPath, `${locale}.json`);
+  await fs.writeFile(filePath, JSON.stringify(mapping, null, 2), 'utf-8');
+}
+
+async function generateEmojiSheetMappingJson() {
+  const emojiMetaPath = path.resolve(emojiPath, 'meta');
+  await fs.mkdir(emojiMetaPath, { recursive: true }); // Ensure the meta directory exists
+  const locales = await calculateLocales();
+  await Promise.all(
+    locales.map((locale) => writeEmojiMappingJson(locale, emojiMetaPath)),
+  );
+}
+
+async function writeEmojiMappingJson(locale: Locale, dirPath: string) {
+  const mapping = await emojibaseToEmojiMart(locale);
+  const filePath = path.join(dirPath, `${locale}.json`);
+  await fs.writeFile(filePath, JSON.stringify(mapping, null, 2), 'utf-8');
+}
+
+Promise.all([generateAllEmojiJsonFiles(), generateEmojiSheetMappingJson()])
+  .then(() => process.exit())
+  .catch(console.error);

package.json

@@ -65,6 +65,7 @@
     "core-js": "^3.30.2",
     "cross-env": "^7.0.3",
     "detect-passive-events": "^2.0.3",
+    "emoji-datasource": "15.1.2",
     "emoji-mart": "npm:emoji-mart-lazyload@latest",
     "emojibase": "^16.0.0",
     "emojibase-data": "^16.0.3",

yarn.lock

@@ -2947,6 +2947,7 @@ __metadata:
     core-js: "npm:^3.30.2"
     cross-env: "npm:^7.0.3"
     detect-passive-events: "npm:^2.0.3"
+    emoji-datasource: "npm:15.1.2"
     emoji-mart: "npm:emoji-mart-lazyload@latest"
     emojibase: "npm:^16.0.0"
     emojibase-data: "npm:^16.0.3"
@@ -6783,6 +6784,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"emoji-datasource@npm:15.1.2":
+  version: 15.1.2
+  resolution: "emoji-datasource@npm:15.1.2"
+  checksum: 10c0/32d914ec1c3d1044200eb98bb867ec78d53d5b316bbe6a1780db5490016c82f77277fa2cace0ce0d76f155197265f1d9952c8c7c201ddc59ca4d6d0d8c6ca5b5
+  languageName: node
+  linkType: hard
+
 "emoji-mart@npm:emoji-mart-lazyload@latest":
   version: 3.0.1-j
   resolution: "emoji-mart-lazyload@npm:3.0.1-j"