diff --git a/app/javascript/mastodon/features/emoji/data.ts b/app/javascript/mastodon/features/emoji/data.ts index 2d1890a038f..af3affce9b1 100644 --- a/app/javascript/mastodon/features/emoji/data.ts +++ b/app/javascript/mastodon/features/emoji/data.ts @@ -1,135 +1,84 @@ -import type { - Category, - Data as EmojiMartData, - SkinVariation, -} from 'emoji-mart/dist-es/utils/data'; +import type { SkinVariation } from 'emoji-mart/dist-es/utils/data'; import { joinShortcodes } from 'emojibase'; -import type { - CompactEmoji, - Locale, - MessagesDataset, - ShortcodesDataset, -} from 'emojibase'; +import type { CompactEmoji, Locale, ShortcodesDataset } from 'emojibase'; -async function fetchEmojibaseData(locale: Locale): Promise { - const [emojiData, shortcodes] = await Promise.all([ - import(`emojibase-data/${locale}/compact.json`) as Promise, - import( - `emojibase-data/${locale}/shortcodes/cldr.json` - ) as Promise, - ]); +async function fetchEmojibaseData( + locale: Locale, + withShortcodes = true, +): Promise { + const { default: emojiData } = (await import( + `emojibase-data/${locale}/compact.json` + )) as { default: CompactEmoji[] }; + if (!withShortcodes) { + return emojiData; + } + const shortcodes = (await import( + `emojibase-data/${locale}/shortcodes/cldr.json` + )) as ShortcodesDataset; return joinShortcodes(emojiData, [shortcodes]); } -function fetchEmojibaseMessages(locale: Locale): Promise { - return import(`emojibase-data/${locale}/messages.json`); -} - export async function fetchEmojiMartData() { - const { default: data } = await import('./emoji_data.json'); + const { default: data } = await import('emoji-datasource/emoji.json'); return data; } -/* -{ - "man-rowing-boat":{ - "a":"Man Rowing Boat", - "b":"1F6A3-200D-2642-FE0F", - "f":true, - "k":[ - 36, - 36 - ], - "c":"1F6A3-200D-2642", - "j":[ - "person_rowing_boat", - "sport", - "move" - ], - "skin_variations":{ - "1F3FB":{ - "unified":"1F6A3-1F3FB-200D-2642-FE0F", - "non_qualified":"1F6A3-1F3FB-200D-2642", - "image":"1f6a3-1f3fb-200d-2642-fe0f.png", - "sheet_x":36, - "sheet_y":37, - "added_in":"4.0", - "has_img_apple":true, - "has_img_google":true, - "has_img_twitter":true, - "has_img_facebook":true - }, - } - } +export type UnicodeMapping = Record; + +export async function emojibaseToUnicodeMapping( + locale: Locale, +): Promise { + const emojibaseCompactEmojis = await fetchEmojibaseData(locale, false); + + const mapping: UnicodeMapping = {}; + + for (const emoji of emojibaseCompactEmojis) { + mapping[emoji.hexcode] = emoji.label; + } + + return mapping; +} + +export interface EmojiMartCompactEmoji { + hexcode: string; + x: number; + y: number; + labels?: string[]; + skins?: Omit[]; } -*/ export async function emojibaseToEmojiMart( locale: Locale, -): Promise { - const emojibaseCompactEmojis = await fetchEmojibaseData(locale); - const emojibaseMessages = await fetchEmojibaseMessages(locale); +): Promise { + const emojibaseCompactEmojis = await fetchEmojibaseData(locale, true); + const emojiMartData = await fetchEmojiMartData(); - const groupNumToEmojiMap = new Map>(); - const emojis: EmojiMartData['emojis'] = {}; - const aliases: EmojiMartData['aliases'] = {}; + const emojibaseMap = new Map(); for (const emoji of emojibaseCompactEmojis) { - const shortcode = emoji.shortcodes?.at(0); - if (!shortcode || !emoji.shortcodes) { - continue; // Skip emojis without shortcodes - } - - for (const alias of emoji.shortcodes.slice(1)) { - aliases[alias] = shortcode; - } - - if (emoji.group) { - const existingEmojis = - groupNumToEmojiMap.get(emoji.group) ?? new Set(); - existingEmojis.add(shortcode); - groupNumToEmojiMap.set(emoji.group, existingEmojis); - } - - emojis[shortcode] = { - name: emoji.label, - unified: emoji.hexcode, - has_img_twitter: true, - sheet_x: 0, - sheet_y: 0, - keywords: emoji.tags, - skin_variations: emoji.skins?.reduce( - (acc, skin) => ({ - ...acc, - [skin.hexcode]: { - unified: skin.hexcode, - non_qualified: skin.hexcode, - image: skin.hexcode, - sheet_x: 0, - sheet_y: 0, - added_in: '1.0', // Placeholder, should be set based on actual version - has_img_apple: true, - has_img_google: true, - has_img_twitter: true, - has_img_facebook: true, - }, - }), - {} as Record, - ), - }; + emojibaseMap.set(emoji.hexcode, emoji); } - const categories: Category[] = emojibaseMessages.groups.map( - (group, index) => ({ - id: group.key, - name: group.message, - emojis: groupNumToEmojiMap.get(index)?.values().toArray() ?? [], - }), - ); - return { - compressed: false, - categories, - emojis, - aliases, - }; + const emojis: EmojiMartCompactEmoji[] = []; + for (const emoji of emojiMartData) { + const emojibaseEmoji = emojibaseMap.get(emoji.unified); + let skins: EmojiMartCompactEmoji['skins']; + if (emoji.skin_variations) { + skins = Object.values(emoji.skin_variations).map( + (skin: SkinVariation) => ({ + hexcode: skin.unified, + x: skin.sheet_x, + y: skin.sheet_y, + }), + ); + } + emojis.push({ + hexcode: emoji.unified, + x: emoji.sheet_x, + y: emoji.sheet_y, + labels: emojibaseEmoji?.shortcodes, + skins, + }); + } + return emojis; } diff --git a/app/javascript/mastodon/features/emoji/generate.ts b/app/javascript/mastodon/features/emoji/generate.ts new file mode 100644 index 00000000000..a21bc1c9550 --- /dev/null +++ b/app/javascript/mastodon/features/emoji/generate.ts @@ -0,0 +1,67 @@ +/* eslint-disable no-console */ +import fs from 'node:fs/promises'; +import path from 'node:path'; + +import type { Locale } from 'emojibase'; +import { SUPPORTED_LOCALES } from 'emojibase'; + +import { emojibaseToEmojiMart, emojibaseToUnicodeMapping } from './data'; + +const projectRoot = process.cwd(); +const emojiPath = path.resolve(projectRoot, 'public/emoji'); + +async function calculateLocales() { + // Get a list of all locales we support. + const localeFilesPath = path.resolve( + projectRoot, + 'app/javascript/mastodon/locales', + ); + const localeDir = await fs.readdir(localeFilesPath, 'utf-8'); + const appLocales: string[] = []; + for (const file of localeDir) { + if (file.endsWith('.json')) { + appLocales.push(path.basename(file, '.json').toLowerCase()); + } + } + // Get the intersection of supported locales and app locales and generate emoji JSON files for those. + return SUPPORTED_LOCALES.filter((locale) => + appLocales.includes(locale.toLowerCase()), + ); +} + +async function generateAllEmojiJsonFiles() { + const emojiLocalePath = path.resolve(emojiPath, 'locales'); + const emojiLocales = await calculateLocales(); + await fs.mkdir(emojiLocalePath, { recursive: true }); // Ensure the locales directory exists + await Promise.all( + emojiLocales.map((locale) => writeEmojiLocaleJson(locale, emojiLocalePath)), + ); + console.log( + `Generated emoji JSON files for locales: ${emojiLocales.join(', ')}`, + ); +} + +async function writeEmojiLocaleJson(locale: Locale, dirPath: string) { + const mapping = await emojibaseToUnicodeMapping(locale); + const filePath = path.join(dirPath, `${locale}.json`); + await fs.writeFile(filePath, JSON.stringify(mapping, null, 2), 'utf-8'); +} + +async function generateEmojiSheetMappingJson() { + const emojiMetaPath = path.resolve(emojiPath, 'meta'); + await fs.mkdir(emojiMetaPath, { recursive: true }); // Ensure the meta directory exists + const locales = await calculateLocales(); + await Promise.all( + locales.map((locale) => writeEmojiMappingJson(locale, emojiMetaPath)), + ); +} + +async function writeEmojiMappingJson(locale: Locale, dirPath: string) { + const mapping = await emojibaseToEmojiMart(locale); + const filePath = path.join(dirPath, `${locale}.json`); + await fs.writeFile(filePath, JSON.stringify(mapping, null, 2), 'utf-8'); +} + +Promise.all([generateAllEmojiJsonFiles(), generateEmojiSheetMappingJson()]) + .then(() => process.exit()) + .catch(console.error); diff --git a/package.json b/package.json index 9e7bd25c2d7..364f57a78cb 100644 --- a/package.json +++ b/package.json @@ -65,6 +65,7 @@ "core-js": "^3.30.2", "cross-env": "^7.0.3", "detect-passive-events": "^2.0.3", + "emoji-datasource": "15.1.2", "emoji-mart": "npm:emoji-mart-lazyload@latest", "emojibase": "^16.0.0", "emojibase-data": "^16.0.3", diff --git a/yarn.lock b/yarn.lock index 8447f19135a..16731e5d5a4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2947,6 +2947,7 @@ __metadata: core-js: "npm:^3.30.2" cross-env: "npm:^7.0.3" detect-passive-events: "npm:^2.0.3" + emoji-datasource: "npm:15.1.2" emoji-mart: "npm:emoji-mart-lazyload@latest" emojibase: "npm:^16.0.0" emojibase-data: "npm:^16.0.3" @@ -6783,6 +6784,13 @@ __metadata: languageName: node linkType: hard +"emoji-datasource@npm:15.1.2": + version: 15.1.2 + resolution: "emoji-datasource@npm:15.1.2" + checksum: 10c0/32d914ec1c3d1044200eb98bb867ec78d53d5b316bbe6a1780db5490016c82f77277fa2cace0ce0d76f155197265f1d9952c8c7c201ddc59ca4d6d0d8c6ca5b5 + languageName: node + linkType: hard + "emoji-mart@npm:emoji-mart-lazyload@latest": version: 3.0.1-j resolution: "emoji-mart-lazyload@npm:3.0.1-j"