mirror of
https://github.com/mastodon/mastodon.git
synced 2025-10-05 16:42:47 +00:00
adds script to generate locale data for emojis
This commit is contained in:
ChaosExAnima
parent
0663bda64f
commit
d5d360b940
|
|
@ -1,135 +1,84 @@
|
|||
import type {
|
||||
Category,
|
||||
Data as EmojiMartData,
|
||||
SkinVariation,
|
||||
} from 'emoji-mart/dist-es/utils/data';
|
||||
import type { SkinVariation } from 'emoji-mart/dist-es/utils/data';
|
||||
import { joinShortcodes } from 'emojibase';
|
||||
import type {
|
||||
CompactEmoji,
|
||||
Locale,
|
||||
MessagesDataset,
|
||||
ShortcodesDataset,
|
||||
} from 'emojibase';
|
||||
import type { CompactEmoji, Locale, ShortcodesDataset } from 'emojibase';
|
||||
|
||||
async function fetchEmojibaseData(locale: Locale): Promise<CompactEmoji[]> {
|
||||
const [emojiData, shortcodes] = await Promise.all([
|
||||
import(`emojibase-data/${locale}/compact.json`) as Promise<CompactEmoji[]>,
|
||||
import(
|
||||
async function fetchEmojibaseData(
|
||||
locale: Locale,
|
||||
withShortcodes = true,
|
||||
): Promise<CompactEmoji[]> {
|
||||
const { default: emojiData } = (await import(
|
||||
`emojibase-data/${locale}/compact.json`
|
||||
)) as { default: CompactEmoji[] };
|
||||
if (!withShortcodes) {
|
||||
return emojiData;
|
||||
}
|
||||
const shortcodes = (await import(
|
||||
`emojibase-data/${locale}/shortcodes/cldr.json`
|
||||
) as Promise<ShortcodesDataset>,
|
||||
]);
|
||||
)) as ShortcodesDataset;
|
||||
|
||||
return joinShortcodes(emojiData, [shortcodes]);
|
||||
}
|
||||
|
||||
function fetchEmojibaseMessages(locale: Locale): Promise<MessagesDataset> {
|
||||
return import(`emojibase-data/${locale}/messages.json`);
|
||||
}
|
||||
|
||||
export async function fetchEmojiMartData() {
|
||||
const { default: data } = await import('./emoji_data.json');
|
||||
const { default: data } = await import('emoji-datasource/emoji.json');
|
||||
return data;
|
||||
}
|
||||
|
||||
/*
|
||||
{
|
||||
"man-rowing-boat":{
|
||||
"a":"Man Rowing Boat",
|
||||
"b":"1F6A3-200D-2642-FE0F",
|
||||
"f":true,
|
||||
"k":[
|
||||
36,
|
||||
36
|
||||
],
|
||||
"c":"1F6A3-200D-2642",
|
||||
"j":[
|
||||
"person_rowing_boat",
|
||||
"sport",
|
||||
"move"
|
||||
],
|
||||
"skin_variations":{
|
||||
"1F3FB":{
|
||||
"unified":"1F6A3-1F3FB-200D-2642-FE0F",
|
||||
"non_qualified":"1F6A3-1F3FB-200D-2642",
|
||||
"image":"1f6a3-1f3fb-200d-2642-fe0f.png",
|
||||
"sheet_x":36,
|
||||
"sheet_y":37,
|
||||
"added_in":"4.0",
|
||||
"has_img_apple":true,
|
||||
"has_img_google":true,
|
||||
"has_img_twitter":true,
|
||||
"has_img_facebook":true
|
||||
},
|
||||
}
|
||||
export type UnicodeMapping = Record<string, string>;
|
||||
|
||||
export async function emojibaseToUnicodeMapping(
|
||||
locale: Locale,
|
||||
): Promise<UnicodeMapping> {
|
||||
const emojibaseCompactEmojis = await fetchEmojibaseData(locale, false);
|
||||
|
||||
const mapping: UnicodeMapping = {};
|
||||
|
||||
for (const emoji of emojibaseCompactEmojis) {
|
||||
mapping[emoji.hexcode] = emoji.label;
|
||||
}
|
||||
|
||||
return mapping;
|
||||
}
|
||||
|
||||
export interface EmojiMartCompactEmoji {
|
||||
hexcode: string;
|
||||
x: number;
|
||||
y: number;
|
||||
labels?: string[];
|
||||
skins?: Omit<EmojiMartCompactEmoji, 'skins' | 'labels'>[];
|
||||
}
|
||||
*/
|
||||
|
||||
export async function emojibaseToEmojiMart(
|
||||
locale: Locale,
|
||||
): Promise<EmojiMartData> {
|
||||
const emojibaseCompactEmojis = await fetchEmojibaseData(locale);
|
||||
const emojibaseMessages = await fetchEmojibaseMessages(locale);
|
||||
): Promise<EmojiMartCompactEmoji[]> {
|
||||
const emojibaseCompactEmojis = await fetchEmojibaseData(locale, true);
|
||||
const emojiMartData = await fetchEmojiMartData();
|
||||
|
||||
const groupNumToEmojiMap = new Map<number, Set<string>>();
|
||||
const emojis: EmojiMartData['emojis'] = {};
|
||||
const aliases: EmojiMartData['aliases'] = {};
|
||||
const emojibaseMap = new Map<string, CompactEmoji>();
|
||||
for (const emoji of emojibaseCompactEmojis) {
|
||||
const shortcode = emoji.shortcodes?.at(0);
|
||||
if (!shortcode || !emoji.shortcodes) {
|
||||
continue; // Skip emojis without shortcodes
|
||||
emojibaseMap.set(emoji.hexcode, emoji);
|
||||
}
|
||||
|
||||
for (const alias of emoji.shortcodes.slice(1)) {
|
||||
aliases[alias] = shortcode;
|
||||
}
|
||||
|
||||
if (emoji.group) {
|
||||
const existingEmojis =
|
||||
groupNumToEmojiMap.get(emoji.group) ?? new Set<string>();
|
||||
existingEmojis.add(shortcode);
|
||||
groupNumToEmojiMap.set(emoji.group, existingEmojis);
|
||||
}
|
||||
|
||||
emojis[shortcode] = {
|
||||
name: emoji.label,
|
||||
unified: emoji.hexcode,
|
||||
has_img_twitter: true,
|
||||
sheet_x: 0,
|
||||
sheet_y: 0,
|
||||
keywords: emoji.tags,
|
||||
skin_variations: emoji.skins?.reduce(
|
||||
(acc, skin) => ({
|
||||
...acc,
|
||||
[skin.hexcode]: {
|
||||
unified: skin.hexcode,
|
||||
non_qualified: skin.hexcode,
|
||||
image: skin.hexcode,
|
||||
sheet_x: 0,
|
||||
sheet_y: 0,
|
||||
added_in: '1.0', // Placeholder, should be set based on actual version
|
||||
has_img_apple: true,
|
||||
has_img_google: true,
|
||||
has_img_twitter: true,
|
||||
has_img_facebook: true,
|
||||
},
|
||||
}),
|
||||
{} as Record<string, SkinVariation>,
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
const categories: Category[] = emojibaseMessages.groups.map(
|
||||
(group, index) => ({
|
||||
id: group.key,
|
||||
name: group.message,
|
||||
emojis: groupNumToEmojiMap.get(index)?.values().toArray() ?? [],
|
||||
const emojis: EmojiMartCompactEmoji[] = [];
|
||||
for (const emoji of emojiMartData) {
|
||||
const emojibaseEmoji = emojibaseMap.get(emoji.unified);
|
||||
let skins: EmojiMartCompactEmoji['skins'];
|
||||
if (emoji.skin_variations) {
|
||||
skins = Object.values(emoji.skin_variations).map(
|
||||
(skin: SkinVariation) => ({
|
||||
hexcode: skin.unified,
|
||||
x: skin.sheet_x,
|
||||
y: skin.sheet_y,
|
||||
}),
|
||||
);
|
||||
return {
|
||||
compressed: false,
|
||||
categories,
|
||||
emojis,
|
||||
aliases,
|
||||
};
|
||||
}
|
||||
emojis.push({
|
||||
hexcode: emoji.unified,
|
||||
x: emoji.sheet_x,
|
||||
y: emoji.sheet_y,
|
||||
labels: emojibaseEmoji?.shortcodes,
|
||||
skins,
|
||||
});
|
||||
}
|
||||
return emojis;
|
||||
}
|
||||
|
|
|
|||
67
app/javascript/mastodon/features/emoji/generate.ts
Normal file
67
app/javascript/mastodon/features/emoji/generate.ts
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
/* eslint-disable no-console */
|
||||
import fs from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
|
||||
import type { Locale } from 'emojibase';
|
||||
import { SUPPORTED_LOCALES } from 'emojibase';
|
||||
|
||||
import { emojibaseToEmojiMart, emojibaseToUnicodeMapping } from './data';
|
||||
|
||||
const projectRoot = process.cwd();
|
||||
const emojiPath = path.resolve(projectRoot, 'public/emoji');
|
||||
|
||||
async function calculateLocales() {
|
||||
// Get a list of all locales we support.
|
||||
const localeFilesPath = path.resolve(
|
||||
projectRoot,
|
||||
'app/javascript/mastodon/locales',
|
||||
);
|
||||
const localeDir = await fs.readdir(localeFilesPath, 'utf-8');
|
||||
const appLocales: string[] = [];
|
||||
for (const file of localeDir) {
|
||||
if (file.endsWith('.json')) {
|
||||
appLocales.push(path.basename(file, '.json').toLowerCase());
|
||||
}
|
||||
}
|
||||
// Get the intersection of supported locales and app locales and generate emoji JSON files for those.
|
||||
return SUPPORTED_LOCALES.filter((locale) =>
|
||||
appLocales.includes(locale.toLowerCase()),
|
||||
);
|
||||
}
|
||||
|
||||
async function generateAllEmojiJsonFiles() {
|
||||
const emojiLocalePath = path.resolve(emojiPath, 'locales');
|
||||
const emojiLocales = await calculateLocales();
|
||||
await fs.mkdir(emojiLocalePath, { recursive: true }); // Ensure the locales directory exists
|
||||
await Promise.all(
|
||||
emojiLocales.map((locale) => writeEmojiLocaleJson(locale, emojiLocalePath)),
|
||||
);
|
||||
console.log(
|
||||
`Generated emoji JSON files for locales: ${emojiLocales.join(', ')}`,
|
||||
);
|
||||
}
|
||||
|
||||
async function writeEmojiLocaleJson(locale: Locale, dirPath: string) {
|
||||
const mapping = await emojibaseToUnicodeMapping(locale);
|
||||
const filePath = path.join(dirPath, `${locale}.json`);
|
||||
await fs.writeFile(filePath, JSON.stringify(mapping, null, 2), 'utf-8');
|
||||
}
|
||||
|
||||
async function generateEmojiSheetMappingJson() {
|
||||
const emojiMetaPath = path.resolve(emojiPath, 'meta');
|
||||
await fs.mkdir(emojiMetaPath, { recursive: true }); // Ensure the meta directory exists
|
||||
const locales = await calculateLocales();
|
||||
await Promise.all(
|
||||
locales.map((locale) => writeEmojiMappingJson(locale, emojiMetaPath)),
|
||||
);
|
||||
}
|
||||
|
||||
async function writeEmojiMappingJson(locale: Locale, dirPath: string) {
|
||||
const mapping = await emojibaseToEmojiMart(locale);
|
||||
const filePath = path.join(dirPath, `${locale}.json`);
|
||||
await fs.writeFile(filePath, JSON.stringify(mapping, null, 2), 'utf-8');
|
||||
}
|
||||
|
||||
Promise.all([generateAllEmojiJsonFiles(), generateEmojiSheetMappingJson()])
|
||||
.then(() => process.exit())
|
||||
.catch(console.error);
|
||||
|
|
@ -65,6 +65,7 @@
|
|||
"core-js": "^3.30.2",
|
||||
"cross-env": "^7.0.3",
|
||||
"detect-passive-events": "^2.0.3",
|
||||
"emoji-datasource": "15.1.2",
|
||||
"emoji-mart": "npm:emoji-mart-lazyload@latest",
|
||||
"emojibase": "^16.0.0",
|
||||
"emojibase-data": "^16.0.3",
|
||||
|
|
|
|||
|
|
@ -2947,6 +2947,7 @@ __metadata:
|
|||
core-js: "npm:^3.30.2"
|
||||
cross-env: "npm:^7.0.3"
|
||||
detect-passive-events: "npm:^2.0.3"
|
||||
emoji-datasource: "npm:15.1.2"
|
||||
emoji-mart: "npm:emoji-mart-lazyload@latest"
|
||||
emojibase: "npm:^16.0.0"
|
||||
emojibase-data: "npm:^16.0.3"
|
||||
|
|
@ -6783,6 +6784,13 @@ __metadata:
|
|||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"emoji-datasource@npm:15.1.2":
|
||||
version: 15.1.2
|
||||
resolution: "emoji-datasource@npm:15.1.2"
|
||||
checksum: 10c0/32d914ec1c3d1044200eb98bb867ec78d53d5b316bbe6a1780db5490016c82f77277fa2cace0ce0d76f155197265f1d9952c8c7c201ddc59ca4d6d0d8c6ca5b5
|
||||
languageName: node
|
||||
linkType: hard
|
||||
|
||||
"emoji-mart@npm:emoji-mart-lazyload@latest":
|
||||
version: 3.0.1-j
|
||||
resolution: "emoji-mart-lazyload@npm:3.0.1-j"
|
||||
|
|
|
|||
Loading…
Reference in New Issue
Block a user