From 3e45a3142b99c27bb950dd1c4730ed0e5d1f5468 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?marcin=20miko=C5=82ajczak?= Date: Wed, 16 Oct 2024 23:27:12 +0200 Subject: [PATCH] pl-fe: WIP: Lazy load emoji data MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: marcin mikołajczak --- packages/pl-fe/src/features/emoji/data.ts | 52 +-------- packages/pl-fe/src/features/emoji/index.ts | 2 +- packages/pl-fe/src/features/emoji/mapping.ts | 107 +------------------ packages/pl-fe/src/features/emoji/search.ts | 11 +- 4 files changed, 11 insertions(+), 161 deletions(-) diff --git a/packages/pl-fe/src/features/emoji/data.ts b/packages/pl-fe/src/features/emoji/data.ts index db8db115b..6a560ff9c 100644 --- a/packages/pl-fe/src/features/emoji/data.ts +++ b/packages/pl-fe/src/features/emoji/data.ts @@ -1,61 +1,11 @@ import data from '@emoji-mart/data/sets/14/twitter.json'; -interface NativeEmoji { - unified: string; - native: string; - x: number; - y: number; -} - -interface CustomEmoji { - src: string; -} - -interface Emoji { - id: string; - name: string; - keywords: string[]; - skins: T[]; - version?: number; -} - -interface EmojiCategory { - id: string; - emojis: string[]; -} - -interface EmojiMap { - [s: string]: Emoji; -} - -interface EmojiAlias { - [s: string]: string; -} - -interface EmojiSheet { - cols: number; - rows: number; -} - -interface EmojiData { - categories: EmojiCategory[]; - emojis: EmojiMap; - aliases: EmojiAlias; - sheet: EmojiSheet; -} +import type { EmojiData } from './types'; const emojiData = data as EmojiData; const { categories, emojis, aliases, sheet } = emojiData; export { - type NativeEmoji, - type CustomEmoji, - type Emoji, - type EmojiCategory, - type EmojiMap, - type EmojiAlias, - type EmojiSheet, - type EmojiData, categories, emojis, aliases, diff --git a/packages/pl-fe/src/features/emoji/index.ts b/packages/pl-fe/src/features/emoji/index.ts index 1e85fc74f..2527d1cb1 100644 --- a/packages/pl-fe/src/features/emoji/index.ts +++ b/packages/pl-fe/src/features/emoji/index.ts @@ -2,8 +2,8 @@ import split from 'graphemesplit'; import unicodeMapping from './mapping'; +import type { Emoji as EmojiMart, CustomEmoji as EmojiMartCustom } from './types'; import type { CustomEmoji as BaseCustomEmoji } from 'pl-api'; -import type { Emoji as EmojiMart, CustomEmoji as EmojiMartCustom } from 'pl-fe/features/emoji/data'; /* * TODO: Consolate emoji object types diff --git a/packages/pl-fe/src/features/emoji/mapping.ts b/packages/pl-fe/src/features/emoji/mapping.ts index 5d5ba507a..d7fb32c52 100644 --- a/packages/pl-fe/src/features/emoji/mapping.ts +++ b/packages/pl-fe/src/features/emoji/mapping.ts @@ -1,106 +1,3 @@ -import data, { EmojiData } from './data'; +import type { UnicodeMap } from './types'; -const stripLeadingZeros = /^0+/; -interface UnicodeMap { - [s: string]: { - unified: string; - shortcode: string; - }; -} - -/* - * Twemoji strips their hex codes from unicode codepoints to make it look "pretty" - * - leading 0s are removed - * - fe0f is removed unless it has 200d - * - fe0f is NOT removed for 1f441-fe0f-200d-1f5e8-fe0f even though it has a 200d - * - * this is all wrong - */ - -const blacklist = { - '1f441-fe0f-200d-1f5e8-fe0f': true, -}; - -const tweaks = { - '#⃣': ['23-20e3', 'hash'], - '*⃣': ['2a-20e3', 'keycap_star'], - '0⃣': ['30-20e3', 'zero'], - '1⃣': ['31-20e3', 'one'], - '2⃣': ['32-20e3', 'two'], - '3⃣': ['33-20e3', 'three'], - '4⃣': ['34-20e3', 'four'], - '5⃣': ['35-20e3', 'five'], - '6⃣': ['36-20e3', 'six'], - '7⃣': ['37-20e3', 'seven'], - '8⃣': ['38-20e3', 'eight'], - '9⃣': ['39-20e3', 'nine'], - '❤‍🔥': ['2764-fe0f-200d-1f525', 'heart_on_fire'], - '❤‍🩹': ['2764-fe0f-200d-1fa79', 'mending_heart'], - '👁‍🗨️': ['1f441-fe0f-200d-1f5e8-fe0f', 'eye-in-speech-bubble'], - '👁️‍🗨': ['1f441-fe0f-200d-1f5e8-fe0f', 'eye-in-speech-bubble'], - '👁‍🗨': ['1f441-fe0f-200d-1f5e8-fe0f', 'eye-in-speech-bubble'], - '🕵‍♂️': ['1f575-fe0f-200d-2642-fe0f', 'male-detective'], - '🕵️‍♂': ['1f575-fe0f-200d-2642-fe0f', 'male-detective'], - '🕵‍♂': ['1f575-fe0f-200d-2642-fe0f', 'male-detective'], - '🕵‍♀️': ['1f575-fe0f-200d-2640-fe0f', 'female-detective'], - '🕵️‍♀': ['1f575-fe0f-200d-2640-fe0f', 'female-detective'], - '🕵‍♀': ['1f575-fe0f-200d-2640-fe0f', 'female-detective'], - '🏌‍♂️': ['1f3cc-fe0f-200d-2642-fe0f', 'man-golfing'], - '🏌️‍♂': ['1f3cc-fe0f-200d-2642-fe0f', 'man-golfing'], - '🏌‍♂': ['1f3cc-fe0f-200d-2642-fe0f', 'man-golfing'], - '🏌‍♀️': ['1f3cc-fe0f-200d-2640-fe0f', 'woman-golfing'], - '🏌️‍♀': ['1f3cc-fe0f-200d-2640-fe0f', 'woman-golfing'], - '🏌‍♀': ['1f3cc-fe0f-200d-2640-fe0f', 'woman-golfing'], - '⛹‍♂️': ['26f9-fe0f-200d-2642-fe0f', 'man-bouncing-ball'], - '⛹️‍♂': ['26f9-fe0f-200d-2642-fe0f', 'man-bouncing-ball'], - '⛹‍♂': ['26f9-fe0f-200d-2642-fe0f', 'man-bouncing-ball'], - '⛹‍♀️': ['26f9-fe0f-200d-2640-fe0f', 'woman-bouncing-ball'], - '⛹️‍♀': ['26f9-fe0f-200d-2640-fe0f', 'woman-bouncing-ball'], - '⛹‍♀': ['26f9-fe0f-200d-2640-fe0f', 'woman-bouncing-ball'], - '🏋‍♂️': ['1f3cb-fe0f-200d-2642-fe0f', 'man-lifting-weights'], - '🏋️‍♂': ['1f3cb-fe0f-200d-2642-fe0f', 'man-lifting-weights'], - '🏋‍♂': ['1f3cb-fe0f-200d-2642-fe0f', 'man-lifting-weights'], - '🏋‍♀️': ['1f3cb-fe0f-200d-2640-fe0f', 'woman-lifting-weights'], - '🏋️‍♀': ['1f3cb-fe0f-200d-2640-fe0f', 'woman-lifting-weights'], - '🏋‍♀': ['1f3cb-fe0f-200d-2640-fe0f', 'woman-lifting-weights'], - '🏳‍🌈': ['1f3f3-fe0f-200d-1f308', 'rainbow_flag'], - '🏳‍⚧️': ['1f3f3-fe0f-200d-26a7-fe0f', 'transgender_flag'], - '🏳️‍⚧': ['1f3f3-fe0f-200d-26a7-fe0f', 'transgender_flag'], - '🏳‍⚧': ['1f3f3-fe0f-200d-26a7-fe0f', 'transgender_flag'], -}; - -const stripcodes = (unified: string, native: string) => { - const stripped = unified.replace(stripLeadingZeros, ''); - - if (unified.includes('200d') && !(unified in blacklist)) { - return stripped; - } else { - return stripped.replaceAll('-fe0f', ''); - } -}; - -const generateMappings = (data: EmojiData): UnicodeMap => { - const result: UnicodeMap = {}; - const emojis = Object.values(data.emojis ?? {}); - - for (const value of emojis) { - for (const item of value.skins) { - const { unified, native } = item; - const stripped = stripcodes(unified, native); - - result[native] = { unified: stripped, shortcode: value.id }; - } - } - - for (const [native, [unified, shortcode]] of Object.entries(tweaks)) { - const stripped = stripcodes(unified, native); - - result[native] = { unified: stripped, shortcode }; - } - - return result; -}; - -const unicodeMapping = generateMappings(data); - -export { generateMappings, unicodeMapping as default }; +export default import.meta.compileTime('./mapping-compiletime.ts'); diff --git a/packages/pl-fe/src/features/emoji/search.ts b/packages/pl-fe/src/features/emoji/search.ts index 0316f6445..5e34595f4 100644 --- a/packages/pl-fe/src/features/emoji/search.ts +++ b/packages/pl-fe/src/features/emoji/search.ts @@ -1,17 +1,20 @@ import FlexSearch from 'flexsearch'; -import data from './data'; - import type { Emoji } from './index'; +import type { EmojiData } from './types'; import type { CustomEmoji } from 'pl-api'; +let emojis: EmojiData['emojis'] = {}; + +import('./data').then(data => emojis = data.emojis).catch(() => { }); + const index = new FlexSearch.Index({ tokenize: 'full', optimize: true, context: true, }); -const sortedEmojis = Object.entries(data.emojis).sort((a, b) => a[0].localeCompare(b[0])); +const sortedEmojis = Object.entries(emojis).sort((a, b) => a[0].localeCompare(b[0])); for (const [key, emoji] of sortedEmojis) { index.add('n' + key, `${emoji.id} ${emoji.name} ${emoji.keywords.join(' ')}`); } @@ -58,7 +61,7 @@ const search = ( } } - const skins = data.emojis[id.slice(1)]?.skins; + const skins = emojis[id.slice(1)]?.skins; if (skins) { return {