feat: function to add optional variations to emoji

packages/utils/package.json

@@ -127,6 +127,11 @@
 			"import": "./lib/emoji/parse-test.mjs",
 			"types": "./lib/emoji/parse-test.d.ts"
 		},
+		"./lib/emoji/variations": {
+			"require": "./lib/emoji/variations.cjs",
+			"import": "./lib/emoji/variations.mjs",
+			"types": "./lib/emoji/variations.d.ts"
+		},
 		"./lib/icon-set/convert-info": {
 			"require": "./lib/icon-set/convert-info.cjs",
 			"import": "./lib/icon-set/convert-info.mjs",

packages/utils/src/emoji/cleanup.ts

@@ -1,13 +1,11 @@
-import { convertEmojiSequenceToUTF32, getEmojiCodePoint } from './convert';
+import { getEmojiCodePoint } from './convert';
 import { emojiTones, joinerEmoji, vs16Emoji } from './data';
 
 /**
  * Get emoji sequence from string
  */
 export function getEmojiSequenceFromString(value: string): number[] {
-	return convertEmojiSequenceToUTF32(
-		value.trim().split(/[\s-]/).map(getEmojiCodePoint)
-	);
+	return value.trim().split(/[\s-]/).map(getEmojiCodePoint);
 }
 
 /**

packages/utils/src/emoji/parse-test.ts

@@ -1,4 +1,5 @@
 import { getEmojiSequenceFromString } from './cleanup';
+import { convertEmojiSequenceToUTF32 } from './convert';
 
 // Emoji types
 type EmojiType =
@@ -19,7 +20,7 @@ const allowedTypes: Set<EmojiType> = new Set([
 /**
  * Get all emoji sequences from test file
  *
- * Returns dash-separated hexadecimal codes
+ * Returns all emojis as UTF-32 sequences
  */
 export function parseEmojiTestFile(data: string): number[][] {
 	const emojis: Set<string> = new Set();
@@ -56,6 +57,8 @@ export function parseEmojiTestFile(data: string): number[][] {
 		emojis.add(code);
 	});
 
-	// Return all emojis as sequences
-	return Array.from(emojis).map(getEmojiSequenceFromString);
+	// Return all emojis as sequences, converted to UTF-32
+	return Array.from(emojis).map((item) =>
+		convertEmojiSequenceToUTF32(getEmojiSequenceFromString(item))
+	);
 }

packages/utils/src/emoji/variations.ts

@@ -0,0 +1,73 @@
+import {
+	getEmojiSequenceFromString,
+	joinEmojiSequences,
+	removeEmojiVariations,
+	splitEmojiSequences,
+} from './cleanup';
+import { convertEmojiSequenceToUTF32 } from './convert';
+import { keycapEmoji, vs16Emoji } from './data';
+import { getEmojiSequenceString } from './format';
+
+/**
+ * Add optional variations to emojis
+ *
+ * Also converts list to UTF-32 as needed
+ *
+ * `testData`, returned by parseEmojiTestFile() is used to check which emojis have `FE0F` variations.
+ * If missing or emoji is missing in test data, `FE0F` is added to every single code emoji.
+ */
+export function addOptionalVariations(
+	sequences: number[][],
+	testData?: number[][]
+): number[][] {
+	// Map test data
+	const testDataMap = Object.create(null) as Record<string, string>;
+	testData?.forEach((sequence) => {
+		const convertedSequence = convertEmojiSequenceToUTF32(sequence);
+
+		// Clean up sequence
+		const key = getEmojiSequenceString(
+			removeEmojiVariations(convertedSequence)
+		);
+		if (testDataMap[key]?.length > convertedSequence.length) {
+			// Already got version with more variations
+			return;
+		}
+
+		testDataMap[key] = getEmojiSequenceString(convertedSequence);
+	});
+
+	// Parse all sequences
+	const set: Set<string> = new Set();
+
+	sequences.forEach((sequence) => {
+		const convertedSequence = convertEmojiSequenceToUTF32(sequence);
+		const cleanSequence = removeEmojiVariations(convertedSequence);
+		const mapKey = getEmojiSequenceString(cleanSequence);
+		if (testDataMap[mapKey]) {
+			// Got item from test data
+			set.add(testDataMap[mapKey]);
+			return;
+		}
+
+		// Emoji is missing in test data: add `FE0F` as needed
+		const parts = splitEmojiSequences(convertedSequence).map((part) => {
+			// Check for `FE0F`
+			if (part.indexOf(vs16Emoji) !== -1) {
+				return part;
+			}
+
+			// Check for keycap
+			if (part.length === 2 && part[1] === keycapEmoji) {
+				return [part[0], vs16Emoji, part[1]];
+			}
+
+			// Add `FE0F` to 1 character emojis
+			return part.length === 1 ? [part[0], vs16Emoji] : part;
+		});
+
+		set.add(getEmojiSequenceString(joinEmojiSequences(parts)));
+	});
+
+	return Array.from(set).map(getEmojiSequenceFromString);
+}

packages/utils/tests/emoji-optional-variations-test.ts

@@ -0,0 +1,138 @@
+/* eslint-disable @typescript-eslint/no-non-null-assertion */
+import { readFile, writeFile, unlink } from 'node:fs/promises';
+import { getEmojiSequenceFromString } from '../lib/emoji/cleanup';
+import { getEmojiSequenceString } from '../lib/emoji/format';
+import { parseEmojiTestFile } from '../lib/emoji/parse-test';
+import { addOptionalVariations } from '../lib/emoji/variations';
+
+describe('Optional variations of emoji sequences', () => {
+	it('Variations without data', () => {
+		const sequences = [
+			// simple emoji, twice to check duplicates
+			'1F601',
+			'1F601 FE0F',
+			// 2 simple emojis
+			'1F635 200D 1F4AB',
+			// simple emoji with variation
+			'00A9 FE0F',
+			// keycap with and without variation
+			'0031 FE0F 20E3',
+			'0033 20E3',
+			// complex emojis
+			'1F1E6 1F1F8',
+			'1F3F4 E0067 E0062 E0065 E006E E0067 E007F',
+			// mix of simple and complex, with and without variation
+			'1F9D7 1F3FE 200D 2640 FE0F',
+			'1F9D7 1F3FF 200D 2642 ',
+		].map(getEmojiSequenceFromString);
+		const results = addOptionalVariations(sequences);
+		expect(
+			results.map((sequence) =>
+				getEmojiSequenceString(sequence, {
+					separator: ' ',
+					case: 'upper',
+					format: 'utf-32',
+					add0: true,
+				})
+			)
+		).toEqual([
+			// simple emoji
+			'1F601 FE0F',
+			// 2 simple emojis
+			'1F635 FE0F 200D 1F4AB FE0F',
+			// simple emoji with variation
+			'00A9 FE0F',
+			// keycap with and without variation
+			'0031 FE0F 20E3',
+			'0033 FE0F 20E3',
+			// complex emojis
+			'1F1E6 1F1F8',
+			'1F3F4 E0067 E0062 E0065 E006E E0067 E007F',
+			// mix of simple and complex, with and without variation
+			'1F9D7 1F3FE 200D 2640 FE0F',
+			'1F9D7 1F3FF 200D 2642 FE0F',
+		]);
+	});
+
+	it('Variations with data', async () => {
+		// Fetch emojis, cache it
+		const version = '15.0';
+		const source = `tests/fixtures/download-emoji-${version}.txt`;
+
+		let data: string | undefined;
+		try {
+			data = await readFile(source, 'utf8');
+		} catch {
+			//
+		}
+
+		if (!data) {
+			data = (
+				await (
+					await fetch(
+						`https://unicode.org/Public/emoji/${version}/emoji-test.txt`
+					)
+				).text()
+			).toString();
+			await writeFile(source, data, 'utf8');
+		}
+
+		// Test content, unlink cache on failure
+		if (data.indexOf(`# Version: ${version}`) === -1) {
+			try {
+				await unlink(source);
+			} catch {
+				//
+			}
+			return;
+		}
+		const testData = parseEmojiTestFile(data);
+
+		const sequences = [
+			// emoji without variation in test file
+			'1F601',
+			'1F635 200D 1F4AB',
+			// emojis without variations in test file, but variations in source
+			'1F60D FE0F',
+			// emoji that has variation in test file
+			'263A',
+			// keycap
+			'0030 20E3',
+			'0034 FE0F 20E3',
+			// complex emoji, exists in file
+			'1F9D1 1F3FE 200D 2764 200D 1F9D1 1F3FB',
+			// simple emoji, not in test file
+			'1234',
+			// fake keycap, not in test file
+			'2345 20E3 200D 1235',
+		].map(getEmojiSequenceFromString);
+		const results = addOptionalVariations(sequences, testData);
+		expect(
+			results.map((sequence) =>
+				getEmojiSequenceString(sequence, {
+					separator: ' ',
+					case: 'upper',
+					format: 'utf-32',
+					add0: true,
+				})
+			)
+		).toEqual([
+			// emoji without variation in test file
+			'1F601',
+			'1F635 200D 1F4AB',
+			// emojis without variations in test file, but variations in source
+			'1F60D',
+			// emoji that has variation in test file
+			'263A FE0F',
+			// keycap
+			'0030 FE0F 20E3',
+			'0034 FE0F 20E3',
+			// complex emoji, exists in file
+			'1F9D1 1F3FE 200D 2764 FE0F 200D 1F9D1 1F3FB',
+			// simple emoji, not in test file
+			'1234 FE0F',
+			// fake keycap, not in test file
+			'2345 FE0F 20E3 200D 1235 FE0F',
+		]);
+	});
+});