Build emoji files and shortcodes map in a plugin (#2789)

* Build emoji files and shortcodes map in a plugin

* update comment

.eslintrc.cjs

@@ -72,6 +72,9 @@ module.exports = {
       'URL',
       'CSS.escape',
     ],
+    'import/core-modules': [
+      'virtual:emoji-shortcodes',
+    ],
   },
 
   overrides: [

package.json

@@ -134,7 +134,6 @@
     "dev": "vite",
     "lint": "eslint --cache . && stylelint --cache src/**/*.css",
     "postpublish": "npm publish -w ./npm",
-    "prepare": "node tasks/emoji.mjs",
     "prerelease": "npm run clean && npm test && npm run prod",
     "prod": "vite build",
     "serve": "u-wave-web",

src/hooks/useEmotes.ts

@@ -1,7 +1,7 @@
 import { useMemo } from 'react';
 import useSWR from 'swr';
+import defaultEmoji from 'virtual:emoji-shortcodes';
 import { useSelector } from './useRedux';
-import defaultEmoji from '../utils/emojiShortcodes';
 import uwFetch, { ListResponse } from '../utils/fetch';
 
 type ServerEmote = {

src/selectors/configSelectors.js

@@ -1,5 +1,5 @@
 import { createSelector } from 'reselect';
-import defaultEmoji from '../utils/emojiShortcodes';
+import defaultEmoji from 'virtual:emoji-shortcodes';
 
 /** @param {import('../redux/configureStore').StoreState} state */
 export const configSelector = (state) => state.config;

src/types.d.ts

@@ -13,6 +13,11 @@ declare module '*.yaml' {
   export default data;
 }
 
+declare module 'virtual:emoji-shortcodes' {
+  const emojiShortcodes: Record<string, string>;
+  export default emojiShortcodes;
+}
+
 declare module '@u-wave/react-translate' {
   import { Translator } from '@u-wave/translate';
 

tasks/emoji.mjs

@@ -1,73 +1,107 @@
-/**
- * Create emoji image files in public/static/emoji.
- * vite will copy these to the /static/emoji folder in the build output.
- * TODO this could maybe be a webpack plugin? we need to be able to generate
- * a file with shortcode to filename mappings that we can import in the web client code.
- */
-import crypto from 'crypto';
-import fs from 'fs/promises';
+import crypto from 'node:crypto';
+import fs from 'node:fs/promises';
+import { fileURLToPath } from 'node:url';
+import assert from 'node:assert';
 import pMap from 'p-map';
+import serveStatic from 'serve-static';
 
 const emojibaseShortcodesPath = new URL('../node_modules/emojibase-data/en/shortcodes/emojibase.json', import.meta.url);
-const emojibaseShortcodes = JSON.parse(await fs.readFile(emojibaseShortcodesPath, { encoding: 'utf8' }));
-
 const joypixelShortcodesPath = new URL('../node_modules/emojibase-data/en/shortcodes/joypixels.json', import.meta.url);
-const joypixelShortcodes = JSON.parse(await fs.readFile(joypixelShortcodesPath, { encoding: 'utf8' }));
-
 const twemojiDir = new URL('../node_modules/twemoji-emojis/vendor/svg/', import.meta.url);
-const twemojis = await fs.readdir(twemojiDir);
-const twemojiNames = twemojis.map((basename) => basename.replace(/\.\w+$/, ''));
 
-const outputDir = new URL('../public/static/emoji/', import.meta.url);
+/**
+ * Collect twemoji images, determine their shortcodes, build a map of shortcode to file names.
+ *
+ * Images are output in the build.
+ * Joypixels shortcodes are used if available: this is for backwards compatibility. Emojibase shortcodes
+ * are used for emoji that do not have a Joypixel shortcode.
+ * The mapping of shortcodes to file names can be imported by JS as `virtual:emoji-shortcodes`.
+ */
+export default function emojiPlugin () {
+  let shortcodeToOutName;
+  const filesToEmit = new Map();
 
-const shortcodes = {};
-function appendShortcode(hex, shortcode) {
-  const imageName = `${hex.toLowerCase()}.svg`
-  if (Array.isArray(shortcode)) {
-    for (const c of shortcode) {
-      shortcodes[c] = imageName;
-    }
-  } else {
-    shortcodes[shortcode] = imageName;
-  }
-}
+  return {
+    name: 'emoji',
+    async configureServer (server) {
+      server.middlewares.use(serveStatic(fileURLToPath(twemojiDir)));
+    },
+    async buildStart () {
+      const emojibaseShortcodes = JSON.parse(await fs.readFile(emojibaseShortcodesPath, { encoding: 'utf8' }));
+      const joypixelShortcodes = JSON.parse(await fs.readFile(joypixelShortcodesPath, { encoding: 'utf8' }));
 
-// Not all emoji have a joypixel shortcode. Track which ones are left over so
-// we can use a different shortcode for those.
-const remainingTwemoji = new Set(twemojiNames);
-for (const [hex, shortcode] of Object.entries(joypixelShortcodes)) {
-  if (!twemojiNames.includes(hex.toLowerCase())) {
-    continue;
-  }
+      const twemojis = await fs.readdir(twemojiDir);
+      const twemojiNames = twemojis.map((basename) => basename.replace(/\.\w+$/, ''));
 
-  appendShortcode(hex, shortcode);
-  remainingTwemoji.delete(hex.toLowerCase());
-}
+      const shortcodes = {};
+      function appendShortcode(hex, shortcode) {
+        const imageName = `${hex.toLowerCase()}.svg`
+        if (Array.isArray(shortcode)) {
+          for (const c of shortcode) {
+            shortcodes[c] = imageName;
+          }
+        } else {
+          shortcodes[shortcode] = imageName;
+        }
+      }
 
-for (const hex of remainingTwemoji) {
-  const shortcode = emojibaseShortcodes[hex.toUpperCase()];
-  if (!shortcode) {
-    continue;
-  }
+      // Not all emoji have a joypixel shortcode. Track which ones are left over so
+      // we can use a different shortcode for those.
+      const remainingTwemoji = new Set(twemojiNames);
+      for (const [hex, shortcode] of Object.entries(joypixelShortcodes)) {
+        if (!twemojiNames.includes(hex.toLowerCase())) {
+          continue;
+        }
 
-  appendShortcode(hex, shortcode);
-}
+        appendShortcode(hex, shortcode);
+        remainingTwemoji.delete(hex.toLowerCase());
+      }
 
-console.log('generating', Object.keys(shortcodes).length, 'emoji...');
+      for (const hex of remainingTwemoji) {
+        const shortcode = emojibaseShortcodes[hex.toUpperCase()];
+        if (!shortcode) {
+          continue;
+        }
 
-await fs.rm(outputDir, { force: true, recursive: true });
-await fs.mkdir(outputDir, { recursive: true });
-const shortcodeHashes = Object.fromEntries(
-  await pMap(Object.entries(shortcodes), async ([shortcode, filename]) => {
-    const bytes = await fs.readFile(new URL(filename, twemojiDir));
-    const hash = crypto.createHash('sha1').update(bytes).digest('hex').slice(0, 7);
-    const outName = `${hash}.svg`;
-    await fs.writeFile(new URL(outName, outputDir), bytes);
-    return [shortcode, outName];
-  }),
-);
-await fs.writeFile(new URL('../src/utils/emojiShortcodes.js', import.meta.url), `
-// GENERATED FILE: run \`npm run emoji\`
-/* eslint-disable */
-export default JSON.parse(${JSON.stringify(JSON.stringify(shortcodeHashes))});
-`.trim());
+        appendShortcode(hex, shortcode);
+      }
+
+      if (this.meta.watchMode) {
+        shortcodeToOutName = Object.entries(shortcodes);
+      } else {
+        shortcodeToOutName = await pMap(Object.entries(shortcodes), async ([shortcode, filename]) => {
+          const bytes = await fs.readFile(new URL(filename, twemojiDir));
+          const hash = crypto.createHash('sha1').update(bytes).digest('hex').slice(0, 7);
+          const outName = `${hash}.svg`;
+          if (filesToEmit.has(outName)) {
+            assert(filesToEmit.get(outName).equals(bytes));
+          } else {
+            filesToEmit.set(outName, bytes);
+          }
+          return [shortcode, outName];
+        });
+      }
+    },
+    async generateBundle () {
+      for (const [outName, bytes] of filesToEmit) {
+        this.emitFile({
+          type: 'asset',
+          fileName: `static/emoji/${outName}`,
+          source: bytes,
+        });
+      }
+    },
+    async resolveId (id) {
+      if (id === 'virtual:emoji-shortcodes') {
+        return '\0virtual:emoji-shortcodes'
+      }
+    },
+    async load (id) {
+      if (id === '\0virtual:emoji-shortcodes') {
+        return `export default JSON.parse(${
+          JSON.stringify(JSON.stringify(Object.fromEntries(shortcodeToOutName)))
+        })`;
+      }
+    },
+  }
+}

vite.config.mjs

@@ -2,6 +2,7 @@ import { readFile, writeFile } from 'node:fs/promises';
 import { defineConfig, splitVendorChunkPlugin } from 'vite';
 import react from '@vitejs/plugin-react';
 import yaml from '@rollup/plugin-yaml';
+import emoji from './tasks/emoji.mjs';
 import prerender from './tasks/prerender.mjs';
 
 const inputPkg = new URL('./package.json', import.meta.url);
@@ -52,6 +53,7 @@ export default defineConfig({
     prerender({ file: 'index.html', source: 'src/index.tsx' }),
     prerender({ file: 'password-reset.html', source: 'src/password-reset/index.jsx' }),
     prerender({ file: 'privacy.html', source: 'src/markdown.tsx', props: { path: 'static/privacy.md' } }),
+    emoji(),
     {
       name: 'u-wave-write-package-version',
       apply: 'build',