Upgrade Perseus from 22.7.0 to 75.7.1

.npmrc

@@ -1 +1,4 @@
 auto-install-peers = true
+# Perseus v75 uses SWC-compiled code that requires @swc/helpers at runtime,
+# but doesn't declare it as a dependency. Hoisting makes it resolvable by webpack.
+public-hoist-pattern[]=@swc/helpers

kolibri/plugins/perseus_viewer/.gitignore

@@ -1,4 +1,2 @@
-submodules
-!static/mathjax
-!frontend/dist
-!frontend/dist/fonts/*.ttf
+# MathJax fonts are copied from the mathjax-full node_modules dep at build time
+static/assets/mathjax/fonts/

kolibri/plugins/perseus_viewer/buildConfig.js

@@ -2,16 +2,103 @@
  * This file defines additional webpack configuration for this plugin.
  * It will be bundled into the webpack configuration at build time.
  */
+var fs = require('fs');
+var path = require('path');
 var webpack = require('webpack');
 
+// mathjax-full is a transitive dependency of @khanacademy/mathjax-renderer
+// and lives as a sibling in pnpm's virtual store.
+var mathjaxRendererDir = path.dirname(
+  require.resolve('@khanacademy/mathjax-renderer/package.json'),
+);
+var mathjaxFontsSource = path.join(
+  mathjaxRendererDir,
+  '../../mathjax-full/ts/output/chtml/fonts/tex-woff-v2',
+);
+var mathjaxFontsTarget = path.resolve(__dirname, 'static/assets/mathjax/fonts');
+
+// Copy MathJax fonts into the plugin's static assets directory before each
+// build. Django serves them at runtime via urls.static('assets/mathjax/fonts'),
+// and MathJaxRenderer fetches them by bare filename, so they must keep their
+// original names. The target directory is gitignored.
+function copyMathJaxFonts() {
+  fs.mkdirSync(mathjaxFontsTarget, { recursive: true });
+  for (const file of fs.readdirSync(mathjaxFontsSource)) {
+    if (!file.endsWith('.woff')) continue;
+    fs.copyFileSync(
+      path.join(mathjaxFontsSource, file),
+      path.join(mathjaxFontsTarget, file),
+    );
+  }
+}
+
+class CopyMathJaxFontsPlugin {
+  apply(compiler) {
+    const run = (_, cb) => {
+      try {
+        copyMathJaxFonts();
+        cb();
+      } catch (err) {
+        cb(err);
+      }
+    };
+    compiler.hooks.beforeRun.tapAsync('CopyMathJaxFontsPlugin', run);
+    compiler.hooks.watchRun.tapAsync('CopyMathJaxFontsPlugin', run);
+  }
+}
+
 module.exports = {
   bundle_id: 'main',
   webpack_config: {
     entry: 'frontend/module.js',
+    resolve: {
+      alias: {
+        // Alias for MathJax fonts so the Perseus CDN URL rewriter can
+        // reference them via ~mathjax-fonts/... in CSS.
+        'mathjax-fonts': mathjaxFontsSource,
+      },
+    },
+    module: {
+      rules: [
+        {
+          // Rewrite the KA CDN font URL in Perseus CSS to our local copy.
+          // Runs as a pre-loader before css-loader processes url() refs.
+          test: /@khanacademy[/\\]perseus[/\\]dist[/\\]index\.css$/,
+          enforce: 'pre',
+          use: [
+            path.resolve(__dirname, 'rewritePerseusUrls.js'),
+            path.resolve(__dirname, 'rewritePerseusRem.js'),
+          ],
+        },
+        {
+          // Convert rem→px in Wonder Blocks design tokens CSS.
+          // These tokens assume 1rem = 10px (KA's root font-size convention).
+          test: /@khanacademy[/\\]wonder-blocks-tokens[/\\].*\.css$/,
+          enforce: 'pre',
+          loader: path.resolve(__dirname, 'rewritePerseusRem.js'),
+        },
+        {
+          // Convert rem→px in math-input CSS (same KA rem convention).
+          test: /@khanacademy[/\\]math-input[/\\].*\.css$/,
+          enforce: 'pre',
+          loader: path.resolve(__dirname, 'rewritePerseusRem.js'),
+        },
+      ],
+    },
     plugins: [
+      new CopyMathJaxFontsPlugin(),
       new webpack.NormalModuleReplacementPlugin(
         /react\/jsx-runtime/,
-        require.resolve('react/jsx-runtime.js'),
+        require.resolve('react/jsx-runtime'),
+      ),
+      // Wonder Blocks components import from react-router-dom-v5-compat,
+      // which re-exports from react-router@6. The Perseus plugin only has
+      // react-router@5, so the v6 APIs (useInRouterContext, useNavigate)
+      // are undefined. Replace with a shim that returns "no router" so
+      // Wonder Blocks falls back to plain <a> tags.
+      new webpack.NormalModuleReplacementPlugin(
+        /react-router-dom-v5-compat/,
+        path.resolve(__dirname, 'frontend', 'reactRouterShim.js'),
       ),
     ],
   },

kolibri/plugins/perseus_viewer/buildPerseus.js

@@ -1,44 +1,3 @@
-/* eslint-disable import-x/no-commonjs, import-x/no-amd, import-x/no-import-module-exports */
-const fs = require('fs');
-const path = require('path');
 const extractPerseusMessages = require('./extractPerseusMessages');
 
-const target = path.resolve(__dirname, './frontend/dist');
-// A regex for detecting paths inside `url` in CSS, paths can either be quoted or unquoted.
-const cssPathRegex = /(url\(['"]?)([^"')]+)?(['"]?\),? ?)/g;
-const cssNonWoffRegex = /, (url\(['"]?)([^"')]+)?(['"]?\),? ?) format\((?!['"]woff['"])['"][a-z0-9]+['"]\)/g;
-// These are the css files that we are modifying to remap static assets.
-const cssFiles = [
-  [
-    path.join(path.dirname(require.resolve('@khanacademy/perseus')), 'index.css'),
-    path.join(target, 'index.css'),
-  ],
-  [
-    path.join(path.dirname(require.resolve('@khanacademy/math-input')), 'index.css'),
-    path.join(target, 'math-input.css'),
-  ],
-];
-
-for (const [indexCssFile, targetCssLocation] of cssFiles) {
-  console.log('Copying file and editing references for: ', indexCssFile);
-  const cssFileContents = fs.readFileSync(indexCssFile, { encoding: 'utf-8' });
-  const modifiedCssContents = cssFileContents.replace(cssPathRegex, function(match, p1, p2, p3) {
-    // Special case for MathJax font loaded from KA CDN
-    if (p2 === 'https://cdn.kastatic.org/fonts/mathjax/MathJax_Main-Regular.woff') {
-      return `${p1}fonts/MathJax_Main-Regular.woff${p3}`;
-    }
-    // Make absolute paths relative
-    const absolute = p2.startsWith('/');
-    const newUrl = absolute ? p2.slice(1) : p2;
-    if (newUrl) {
-      // If so, replace the instance with the new URL.
-      return `${p1}${newUrl}${p3}`;
-    }
-    // Otherwise just return empty string so that we remove the unfound file from the CSS.
-    return '';
-  }).replace(cssNonWoffRegex, '').replace(/\s+src: url\(fonts\/Symbola\.eot\);/, '');
-  fs.writeFileSync(targetCssLocation, modifiedCssContents, { encoding: 'utf-8' });
-}
-
-// Now that the file has been built, we can extract all the perseus messages.
 extractPerseusMessages();

kolibri/plugins/perseus_viewer/extractPerseusMessages.js

@@ -1,42 +1,23 @@
 /* eslint-disable import-x/no-commonjs, import-x/no-amd, import-x/no-import-module-exports */
 /*
- * A utility that extracts Perseus messages into a Javascript file for compatibility
- * with our i18n machinery. Also converts them into ICU format in the process.
+ * Extracts Perseus and math-input strings into a translator module compatible
+ * with our i18n machinery. Reads the strings objects straight out of each
+ * package's built dist, converts gettext-style %(name)s tokens to ICU syntax,
+ * and preserves any `context` values already present in the existing
+ * translator.js so hand-maintained context notes survive re-runs.
  */
 const fs = require('node:fs');
 const path = require('node:path');
-const https = require('node:https');
 
-
-// We already have lodash installed, so use it for templating the code we generate
 const lodash = require('lodash');
 
-const typescript = require('typescript');
-
 const { writeSourceToFile } = require('kolibri-format');
 
 const { replacePiText } = require('./frontend/translationUtils');
-const packageJson = require('./package.json');
-
-const perseusVersion = packageJson.dependencies['@khanacademy/perseus'];
-
-// Auto generate a module that creates the translator so that it can
-// be imported into our special i18n code for Perseus.
-
-const perseusStringFileUrl = `https://raw.githubusercontent.com/Khan/perseus/@khanacademy/perseus@${perseusVersion}/packages/perseus/src/strings.ts`;
-const mathInputStringFileUrl = `https://raw.githubusercontent.com/Khan/perseus/@khanacademy/perseus@${perseusVersion}/packages/math-input/src/strings.ts`;
 
 // Regex taken from perseus/lib/i18n.js interpolationMarker variable
 const gettextRegex = /%\(([\w_]+)\)s/g;
 
-/*
- * A function to transform Perseus' gettext formatted messages to ICU message syntax
- * Can be used replace all strings in a source file,
- * Or on a string by string basis to convert gettext formatted strings into ICU syntax,
- * For example when importing Khan Academy's gettext format translated strings.
- * It also normalizes the way pi is represented to make it not cause errors for ICU message syntax.
- * Finally, it escapes all backslashes to prevent errors in ICU token parsing.
- */
 function normalizeString(string) {
   return replacePiText(string.replace(gettextRegex, '{ $1 }')).replace(/\\/g, '\\\\');
 }
@@ -45,15 +26,15 @@ function normalizeStringObject(stringObject) {
   const normalizedObject = {};
   for (const key in stringObject) {
     if (lodash.isPlainObject(stringObject[key])) {
-      if (stringObject[key]['message']) {
+      if (stringObject[key].message) {
         normalizedObject[key] = {
-          message: normalizeString(stringObject[key]['message']),
-          context: stringObject[key]['context'],
+          message: normalizeString(stringObject[key].message),
+          context: stringObject[key].context,
         };
-      } else if (stringObject[key]['one'] && stringObject[key]['other']) {
-        const oneMessage = normalizeString(stringObject[key]['one']).trim();
-        const otherMessage = normalizeString(stringObject[key]['other']).trim();
-        const varName = gettextRegex.exec(stringObject[key]['one'])[1];
+      } else if (stringObject[key].one && stringObject[key].other) {
+        const oneMessage = normalizeString(stringObject[key].one).trim();
+        const otherMessage = normalizeString(stringObject[key].other).trim();
+        const varName = gettextRegex.exec(stringObject[key].one)[1];
         normalizedObject[key] = `{${varName}, plural, one {${oneMessage}} other {${otherMessage}}}`;
       } else {
         console.error('Unrecognized string object:', stringObject[key]);
@@ -65,52 +46,71 @@ function normalizeStringObject(stringObject) {
   return normalizedObject;
 }
 
-
-async function downloadFileAndGetMessages(urlPath, moduleName) {
+const translatorPath = path.join(__dirname, 'frontend/translator.js');
+
+// Capture contexts from the existing translator.js by stubbing createTranslator
+// and evaluating the file. Upstream flattened several strings to plain values in
+// v75, so preserving our own contexts is the only way to keep them.
+function readExistingContexts() {
+  if (!fs.existsSync(translatorPath)) return {};
+  const source = fs.readFileSync(translatorPath, 'utf8');
+  const captured = {};
+  const stub = (_name, strings) => Object.assign(captured, strings);
+  const evaluate = new Function('createTranslator', `
+    ${source.replace(/^import\s+.*$/gm, '').replace(/export\s+default\s+/, 'return ')}
+  `);
   try {
-    const data = await new Promise((resolve, reject) => {
-      const req = https.get(encodeURI(urlPath), (res) => {
-        if (res.statusCode !== 200) {
-          console.error('Error downloading file:', res);
-          return;
-        }
-      const chunks = [];
-      res.on('data', (chunk) => chunks.push(chunk));
-      res.on('end', () => {
-        const buffer = Buffer.concat(chunks);
-        resolve(buffer);
-      });
-      });
-      req.on('error', (error) => reject(error));
-      req.end();
-    });
-    const tempFilePath = path.join(__dirname, moduleName + 'tempFile.js');
-    const jsSource = typescript.transpileModule(data.toString(), { compilerOptions: { module: typescript.ModuleKind.CommonJS }});
-    fs.writeFileSync(tempFilePath, Buffer.from(jsSource.outputText));
-    const module = require(tempFilePath);
-    fs.unlinkSync(tempFilePath);
-    return normalizeStringObject(module.strings);
-  } catch (error) {
-    console.error('Error downloading file:', error);
-    throw error;
+    evaluate(stub);
+  } catch (err) {
+    console.error('Could not parse existing translator.js for context preservation:', err);
+    return {};
+  }
+  const contexts = {};
+  for (const key of Object.keys(captured)) {
+    if (lodash.isPlainObject(captured[key]) && captured[key].context) {
+      contexts[key] = captured[key].context;
+    }
+  }
+  return contexts;
+}
+
+function applyPreservedContexts(allStrings, existingContexts) {
+  for (const key of Object.keys(existingContexts)) {
+    const ctx = existingContexts[key];
+    const current = allStrings[key];
+    if (typeof current === 'string') {
+      allStrings[key] = { message: current, context: ctx };
+    } else if (lodash.isPlainObject(current) && !current.context) {
+      current.context = ctx;
+    }
   }
 }
 
-module.exports = async function() {
-  const perseusStrings = await downloadFileAndGetMessages(perseusStringFileUrl, 'perseus');
-  const mathInputStrings = await downloadFileAndGetMessages(mathInputStringFileUrl, 'mathInput');
+module.exports = function extractPerseusMessages() {
+  const perseusStrings = normalizeStringObject(
+    require('@khanacademy/perseus/strings').strings,
+  );
+  const mathInputStrings = normalizeStringObject(
+    require('@khanacademy/math-input/strings').strings,
+  );
 
   const allStrings = {
     ...perseusStrings,
-    // There is one duplicate key between the two files
-    // we will prefer the math input one for now, as it seems more useful.
+    // There is one duplicate key between the two files; prefer math-input.
     ...mathInputStrings,
   };
 
   for (const key in allStrings) {
-    if (perseusStrings[key] && mathInputStrings[key] && perseusStrings[key] !== mathInputStrings[key]) {
+    if (
+      perseusStrings[key] &&
+      mathInputStrings[key] &&
+      perseusStrings[key] !== mathInputStrings[key]
+    ) {
       if (lodash.isPlainObject(perseusStrings[key]) && lodash.isPlainObject(mathInputStrings[key])) {
-        if (perseusStrings[key].message === mathInputStrings[key].message && perseusStrings[key].context === mathInputStrings[key].context) {
+        if (
+          perseusStrings[key].message === mathInputStrings[key].message &&
+          perseusStrings[key].context === mathInputStrings[key].context
+        ) {
           continue;
         }
       }
@@ -120,18 +120,12 @@ module.exports = async function() {
     }
   }
 
-  // Use lodash template to fill in the above 'messages' into the template
-  let outputCode = `
+  applyPreservedContexts(allStrings, readExistingContexts());
 
-  import { createTranslator } from 'kolibri.utils.i18n';
-
-
-  export default createTranslator('PerseusInternalMessages',
-    `;
-  outputCode += JSON.stringify(allStrings, null, 2);
-  outputCode += ');'
-
-  // Write out the module to src files
+  const outputCode =
+    "\n\n  import { createTranslator } from 'kolibri/utils/i18n';\n\n\n  export default createTranslator('PerseusInternalMessages',\n    " +
+    JSON.stringify(allStrings, null, 2) +
+    ');';
 
   writeSourceToFile('./frontend/translator.js', outputCode);
-}
+};

kolibri/plugins/perseus_viewer/frontend/__tests__/fixtures/sorter-item.json

@@ -0,0 +1,29 @@
+{
+  "question": {
+    "content": "Sort the numbers in ascending order.\n\n[[☃ sorter 1]]",
+    "images": {},
+    "widgets": {
+      "sorter 1": {
+        "type": "sorter",
+        "alignment": "default",
+        "static": false,
+        "graded": true,
+        "options": {
+          "correct": ["1", "2", "3", "4", "5"],
+          "layout": "horizontal",
+          "padding": true
+        },
+        "version": { "major": 0, "minor": 0 }
+      }
+    }
+  },
+  "answerArea": {
+    "calculator": false,
+    "chi2Table": false,
+    "periodicTable": false,
+    "tTable": false,
+    "zTable": false
+  },
+  "itemDataVersion": { "major": 0, "minor": 1 },
+  "hints": []
+}