Implement updates tracking OWID grapher

.bundlemonrc.json

@@ -4,7 +4,7 @@
     "files": [
         {
             "path": "owid.mjs",
-            "maxSize": "820kB",
+            "maxSize": "840kB",
             "maxPercentIncrease": 5
         },
         {

.claude/skills/create-migration/SKILL.md

@@ -0,0 +1,18 @@
+---
+name: create-migration
+description: Create a new database migration file for the OWID MySQL database. Use when the user needs to create a database schema change or migration.
+---
+
+# Create Database Migration
+
+Create a new database migration file for the OWID MySQL 8 database.
+
+## Steps
+
+1. Run `yarn createDbMigration db/migration/<NewMigrationName>` where `<NewMigrationName>` is a descriptive name for the migration
+2. The generated filename will contain a timestamp prefix, so scan the `db/migration/` directory to find the actual path of the new file
+3. Report the new file path to the user
+
+## Naming Guidelines
+
+Choose a descriptive name for the migration that clearly indicates what schema change is being made (e.g., `AddUserEmailIndex`, `CreateAuditLogTable`, `RemoveDeprecatedColumns`).

.clinerules/GENERAL-RULES.md

@@ -2,8 +2,8 @@
 
 - yarn typecheck: runs the typescript typechecker across all files
 - yarn testLintChanged: run eslint on changed files
-- yarn testPrettierChanged: run prettier on changed files
-- yarn fixPrettierChanged: attempt to fix prettier issues on changed files
+- yarn testFormatChanged: check formatting on changed files
+- yarn fixFormatChanged: fix formatting on changed files
 - yarn test run --reporter dot: run unit tests. Uses vitest, can take one or more test filenames to only run a subset.
 - make migrate: apply migrations
 - make dbtest: run database and api tests

.codex/skills

@@ -0,0 +1 @@
+../.claude/skills

.devcontainer/devcontainer.json

@@ -11,7 +11,7 @@
     // },
 
     // Add the IDs of extensions you want installed when the container is created.
-    "extensions": ["dbaeumer.vscode-eslint", "esbenp.prettier-vscode"],
+    "extensions": ["dbaeumer.vscode-eslint", "oxc.oxc-vscode"],
 
     // Use 'forwardPorts' to make a list of ports inside the container available locally.
     // "forwardPorts": [3000, 27017],

.editorconfig

@@ -7,7 +7,7 @@ charset = utf-8
 indent_size = 4
 indent_style = space
 trim_trailing_whitespace = true
-insert_final_newline = true # somehow required for prettier to add a new line at the end of composer.json
+insert_final_newline = true
 
 [*.md]
 trim_trailing_whitespace = false

.env.archive

@@ -1,5 +1,6 @@
 IS_ARCHIVE=true
 ARCHIVE_BASE_URL=https://archive.ourworldindata.org
+WIKIPEDIA_ARCHIVE_BASE_URL=https://wikipedia-archive.ourworldindata.org
 ENVIRONMENT=production
 BAKED_BASE_URL=
 GRAPHER_DYNAMIC_THUMBNAIL_URL=

.env.devcontainer

@@ -1,8 +1,5 @@
 TZ=utc
 
-# Essential settings
-SECRET_KEY='fejwiaof jewiafo jeioa fjieowajf isa fjidosajfgj'
-
 GRAPHER_DB_NAME=grapher
 GRAPHER_DB_USER=grapher
 GRAPHER_DB_PASS=grapher

.env.example-full

@@ -1,8 +1,5 @@
 TZ=utc
 
-# Essential settings
-SECRET_KEY='fejwiaof jewiafo jeioa fjieowajf isa fjidosajfgj'
-
 # Instance configuration (for running multiple parallel dev environments)
 TMUX_SESSION_NAME=grapher  # optional; tmux session name to avoid conflicts
 COMPOSE_PROJECT_NAME=owid-grapher  # optional; docker compose project name for isolated volumes
@@ -69,6 +66,7 @@ DATA_API_URL= # optional
 
 SLACK_BOT_OAUTH_TOKEN= # optional
 SLACK_DI_PITCHES_CHANNEL_ID= # optional; #data-insight-pitches channel id
+SLACK_ALGOLIA_INDEXING_CHANNEL_ID= # optional; channel for Algolia indexing failure reports
 
 # for remote dev on tailscale without localhost + port forwarding
 ADMIN_SERVER_HOST= # optional; e.g: YOURMACHINE.TAILXXX.ts.net (find this in the tailscale admin panel)

.env.example-grapher

@@ -1,8 +1,5 @@
 TZ=utc
 
-# Essential settings
-SECRET_KEY='fejwiaof jewiafo jeioa fjieowajf isa fjidosajfgj'
-
 # Instance configuration (for running multiple parallel dev environments)
 TMUX_SESSION_NAME=grapher  # optional; tmux session name to avoid conflicts
 COMPOSE_PROJECT_NAME=owid-grapher  # optional; docker compose project name for isolated volumes

.github/pull_request_template.md

@@ -27,10 +27,9 @@ Step-by-step instructions on how to test this change
 
 If DB migrations exists:
 
-- [ ] If columns have been added/deleted, all necessary views were recreated
+- [ ] If columns have been added/deleted, all necessary views were recreated and ETL and Analytics team members have been informed of the incoming changes
 - [ ] The DB type definitions have been updated
 - [ ] The DB types in the ETL have been updated
-- [ ] If tables/views were added/removed, the Datasette export has been updated to take this into account
 - [ ] Update the documentation in db/docs
 
 ### After merging

.github/workflows/check-default-grapher-config.yml

@@ -47,8 +47,8 @@ jobs:
                       yarn tsx --tsconfig tsconfig.tsx.json devTools/schema/generate-default-object-from-schema.ts $json.name --save-ts packages/@ourworldindata/grapher/src/schema/defaultGrapherConfig.ts })
               shell: nu {0}
 
-            - name: Run prettier
-              run: yarn fixPrettierAll
+            - name: Run formatter
+              run: yarn fixFormatAll
 
             - uses: stefanzweifel/git-auto-commit-action@v7
               with:

.github/workflows/ci.yml

@@ -1,5 +1,9 @@
 name: CI
-on: [push, pull_request]
+on:
+    pull_request:
+    push:
+        branches:
+            - master
 
 jobs:
     # Checks for prettify errors, TypeScript errors and runs vitest tests.
@@ -45,8 +49,21 @@ jobs:
 
             - name: Run typecheck
               run: yarn typecheck
+    typecheck-legacy:
+        runs-on: ubuntu-latest
+
+        steps:
+            - name: Clone repository
+              uses: actions/checkout@v5
+
+            - uses: ./.github/actions/setup-node-yarn-deps
+              with:
+                  runPostinstallScripts: false
+
+            - name: Run typecheck (legacy)
+              run: yarn tsc -b
 
-    prettier:
+    format:
         runs-on: ubuntu-latest
 
         steps:
@@ -57,8 +74,8 @@ jobs:
               with:
                   runPostinstallScripts: false
 
-            - name: Run prettier
-              run: yarn testPrettierAll
+            - name: Check formatting
+              run: yarn testFormatAll
 
     eslint:
         runs-on: ubuntu-latest

.github/workflows/prettify.yml → .github/workflows/format.yml

@@ -1,15 +1,15 @@
-name: Prettify
+name: Format
 on:
     push:
         branches: [master]
     pull_request:
         branches: [master]
 
-# Runs `prettier` on a pull request so everything is nice and pretty!
+# Runs `oxfmt` on a pull request so everything is nice and formatted!
 # If something's wrong, a new commit fixing the issues will automatically be pushed to the PR branch.
 
 jobs:
-    prettify:
+    format:
         runs-on: ubuntu-latest
 
         steps:
@@ -22,9 +22,9 @@ jobs:
               with:
                   runPostinstallScripts: false
 
-            - name: Run prettier
-              run: yarn fixPrettierAll
+            - name: Run formatter
+              run: yarn fixFormatAll
 
             - uses: stefanzweifel/git-auto-commit-action@v7
               with:
-                  commit_message: "🤖 style: prettify code"
+                  commit_message: "🤖 style: format code"

.github/workflows/update-regions.yml

@@ -40,13 +40,12 @@ jobs:
                       Automated update of regions data
 
                       TODOs:
-                      - [ ] Update the [owid-detect-country worker](https://github.com/owid/cloudflare-workers/tree/main/workers/owid-detect-country) using its update script
                       - [ ] Check if there were any country renames, and if so:
                           - [ ] Create redirects from old to new slugs
                           - [ ] Take extra care :)
                   branch: "bot/update-regions-${{ steps.date.outputs.date }}"
                   base: "master"
-                  add-paths: "packages/@ourworldindata/utils/src/regions.json"
+                  add-paths: "packages/@ourworldindata/utils/src/regions/regions.data.ts"
 
 permissions:
     contents: write

.gitignore

@@ -5,9 +5,9 @@ logs/
 db.sqlite3
 .sublime*
 .nova/
-.yarn/*
-!.yarn/patches
-!.yarn/releases
+**/.yarn/*
+!**/.yarn/patches
+!**/.yarn/releases
 yarn-error.log
 .vscode/tags
 .awcache
@@ -46,6 +46,15 @@ dist/
 cfstorage/
 vite.*.mjs
 archive/*
+wikipedia-archive/*
+
+# Playwright
+test-results/
+**/.features-gen/**/*.spec.js
+playwright-report/
 
 # Sentry Config File
 .env.sentry-build-plugin
+
+# Claude
+.claude/settings.local.json

.gitpod.yml

@@ -46,4 +46,4 @@ github:
 vscode:
     extensions:
         - dbaeumer.vscode-eslint
-        - esbenp.prettier-vscode
+        - oxc.oxc-vscode

.oxfmtrc.json

@@ -0,0 +1,22 @@
+{
+    "$schema": "./node_modules/oxfmt/configuration_schema.json",
+    "printWidth": 80,
+    "semi": false,
+    "tabWidth": 4,
+    "trailingComma": "es5",
+    "ignorePatterns": [
+        ".tscache",
+        "/bakedSite",
+        "/localBake",
+        "datasetsExport",
+        "itsJustJavascript",
+        "package.json",
+        "coverage",
+        "packages/@ourworldindata/*/dist/",
+        "dist/",
+        "grapherData/",
+        ".vscode/",
+        ".claude/settings.local.json",
+        "*.handlebars"
+    ]
+}

.vscode/extensions.json

@@ -2,8 +2,10 @@
     "recommendations": [
         "dbaeumer.vscode-eslint",
         "editorconfig.editorconfig",
-        "esbenp.prettier-vscode",
+        "oxc.oxc-vscode",
         "mrmlnc.vscode-scss",
-        "vitest.explorer"
+        "vitest.explorer",
+        "ms-playwright.playwright",
+        "cucumberopen.cucumber-official"
     ]
 }

.vscode/settings.json

@@ -3,7 +3,9 @@
     "editor.insertSpaces": true,
     "editor.tabSize": 4,
     "editor.detectIndentation": false,
-    "editor.defaultFormatter": "esbenp.prettier-vscode",
+    "editor.defaultFormatter": "oxc.oxc-vscode",
+
+    "oxc.fmt.configPath": ".oxfmtrc.json",
     "files.insertFinalNewline": false,
     "search.exclude": {
         ".yarn": true,
@@ -20,10 +22,18 @@
     "typescript.preferences.importModuleSpecifierEnding": "js",
     "javascript.preferences.importModuleSpecifierEnding": "js",
     "[sql]": {
-        "editor.defaultFormatter": "inferrinizzard.prettier-sql-vscode"
+        "editor.defaultFormatter": "ReneSaarsoo.sql-formatter-vsc"
     },
-    "Prettier-SQL.keywordCase": "upper",
-    "Prettier-SQL.SQLFlavourOverride": "mysql",
-    "Prettier-SQL.expressionWidth": 80,
-    "prettier.semi": false
+    "SQL-Formatter-VSCode.denseOperators": true,
+    "SQL-Formatter-VSCode.linesBetweenQueries": 1,
+    "SQL-Formatter-VSCode.insertSpacesOverride": true,
+    "SQL-Formatter-VSCode.ignoreTabSettings": true,
+    "SQL-Formatter-VSCode.tabSizeOverride": 2,
+    "SQL-Formatter-VSCode.dialect": "mysql",
+    "[typescriptreact]": {
+        "editor.defaultFormatter": "oxc.oxc-vscode"
+    },
+    "[typescript]": {
+        "editor.defaultFormatter": "oxc.oxc-vscode"
+    }
 }

.yarnrc.yml

@@ -9,8 +9,10 @@ logFilters:
 nodeLinker: node-modules
 npmMinimalAgeGate: 1440 # 24 hours
 npmPreapprovedPackages:
+    - "@typescript/*"
     - react
     - react-dom
     - typescript
     - vite
     - vitest
+enableScripts: false

CLAUDE.md

@@ -2,13 +2,13 @@
 
 - yarn typecheck: runs the typescript typechecker across all files
 - yarn testLintChanged: run eslint on changed files
-- yarn testPrettierChanged: run prettier on changed files
-- yarn fixPrettierChanged: attempt to fix prettier issues on changed files
+- yarn testFormatChanged: check formatting on changed files
+- yarn fixFormatChanged: fix formatting on changed files
 - yarn test run --reporter dot: run unit tests. Uses vitest, can take one or more test filenames to only run a subset.
 - make migrate: apply migrations
 - make dbtest: run database and api tests
 
-When you have completed implementing a big set of changes, run `yarn typecheck` and fix any errors you have.
+When you have completed implementing a big set of changes, run `yarn fixFormatChanged > /dev/null 2>&1 && yarn typecheck` and fix any errors you have.
 
 When you want to create a git commit, refer to docs/agent-guidelines/commit-messages.md for instructions.
 
@@ -37,14 +37,16 @@ Some key directories, going roughly along the dependency chain from the most sta
 - ./baker - code that "bakes" our website by rendering React to static HTML
 - ./adminSiteServer - internal API for our admin
 - ./adminSiteClient - client UI of the admin
+- ./bespoke - self-contained custom data viz components embedded in articles via Shadow DOM. Each project under bespoke/projects/ has its own build. See bespoke/readme.md for details.
 - ./devTools - various utilities
 - ./functions - CloudFlare Functions. Most of our website is static but all our charts under https://ourworldindata.org/grapher/* are behind CF functions. These handle dynamic thumbnail generation, data downloads for end users etc.
 
 # Database documentation
 
 Our main datastore is a mysql 8 database. The documentation for this lives in db/docs - there is README.md file which is a good overview and starting point, then one TABLE-NAME.yml file per table describing the table in more detail. ALWAYS list the directory db/docs/ to understand which tables are available and read the relevant table description files before constructing a query or writing a migration.
+When creating a new migration, make sure to read the [database README](./db/readme.md).
 
-You can run (read only) queries against the database with `yarn query "QUERY TEXT"` - e.g. if you need to understand the contents of a table of the cardinality of various tables.
+You can run (read only) queries against the database with `yarn query "QUERY TEXT"` - e.g. if you need to understand the contents of a table or the cardinality of various tables. Use `yarn query -s "QUERY TEXT"` to query the staging database for the current git branch (e.g., on branch `images-pageviews` it connects to `staging-site-images-pageviews`).
 
 # Additional documentation