diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md index 32412f1e3..0a05c93f7 100644 --- a/DEPLOYMENT.md +++ b/DEPLOYMENT.md @@ -2,12 +2,15 @@ The Machine Learning for Kids site is made up of a few different pieces. This document is here to describe what they all are and where they go. -- [The bits that make up the site](#the-bits-that-make-up-the-site) -- [Where HTTP requests go](#where-http-requests-go) -- [Where data lives](#where-data-lives) -- [Where users are authenticated](#where-users-are-authenticated) -- [Where third-party APIs are accessed](#where-third-party-apis-are-accessed) - +- [Machine Learning for Kids production deployment](#machine-learning-for-kids-production-deployment) + - [The bits that make up the site](#the-bits-that-make-up-the-site) + - [Where HTTP requests go](#where-http-requests-go) + - [Where data lives](#where-data-lives) + - [Where users are authenticated](#where-users-are-authenticated) + - [Where third-party APIs are accessed](#where-third-party-apis-are-accessed) + - [Deploying mlforkids-api to Heroku (Docker)](#deploying-mlforkids-api-to-heroku-docker) + - [Setup Steps](#setup-steps) + - [Minimum Configuration for Heroku](#minimum-configuration-for-heroku) --- ## The bits that make up the site @@ -82,3 +85,50 @@ Data from third-party services (Spotify and Wikipedia) is made available in Scra ![deployment components](./docs/05-third-party.png) --- + +## Deploying mlforkids-api to Heroku (Docker) + +The **mlforkids-api** service can be deployed to Heroku using the Dockerfile in `./mlforkids-api` directory with the Heroku Container Registry. + +### Setup Steps + +1. From the repository root, change into `mlforkids-api`: + ```sh + cd mlforkids-api + ``` + +2. Create a Heroku app: + ```sh + heroku create your-app-name + ``` + +3. Add Heroku Postgres addon (automatically sets `DATABASE_URL`): + ```sh + heroku addons:create heroku-postgresql:essential-0 -a your-app-name + ``` + +4. Set required config vars (see [Minimum Configuration](#minimum-configuration-for-heroku) below): + ```sh + heroku config:set DEPLOYMENT=heroku AUTH0_DOMAIN=your.auth0.com -a your-app-name + # ... set other required vars + ``` + +5. Build and deploy with Docker: + ```sh + npm run build + docker build -t mlforkids-api . + heroku container:push web -a your-app-name + heroku container:release web -a your-app-name + ``` + +6. View logs to verify startup: + ```sh + heroku logs --tail -a your-app-name + ``` + +### Minimum Configuration for Heroku + +Since `DATABASE_URL` is being used, individual PostgreSQL environment variables (`POSTGRESQLHOST`, `POSTGRESQLPORT`, etc.) are automatically skipped. The Heroku Postgres addon provides `DATABASE_URL` automatically. + +**Minimum required config vars** (aside from `DATABASE_URL`): +- `HOST` - Set to `0.0.0.0` diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 8cacd0e44..e8832a041 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -103,3 +103,35 @@ npm start ``` The website should be running at the host and port you requested in the environment variables. + +## Alternative: Running mlforkids-api with Docker + +If you prefer to run the API in a Docker container locally, you can build and run the Dockerfile directly. + +### Building the Docker image + +From the `mlforkids-api` directory: + +```sh +docker build -t mlforkids-api . +``` + +### Running the Docker container + +Make sure PostgreSQL is running on your host machine, then run: + +```sh +docker run -p 3000:3000 \ + -e POSTGRESQLHOST=host.docker.internal \ + -e POSTGRESQLPORT=5432 \ + -e POSTGRESQLUSER=ml4kdbuser \ + -e POSTGRESQLPASSWORD=ml4kdbpwd \ + -e POSTGRESQLDATABASE=mlforkidsdb \ + -e PORT=3000 \ + -e HOST=0.0.0.0 \ + mlforkids-api +``` + +**Note:** Use `host.docker.internal` on macOS and Windows, or `172.17.0.1` on Linux to reference the host's PostgreSQL server from within the container. + +The website will be running at `http://localhost:3000`. diff --git a/mlforkids-api/Dockerfile b/mlforkids-api/Dockerfile index 6a8f3960a..ab76e898a 100644 --- a/mlforkids-api/Dockerfile +++ b/mlforkids-api/Dockerfile @@ -69,4 +69,8 @@ COPY --from=builder --chown=node:node /usr/src/build/dist/lib dist/lib # confirm prepared app RUN ls -l /usr/src/app +# handle dynamically assigned ports in certain environments / on certain platforms (via the $PORT environment variable) +# otherwise default to 3000 +EXPOSE ${PORT:-3000} + CMD ["npm", "start"] diff --git a/mlforkids-api/heroku.yml b/mlforkids-api/heroku.yml new file mode 100644 index 000000000..0494a69f6 --- /dev/null +++ b/mlforkids-api/heroku.yml @@ -0,0 +1,7 @@ +build: + docker: + web: Dockerfile +release: + image: web +run: + web: npm start diff --git a/mlforkids-api/package-lock.json b/mlforkids-api/package-lock.json index 894e901b1..6d991808f 100644 --- a/mlforkids-api/package-lock.json +++ b/mlforkids-api/package-lock.json @@ -41,7 +41,7 @@ "papaparse": "5.5.3", "pg": "8.18.0", "randomstring": "1.3.1", - "sharp": "0.34.5", + "sharp": "^0.34.5", "syllable": "4.1.0", "tmp": "0.2.5", "uuid": "11.1.0" @@ -124,7 +124,7 @@ }, "engines": { "node": "22.x", - "npm": "10.x" + "npm": "11.x" } }, "node_modules/@babel/code-frame": { @@ -9640,6 +9640,7 @@ "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.34.5.tgz", "integrity": "sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg==", "hasInstallScript": true, + "license": "Apache-2.0", "dependencies": { "@img/colour": "^1.0.0", "detect-libc": "^2.1.2", diff --git a/mlforkids-api/package.json b/mlforkids-api/package.json index eb9417f98..d5b72abd4 100644 --- a/mlforkids-api/package.json +++ b/mlforkids-api/package.json @@ -6,18 +6,15 @@ "lint": "./node_modules/.bin/eslint .", "coverage": "nyc --reporter=html mocha --timeout=120000 dist/tests/**", "codeenginejob": "node dist/lib/batch.js", - "___comment1": "=== BUILD SYSTEM ===", "clean": "node build-scripts/clean.js", "compile": "tsc", "compile:watch": "tsc --watch", - "___comment2": "--- UI Dependencies ---", "npm-frontend": "node build-scripts/npm-frontend.js", "tfjs": "node build-scripts/tfjs.js", "boweroverrides": "node build-scripts/boweroverrides.js", "uidependencies": "npm-run-all -s npm-frontend tfjs boweroverrides", - "___comment3": "--- Web Assets (individual tasks) ---", "css": "node build-scripts/css.js", "html": "node build-scripts/html.js", @@ -33,12 +30,10 @@ "languages": "node build-scripts/languages.js", "languages:prod": "node build-scripts/languages.js --prod", "datasets": "node build-scripts/datasets.js", - "___comment4": "--- Composed Build Tasks ---", "web": "npm-run-all -s css js:minify images html angularcomponents languages:prod datasets", "web:prod": "npm-run-all -p robots css js:minify:prod images html:prod angularcomponents languages:prod scratchblocks stories -s datasets", "scratch": "npm-run-all -p scratch3 scratchblocks", - "___comment5": "--- Main Builds ---", "build": "npm-run-all -p web compile", "build:notest": "npm-run-all -s compile web", @@ -48,7 +43,7 @@ }, "engines": { "node": "22.x", - "npm": "10.x" + "npm": "11.x" }, "nyc": { "include": [ @@ -59,13 +54,7 @@ "author": "Dale Lane", "devDependencies": { "@mediapipe/face_mesh": "0.4.1633559619", - "tensorflow-models-face-landmarks-detection": "1.0.6-4.22.0", - "tensorflow-models-handpose": "0.0.7-4.22.0", - "tensorflow-models-posenet": "2.2.2-4.22.0", - "tensorflow-models-speech-commands": "0.5.4-4.22.0", "@tensorflow/tfjs": "4.22.0", - "ydf-inference": "0.0.4", - "@types/async": "3.2.25", "@types/body-parser": "1.19.6", "@types/bunyan": "1.8.11", @@ -93,14 +82,33 @@ "@types/unzipper": "0.10.11", "@typescript-eslint/eslint-plugin": "8.54.0", "@typescript-eslint/parser": "8.54.0", + "@uirouter/angularjs": "1.1.2", + "angular": "1.8.3", + "angular-animate": "1.8.3", + "angular-aria": "1.8.3", + "angular-jwt": "0.1.11", + "angular-lock": "3.1.0", + "angular-material": "1.2.5", + "angular-messages": "1.8.3", + "angular-sanitize": "1.8.3", + "angular-scroll": "1.0.2", + "angular-timeago": "0.4.6", + "angular-translate": "2.19.1", + "angular-translate-loader-static-files": "2.19.1", + "auth0-js": "9.30.0", + "auth0-lock": "11.35.1", "autoprefixer": "10.4.24", + "blueimp-canvas-to-blob": "3.29.0", + "bootstrap": "3.4.1", "clean-css": "5.3.3", "clone": "2.1.2", + "d3": "5.16.0", "eslint": "9.39.2", "eslint-plugin-node": "11.1.0", "filecompare": "1.0.4", "glob": "13.0.0", "html-minifier-terser": "7.2.0", + "jquery": "3.7.1", "mocha": "11.7.5", "ng-annotate-patched": "1.15.0", "npm-run-all": "4.1.5", @@ -110,30 +118,15 @@ "sinon": "18.0.1", "sinon-express-mock": "2.2.1", "supertest": "7.2.2", + "tensorflow-models-face-landmarks-detection": "1.0.6-4.22.0", + "tensorflow-models-handpose": "0.0.7-4.22.0", + "tensorflow-models-posenet": "2.2.2-4.22.0", + "tensorflow-models-speech-commands": "0.5.4-4.22.0", "terser": "5.46.0", "typescript": "5.9.3", "typescript-require": "0.3.0", "unzipper": "0.12.3", - - "@uirouter/angularjs": "1.1.2", - "angular": "1.8.3", - "angular-animate": "1.8.3", - "angular-aria": "1.8.3", - "angular-jwt": "0.1.11", - "angular-lock": "3.1.0", - "angular-material": "1.2.5", - "angular-messages": "1.8.3", - "angular-sanitize": "1.8.3", - "angular-scroll": "1.0.2", - "angular-timeago": "0.4.6", - "angular-translate": "2.19.1", - "angular-translate-loader-static-files": "2.19.1", - "auth0-js": "9.30.0", - "auth0-lock": "11.35.1", - "blueimp-canvas-to-blob": "3.29.0", - "bootstrap": "3.4.1", - "d3": "5.16.0", - "jquery": "3.7.1" + "ydf-inference": "0.0.4" }, "dependencies": { "@slack/webhook": "7.0.6", @@ -169,7 +162,7 @@ "papaparse": "5.5.3", "pg": "8.18.0", "randomstring": "1.3.1", - "sharp": "0.34.5", + "sharp": "^0.34.5", "syllable": "4.1.0", "tmp": "0.2.5", "uuid": "11.1.0" diff --git a/mlforkids-api/src/lib/app.ts b/mlforkids-api/src/lib/app.ts index 60f16cafb..f216ae082 100644 --- a/mlforkids-api/src/lib/app.ts +++ b/mlforkids-api/src/lib/app.ts @@ -1,3 +1,8 @@ +// startup logging for debugging +console.log('[STARTUP] Application starting...'); +console.log('[STARTUP] Node version:', process.version); +console.log('[STARTUP] Environment:', process.env.NODE_ENV || 'development'); + // external dependencies import * as express from 'express'; import { Server } from 'http'; @@ -18,21 +23,39 @@ import loggerSetup from './utils/logger'; const log = loggerSetup(); let server: Server | undefined; -// do this before doing anything! -confirmRequiredEnvironment(); - // log any uncaught errors before crashing -process.on('uncaughtException', shutdown.crash); +process.on('uncaughtException', (err: Error) => { + console.error('[STARTUP] Uncaught exception:'); + console.error('[STARTUP]', err.message); + console.error(err.stack); + shutdown.crash(err); +}); // terminate quickly if we get a SIGTERM signal process.on('SIGTERM', () => { shutdown.now('SIGTERM', server); server = undefined; }); process.on('SIGINT', () => { shutdown.now('SIGINT', server); server = undefined; }); +console.log('[STARTUP] Confirming required environment variables...'); +// do this before doing anything! +try { + confirmRequiredEnvironment(); + console.log('[STARTUP] Environment variables confirmed successfully'); +} catch (err) { + const errorMsg = err instanceof Error ? err.message : String(err); + console.error('[STARTUP] FATAL: Failed to confirm required environment variables'); + console.error('[STARTUP] ERROR:', errorMsg); + if (log && typeof log.error === 'function') { + log.error(err, 'Failed to confirm required environment variables'); + } + process.exit(1); +} + // check if the site is running in read-only mode if (env.inMaintenanceMode()) { log.error('Site is running in maintenance mode'); } +console.log('[STARTUP] Initializing services...'); // prepare Slack API for reporting alerts slack.init(); // prepare SMTP pool for sending notification emails @@ -47,20 +70,40 @@ iamcache.init(); // initialise the cache for checking API key requirements credentialscheck.init(); +console.log('[STARTUP] Connecting to database...'); // connect to DB store.init() .then(() => { + console.log('[STARTUP] Database connected successfully'); + console.log('[STARTUP] Refreshing site alerts cache...'); // check for current site alerts - sitealerts.refreshCache(); - + return sitealerts.refreshCache(); + }) + .then(() => { + console.log('[STARTUP] Creating Express server...'); // create server const app = express(); const host: string = process.env.HOST || '0.0.0.0'; const port: number = env.getPortNumber(); + console.log('[STARTUP] Setting up REST API...'); // setup server and run restapi(app); + + console.log('[STARTUP] Starting server on', host, ':', port); server = app.listen(port, host, () => { + console.log('[STARTUP] Server is now listening'); log.info({ host, port }, 'Running'); }); + }) + .catch((err) => { + console.error('[STARTUP] FATAL: Error during initialization'); + console.error('[STARTUP] Error type:', err instanceof Error ? err.constructor.name : typeof err); + console.error('[STARTUP] Error message:', err instanceof Error ? err.message : String(err)); + if (err instanceof Error && err.stack) { + console.error('[STARTUP] Stack trace:'); + console.error(err.stack); + } + log.error({ err }, 'Failed to initialize application'); + process.exit(1); }); diff --git a/mlforkids-api/src/lib/db/postgresqldb.ts b/mlforkids-api/src/lib/db/postgresqldb.ts index f2f9ab204..a1834b7e5 100644 --- a/mlforkids-api/src/lib/db/postgresqldb.ts +++ b/mlforkids-api/src/lib/db/postgresqldb.ts @@ -9,25 +9,47 @@ let connectionPool: pg.Pool | undefined; const log = loggerSetup(); +function parseConnectionUrl(url: string): pg.PoolConfig { + const parsedUrl = new URL(url); + return { + host: parsedUrl.hostname || undefined, + port: parsedUrl.port ? parseInt(parsedUrl.port, 10) : undefined, + user: parsedUrl.username || undefined, + password: parsedUrl.password || undefined, + database: parsedUrl.pathname?.slice(1) || undefined, + }; +} export async function connect(): Promise { if (!connectionPool) { pg.types.setTypeParser(20, parseInt); - const connectionOptions: pg.PoolConfig = { - host: process.env[env.POSTGRESQLHOST], - port: parseInt(process.env[env.POSTGRESQLPORT] as string, 10), - user: process.env[env.POSTGRESQLUSER], - password: process.env[env.POSTGRESQLPASSWORD], - database: process.env[env.POSTGRESQLDATABASE], - }; - - if (process.env[env.POSTGRESQLCERT]) { - connectionOptions.ssl = { - ca : await fileutils.read(process.env[env.POSTGRESQLCERT] as string), + let connectionOptions: pg.PoolConfig; + + // use DATABASE_URL if available, otherwise use individual env vars + if (process.env.DATABASE_URL) { + connectionOptions = parseConnectionUrl(process.env.DATABASE_URL); + // require SSL for DATABASE_URL connections + connectionOptions.ssl = { rejectUnauthorized: false }; + } else { + connectionOptions = { + host: process.env[env.POSTGRESQLHOST], + port: parseInt(process.env[env.POSTGRESQLPORT] as string, 10), + user: process.env[env.POSTGRESQLUSER], + password: process.env[env.POSTGRESQLPASSWORD], + database: process.env[env.POSTGRESQLDATABASE], }; + + if (process.env[env.POSTGRESQLCERT]) { + connectionOptions.ssl = { + ca : await fileutils.read(process.env[env.POSTGRESQLCERT] as string), + }; + } } + // explicitly set schema search path to mlforkidsdb + connectionOptions.options = '-c search_path=mlforkidsdb,public'; + connectionPool = new pg.Pool(connectionOptions); connectionPool.on('error', (err) => { diff --git a/mlforkids-api/src/lib/utils/env.ts b/mlforkids-api/src/lib/utils/env.ts index ce416d804..d6535d460 100644 --- a/mlforkids-api/src/lib/utils/env.ts +++ b/mlforkids-api/src/lib/utils/env.ts @@ -61,11 +61,23 @@ export function confirmRequiredEnvironment() { function checkEnv(env: string) { + if (process.env.DATABASE_URL && isPostgresEnvVar(env)) { + return; + } if (!process.env[env]) { throw new Error('Missing required environment variable ' + env); } } +function isPostgresEnvVar(env: string): boolean { + return env === POSTGRESQLHOST || + env === POSTGRESQLPORT || + env === POSTGRESQLUSER || + env === POSTGRESQLPASSWORD || + env === POSTGRESQLDATABASE || + env === POSTGRESQLCERT; +} + export function inMaintenanceMode() { return process.env[MAINTENANCE_MODE] === 'true'; }