From 14605c14e3a3db3bd421bdbc2d5089d4c040c2dc Mon Sep 17 00:00:00 2001 From: ZorTik Date: Sun, 19 Apr 2026 13:31:47 +0200 Subject: [PATCH 01/53] refactor: mkdirTemp --- src/engine/docker/action/build.ts | 28 +++------------ src/engine/docker/action/build.worker.ts | 43 ------------------------ src/engine/docker/index.ts | 3 +- src/filestructure.ts | 13 +++++++ 4 files changed, 18 insertions(+), 69 deletions(-) delete mode 100644 src/engine/docker/action/build.worker.ts diff --git a/src/engine/docker/action/build.ts b/src/engine/docker/action/build.ts index 370e4eb..cee4618 100644 --- a/src/engine/docker/action/build.ts +++ b/src/engine/docker/action/build.ts @@ -2,13 +2,11 @@ import DockerClient from "dockerode"; import fs from "fs"; import path from "path"; import tar from "tar"; -import {currentContext, currentContext as ctx} from "../../../app"; import {MessageListener, ServiceEngine} from "@nsm/engine"; import {clock} from "@nsm/util/clock"; -import {Worker} from "worker_threads"; import {getRootFilesFiltered} from "@nsm/engine/ignore"; -import {Paths} from "env-paths"; -import {getTempPath} from "@nsm/filestructure"; +import {mkdirTemp} from "@nsm/filestructure"; +import {currentContext} from "@nsm/app"; async function prepareImage( args: { @@ -66,21 +64,6 @@ async function prepareImage( resolve(msg); } } - /*if (ctx.workers) { - // Build using workers - const w = new Worker(__dirname + path.sep + 'build.worker.js', { - workerData: { - archive, - imageTag, - env, - appConfig: ctx.appConfig, - debug: ctx.debug - } - }); - w.on('message', msgHandler); - } else { - // Here comes the normal build - }*/ // In container, worker threads are not supported. Or they // are disabled. client.buildImage(archive, { t: imageTag, buildargs: env }).then(stream => { @@ -124,11 +107,8 @@ async function prepareImage( ); } -export default function (client: DockerClient, paths: Paths): ServiceEngine['build'] { - const arDir = path.join(getTempPath(), "archives"); - if (!fs.existsSync(arDir)) { - fs.mkdirSync(arDir, { recursive: true }); - } +export default function (client: DockerClient): ServiceEngine['build'] { + const arDir = mkdirTemp("archives"); return async (imageId, buildDir, options, messageListener) => { const imageBuildClock = clock(); diff --git a/src/engine/docker/action/build.worker.ts b/src/engine/docker/action/build.worker.ts deleted file mode 100644 index 9e45d67..0000000 --- a/src/engine/docker/action/build.worker.ts +++ /dev/null @@ -1,43 +0,0 @@ -import {workerData, parentPort} from "worker_threads"; -import fs from "fs"; -import {initDockerClient} from "../client"; - -const appConfig = workerData['appConfig'] as any; -const archive = workerData['archive'] as string; -const tag = workerData['imageTag'] as string; -const env = workerData['env'] as any; -const debug = workerData['debug'] as boolean; - -const client = initDockerClient(appConfig); -const logs = []; - -if (debug) { - console.log("Running image build inside worker."); -} - -client.buildImage(archive, { t: tag, buildargs: env }).then(stream => { - logs.push('--------- Begin Build Log ---------'); - client.modem.followProgress(stream, (err, res) => { - if (err) { - console.error(err); - } else { - res.forEach(r => { - if (r.errorDetail) { - console.error(new Error(r.errorDetail)); - } else { - const msg = r.stream?.trim(); - //ctx.logger.info(msg); - logs.push(msg); - } - }); - logs.push('--------- End Of Build Log ---------\n'); - fs.unlinkSync(archive); - parentPort.postMessage(logs); - parentPort.postMessage(tag); - } - }); -}); - -if (debug) { - console.log("End of worker."); -} \ No newline at end of file diff --git a/src/engine/docker/index.ts b/src/engine/docker/index.ts index 4814c81..26b43a3 100644 --- a/src/engine/docker/index.ts +++ b/src/engine/docker/index.ts @@ -16,7 +16,6 @@ import stat from "./action/stat"; import statAll from "./action/statall"; import calcHostUsage from "./action/calcHostUsage"; import listRunning from "./action/listRunning"; -import {currentPaths} from "@nsm/filestructure"; import {AppConfig} from "@nsm/config"; export default function buildDockerEngine(appConfig: AppConfig) { @@ -27,7 +26,7 @@ export default function buildDockerEngine(appConfig: AppConfig) { engine.dockerClient = client; engine.rws = {}; // engine.cast - Being replaced in manager. - engine.build = build(client, currentPaths); + engine.build = build(client); engine.run = run(engine, client); engine.stop = stop(client); engine.kill = kill(client); diff --git a/src/filestructure.ts b/src/filestructure.ts index 6aca09d..682bf6f 100644 --- a/src/filestructure.ts +++ b/src/filestructure.ts @@ -27,6 +27,19 @@ export const getTempPath = () => { return currentPaths.temp; } +export const mkdirTemp = (...p: string[]) => { + const dir = path.join(getTempPath(), ...p); + if (fs.existsSync(dir)) { + if (!fs.statSync(dir).isDirectory()) { + throw new Error('Temp path already exists and is not a directory: ' + dir); + } + } else { + fs.mkdirSync(dir, { recursive: true }); + } + + return dir; +} + export const prepareFolders = () => { const resourcesTargetPath = getResourcesTargetPath(); if (!fs.existsSync(resourcesTargetPath)) { From 41733615abdca2e49ef427cab56ac877c1f7de55 Mon Sep 17 00:00:00 2001 From: ZorTik Date: Fri, 8 May 2026 20:48:52 +0200 Subject: [PATCH 02/53] refactor: better error logging & remove unused function --- src/app.ts | 10 --------- src/engine/manager.ts | 10 +++------ src/engine/middle.ts | 9 ++++++-- src/lib/isDocker.ts | 41 ------------------------------------ src/lib/isInsideContainer.ts | 35 ------------------------------ tests/api/api.test.ts | 1 - 6 files changed, 10 insertions(+), 96 deletions(-) delete mode 100644 src/lib/isDocker.ts delete mode 100644 src/lib/isInsideContainer.ts diff --git a/src/app.ts b/src/app.ts index a84cc8b..6d4d4b5 100644 --- a/src/app.ts +++ b/src/app.ts @@ -22,7 +22,6 @@ import * as logging from "./logger"; import winston from "winston"; import {Application} from "express-ws"; import fs from "fs"; -import isInsideContainer from "@nsm/lib/isInsideContainer"; import {middleLayer} from "@nsm/engine/middle"; import {SessionManager} from "@nsm/engine/session"; import {mkdirResource, saveResource} from "@nsm/resources"; @@ -40,12 +39,10 @@ export type AppContext = { appConfig: AppConfig; logger: winston.Logger; debug: boolean; - workers: boolean; }; export type AppBootOptions = { test?: boolean; - disableWorkers?: boolean; } export let currentContext: AppContext; @@ -112,7 +109,6 @@ export const init = async (router: Application, options?: AppBootOptions): Promi appConfig, logger, debug: process.env.DEBUG === 'true', - workers: !options?.disableWorkers && !isInsideContainer() }; // Service (virtualization) layer @@ -133,12 +129,6 @@ export const init = async (router: Application, options?: AppBootOptions): Promi // Start the server steps('BEFORE_SERVER', ctx); - if (isInsideContainer()) { - logger.info('Running in container! Worker threads will be unavailable.'); - } else if(!ctx.workers) { - logger.info('Worker threads are forcibly disabled.'); - } - let srv = undefined; if (options?.test == undefined || options.test == false) { logger.info(`Starting server`); diff --git a/src/engine/manager.ts b/src/engine/manager.ts index 8e7b99b..01f08f2 100644 --- a/src/engine/manager.ts +++ b/src/engine/manager.ts @@ -306,7 +306,7 @@ export type State = 'RUNNING' | 'BUILDING' | 'STOPPED'; // 1 = unknown, 2 = conflict, 3 = not found export type StatusCode = 1 | 2 | 3; -class _InternalError extends Error { +export class _InternalError extends Error { readonly code: StatusCode; readonly msg: string; @@ -513,7 +513,6 @@ export async function resumeService(id: string) { .reduce((obj, [key, value]) => ({ ...obj, [key]: value }), {}), } - const meta = metaStorageForService(id); const unlock = lockBusyAction(id, 'resume'); @@ -803,10 +802,6 @@ function metaStorageForService(id: string): MetaStorage { // service id }; } -export function initialized() { - return engine !== undefined; -} - export async function initEngineForcibly() { if (engine) { throw new Error("Engine is already loaded."); @@ -941,7 +936,8 @@ function reqNotRunning(id: string) { function reqTemplate(id: string) { const template = getTemplate(id); if (!template) { - throw new _InternalError('Template not found.', 3); + throw new _InternalError('' + + 'Template not found.', 3); } return template; diff --git a/src/engine/middle.ts b/src/engine/middle.ts index 2b736c6..f29c912 100644 --- a/src/engine/middle.ts +++ b/src/engine/middle.ts @@ -1,4 +1,4 @@ -import {ServiceManager} from "@nsm/engine/manager"; +import {_InternalError, ServiceManager} from "@nsm/engine/manager"; import {currentContext} from "@nsm/app"; export type ServiceActionType = 'create' | 'resume' | 'stop' | 'forceStop' | 'sendStopSignal' | 'delete'; @@ -69,7 +69,12 @@ const decorateFunc = ) => Promise>( }; await publishError(action); - currentContext.logger.error(`${action.serviceId ? `Service ${action.serviceId} f` : "F"}ailed action ${action.type}: ${action.message}`, e); + // don't log stack trace of known errors + const errorMeta: any[] = e instanceof _InternalError && e.code != 1 ? [] : [e]; + currentContext.logger.error( + `${action.serviceId ? `Service ${action.serviceId} f` : "F"}ailed action ${action.type}: ${action.message}`, + ...errorMeta + ); throw e; } diff --git a/src/lib/isDocker.ts b/src/lib/isDocker.ts deleted file mode 100644 index 1dbbdb9..0000000 --- a/src/lib/isDocker.ts +++ /dev/null @@ -1,41 +0,0 @@ -/* -MIT License - -Copyright (c) Sindre Sorhus (https://sindresorhus.com) - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -import fs from 'node:fs'; - -let isDockerCached; - -function hasDockerEnv() { - try { - fs.statSync('/.dockerenv'); - return true; - } catch { - return false; - } -} - -function hasDockerCGroup() { - try { - return fs.readFileSync('/proc/self/cgroup', 'utf8').includes('docker'); - } catch { - return false; - } -} - -export default function isDocker() { - // TODO: Use `??=` when targeting Node.js 16. - if (isDockerCached === undefined) { - isDockerCached = hasDockerEnv() || hasDockerCGroup(); - } - - return isDockerCached; -} \ No newline at end of file diff --git a/src/lib/isInsideContainer.ts b/src/lib/isInsideContainer.ts deleted file mode 100644 index f55695c..0000000 --- a/src/lib/isInsideContainer.ts +++ /dev/null @@ -1,35 +0,0 @@ -/* -MIT License - -Copyright (c) Sindre Sorhus (https://sindresorhus.com) - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - */ - -import fs from 'node:fs'; -import isDocker from "@nsm/lib/isDocker"; - -let cachedResult; - -// Podman detection -const hasContainerEnv = () => { - try { - fs.statSync('/run/.containerenv'); - return true; - } catch { - return false; - } -}; - -export default function isInsideContainer() { - // TODO: Use `??=` when targeting Node.js 16. - if (cachedResult === undefined) { - cachedResult = hasContainerEnv() || isDocker(); - } - - return cachedResult; -} \ No newline at end of file diff --git a/tests/api/api.test.ts b/tests/api/api.test.ts index 4e79651..25abb8b 100644 --- a/tests/api/api.test.ts +++ b/tests/api/api.test.ts @@ -43,7 +43,6 @@ describe("Test v1 API models", () => { beforeAll((done) => { const options: AppBootOptions = { test: true, - disableWorkers: true, }; boot(server, options).then((ctx_) => { ctx = ctx_; From f361f8983085542ecb859fa951ae589b7031bf5f Mon Sep 17 00:00:00 2001 From: ZorTik Date: Sun, 10 May 2026 19:13:02 +0200 Subject: [PATCH 03/53] refactor: prettify --- .github/workflows/jest.yml | 14 +- .github/workflows/push-image.yml | 12 +- .prettierignore | 3 + .prettierrc | 1 + README.md | 8 + addons/example_addon/index.ts | 16 +- babel.config.js | 10 +- docker-compose.yml | 14 +- index.ts | 14 +- installTempDeps.js | 26 +- jest.config.js | 17 +- openapi.yml | 2 +- package.json | 1 + resources/config.yml | 8 +- .../template/example/example_settings.yml | 10 +- resources/template/test/test_settings.yml | 8 +- src/addon.ts | 181 +- src/app.ts | 230 +-- src/cleanup.ts | 80 +- src/config.ts | 40 +- src/database/image.ts | 93 +- src/database/index.ts | 52 +- src/database/meta.ts | 15 +- src/database/models.ts | 183 +- src/database/perma.ts | 55 +- src/database/serviceLog.ts | 24 +- src/database/serviceMeta.ts | 20 +- src/database/session.ts | 24 +- src/depend.ts | 12 +- src/engine/asyncp.ts | 70 +- src/engine/docker/action/build.ts | 204 +-- src/engine/docker/action/calcHostUsage.ts | 6 +- src/engine/docker/action/cmd.ts | 27 +- src/engine/docker/action/deletei.ts | 10 +- src/engine/docker/action/deletev.ts | 27 +- src/engine/docker/action/getLabels.ts | 8 +- src/engine/docker/action/kill.ts | 32 +- src/engine/docker/action/listRunning.ts | 14 +- src/engine/docker/action/listc.ts | 33 +- src/engine/docker/action/listp.ts | 27 +- src/engine/docker/action/reattach.ts | 73 +- src/engine/docker/action/run.ts | 96 +- src/engine/docker/action/stat.ts | 19 +- src/engine/docker/action/statall.ts | 14 +- src/engine/docker/action/stop.ts | 32 +- src/engine/docker/client.ts | 52 +- src/engine/docker/index.ts | 72 +- src/engine/docker/util/env.ts | 6 +- src/engine/docker/util/labels.ts | 8 +- src/engine/docker/util/logging.ts | 18 +- src/engine/engine.ts | 507 +++--- src/engine/ignore.ts | 93 +- src/engine/image.ts | 101 +- src/engine/index.ts | 2 +- src/engine/manager.ts | 1495 +++++++++-------- src/engine/middle.ts | 114 +- src/engine/monitoring/templateDirWatcher.ts | 46 +- src/engine/monitoring/util.ts | 6 +- src/engine/session.ts | 114 +- src/engine/template.ts | 192 ++- src/filestructure.ts | 22 +- src/helpers.ts | 12 +- src/logger.ts | 61 +- src/networking/manager.ts | 93 +- src/profiler/index.ts | 12 +- src/resources.ts | 16 +- src/router/index.ts | 88 +- src/router/util/preconditions.ts | 30 +- src/router/v1/index.ts | 4 +- src/router/v1/service/createRoute.ts | 112 +- src/router/v1/service/deleteRoute.ts | 61 +- src/router/v1/service/listRoute.ts | 135 +- src/router/v1/service/logsRoute.ts | 56 +- src/router/v1/service/lookupRoute.ts | 85 +- src/router/v1/service/optionsRoute.ts | 79 +- src/router/v1/service/powerStatusRoute.ts | 65 +- src/router/v1/service/rebootRoute.ts | 93 +- src/router/v1/service/resumeRoute.ts | 81 +- src/router/v1/service/sessionsRoute.ts | 32 +- src/router/v1/service/stopCmdRoute.ts | 86 +- src/router/v1/service/stopRoute.ts | 94 +- src/router/v1/session/sessionLogsRoute.ts | 39 +- src/router/v1/status/index.ts | 147 +- src/security/index.ts | 18 +- src/security/token/index.ts | 59 +- src/server.ts | 8 +- src/util/clock.ts | 16 +- src/util/docker.ts | 62 +- src/util/env.ts | 24 +- src/util/port.ts | 78 +- src/util/promises.ts | 14 +- src/util/routes.ts | 24 +- src/util/services.ts | 14 +- src/util/yaml.ts | 4 +- tests/api/api.test.ts | 413 ++--- tests/database/manager.test.ts | 60 +- tests/engine/image.test.ts | 77 +- tests/engine/middle.test.ts | 20 +- tests/testUtils.ts | 16 +- tsconfig.json | 30 +- 100 files changed, 3839 insertions(+), 3292 deletions(-) create mode 100644 .prettierignore create mode 100644 .prettierrc diff --git a/.github/workflows/jest.yml b/.github/workflows/jest.yml index 2560713..e068de1 100644 --- a/.github/workflows/jest.yml +++ b/.github/workflows/jest.yml @@ -13,9 +13,9 @@ jobs: test: runs-on: ubuntu-latest env: - DATABASE_URL: 'mysql://test:test@localhost:3306/test' - CONFIG_DOCKER_HOST: '///var/run/docker.sock' - DEBUG: 'true' + DATABASE_URL: "mysql://test:test@localhost:3306/test" + CONFIG_DOCKER_HOST: "///var/run/docker.sock" + DEBUG: "true" steps: - name: Checkout uses: actions/checkout@v2 @@ -31,9 +31,9 @@ jobs: - name: Setup MySQL uses: mirromutth/mysql-action@v1.1 with: - mysql database: 'test' - mysql user: 'test' - mysql password: 'test' + mysql database: "test" + mysql user: "test" + mysql password: "test" - name: Install Node.js uses: actions/setup-node@v6 with: @@ -48,4 +48,4 @@ jobs: run: npm test - name: Publish Test Summary Results run: npx github-actions-ctrf ctrf/ctrf-report.json - if: always() \ No newline at end of file + if: always() diff --git a/.github/workflows/push-image.yml b/.github/workflows/push-image.yml index 25c76e9..81787a9 100644 --- a/.github/workflows/push-image.yml +++ b/.github/workflows/push-image.yml @@ -10,7 +10,7 @@ on: workflow_dispatch: inputs: tag: - description: 'Tag to use for the image (optional, defaults to branch name and short SHA)' + description: "Tag to use for the image (optional, defaults to branch name and short SHA)" required: false jobs: @@ -30,20 +30,20 @@ jobs: run: | IMAGE_OWNER=$(echo "${{ github.repository_owner }}" | tr '[:upper:]' '[:lower:]') IMAGE_NAME=ghcr.io/$IMAGE_OWNER/node-server-manager - + if [ "${{ github.event_name }}" = "release" ]; then TAG=${{ github.event.release.tag_name }} echo "IS_RELEASE=true" >> $GITHUB_ENV - + elif [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ -n "${{ github.event.inputs.tag }}" ]; then TAG=${{ github.event.inputs.tag }} echo "IS_RELEASE=true" >> $GITHUB_ENV - + else TAG=${GITHUB_REF_NAME}-$(echo $GITHUB_SHA | cut -c1-7) echo "IS_RELEASE=false" >> $GITHUB_ENV fi - + echo "IMAGE_NAME=$IMAGE_NAME" >> $GITHUB_ENV echo "IMAGE_TAG=$TAG" >> $GITHUB_ENV @@ -66,4 +66,4 @@ jobs: if: env.IS_RELEASE == 'true' run: | docker tag $IMAGE_NAME:$IMAGE_TAG $IMAGE_NAME:latest - docker push $IMAGE_NAME:latest \ No newline at end of file + docker push $IMAGE_NAME:latest diff --git a/.prettierignore b/.prettierignore new file mode 100644 index 0000000..1b8ac88 --- /dev/null +++ b/.prettierignore @@ -0,0 +1,3 @@ +# Ignore artifacts: +build +coverage diff --git a/.prettierrc b/.prettierrc new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/.prettierrc @@ -0,0 +1 @@ +{} diff --git a/README.md b/README.md index 1b2073f..0c8201c 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ NSM is a robust service manager built on Docker Engine. Its primary purpose is t - **Resources usage management**: NSM provides ability to limit or extend resources limits and view current usage. ## API Specification + Specification is hosted on external repository here ## Prerequisites @@ -27,16 +28,20 @@ Ensure you have the following installed before proceeding with the installation: Follow these steps to install and set up NSM: 1. **Clone the Repository** + ```sh git clone https://github.com/ZorTik/node-server-manager ``` + Alternatively, download the latest release from the [NSM repository](https://github.com/ZorTik/node-server-manager) and extract it. 2. **Configure Environment Variables** Copy the example environment file and fill in the required values: + ```sh cp .env.example .env ``` + Open the `.env` file and provide the necessary configuration values. 3. **Edit Configuration** @@ -44,18 +49,21 @@ Follow these steps to install and set up NSM: 4. **Install Dependencies** Install the required Node.js packages: + ```sh npm install ``` 5. **Generate Prisma Client** Generate the Prisma client for database interaction: + ```sh npx prisma generate ``` 6. **Sync Database Schema** Apply the database schema migrations: + ```sh npx prisma migrate deploy ``` diff --git a/addons/example_addon/index.ts b/addons/example_addon/index.ts index 9c93283..3c2be3a 100644 --- a/addons/example_addon/index.ts +++ b/addons/example_addon/index.ts @@ -1,14 +1,14 @@ -import {Addon} from "@nsm/addon"; +import { Addon } from "@nsm/addon"; import winston from "winston"; async function initAfterLogger(ctx: { logger: winston.Logger }) { - ctx.logger.info('Hello from example addon!'); + ctx.logger.info("Hello from example addon!"); } export default { - name: 'example_addon', - disabled: true, - steps: { - BEFORE_CONFIG: initAfterLogger, - } -} as Addon; \ No newline at end of file + name: "example_addon", + disabled: true, + steps: { + BEFORE_CONFIG: initAfterLogger, + }, +} as Addon; diff --git a/babel.config.js b/babel.config.js index 9127806..dd242dc 100644 --- a/babel.config.js +++ b/babel.config.js @@ -1,6 +1,6 @@ module.exports = { - presets: [ - ['@babel/preset-env', {targets: {node: 'current'}}], - '@babel/preset-typescript', - ], -}; \ No newline at end of file + presets: [ + ["@babel/preset-env", { targets: { node: "current" } }], + "@babel/preset-typescript", + ], +}; diff --git a/docker-compose.yml b/docker-compose.yml index 19fe4b6..fe72ad5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,14 +2,14 @@ services: nsm: build: . volumes: - - '/var/run/docker.sock:/var/run/docker.sock' + - "/var/run/docker.sock:/var/run/docker.sock" ports: - - '3000:3000' + - "3000:3000" extra_hosts: - - 'docker.host.internal:host-gateway' + - "docker.host.internal:host-gateway" environment: - - 'CONFIG_DOCKER_HOST=///var/run/docker.sock' - - 'DATABASE_URL=mysql://root:test@db:3306/nsm' + - "CONFIG_DOCKER_HOST=///var/run/docker.sock" + - "DATABASE_URL=mysql://root:test@db:3306/nsm" depends_on: db: condition: service_healthy @@ -35,7 +35,7 @@ services: MARIADB_ROOT_PASSWORD: test MARIADB_DATABASE: nsm ports: - - '3306:3306' + - "3306:3306" volumes: - nsm_db:/var/lib/mysql healthcheck: @@ -46,4 +46,4 @@ services: start_period: 10s volumes: - nsm_db: \ No newline at end of file + nsm_db: diff --git a/index.ts b/index.ts index 4c48f2c..a458ae6 100644 --- a/index.ts +++ b/index.ts @@ -1,10 +1,10 @@ -import {init} from "@nsm/app"; -import {postInit} from "@nsm/cleanup"; +import { init } from "@nsm/app"; +import { postInit } from "@nsm/cleanup"; import server from "@nsm/server"; init(server) - // Run some cleanup tasks and register handlers - .then(postInit) - .catch((e) => { - console.log(e); - }); + // Run some cleanup tasks and register handlers + .then(postInit) + .catch((e) => { + console.log(e); + }); diff --git a/installTempDeps.js b/installTempDeps.js index a93900e..a6afbb1 100644 --- a/installTempDeps.js +++ b/installTempDeps.js @@ -1,18 +1,20 @@ const fs = require("fs"); const npm = require("npm"); -console.log('Preinstalling dependencies for build...'); +console.log("Preinstalling dependencies for build..."); npm.load().then(() => { - for (let addon of fs.readdirSync(process.cwd() + '/addons')) { - const libFPath = process.cwd() + '/addons/' + addon + '/libraries.txt'; - if (!fs.existsSync(libFPath)) { - continue; - } - const libs = fs.readFileSync(libFPath, 'utf8').split('\n') - .map((lib) => lib.split('=')[0] + '@' + lib.split('=')[1]); - npm.commands.install(libs, (err) => { - console.log(err); - }); + for (let addon of fs.readdirSync(process.cwd() + "/addons")) { + const libFPath = process.cwd() + "/addons/" + addon + "/libraries.txt"; + if (!fs.existsSync(libFPath)) { + continue; } -}); \ No newline at end of file + const libs = fs + .readFileSync(libFPath, "utf8") + .split("\n") + .map((lib) => lib.split("=")[0] + "@" + lib.split("=")[1]); + npm.commands.install(libs, (err) => { + console.log(err); + }); + } +}); diff --git a/jest.config.js b/jest.config.js index 216f8fc..14ef211 100644 --- a/jest.config.js +++ b/jest.config.js @@ -1,13 +1,8 @@ -const tsconfig = require("./tsconfig.json") -const moduleNameMapper = require("tsconfig-paths-jest")(tsconfig) +const tsconfig = require("./tsconfig.json"); +const moduleNameMapper = require("tsconfig-paths-jest")(tsconfig); module.exports = { - moduleNameMapper, - transformIgnorePatterns: [ - "/node_modules/(?!(env-paths)/)", - ], - reporters: [ - 'default', - ['jest-ctrf-json-reporter', {}], - ], -} \ No newline at end of file + moduleNameMapper, + transformIgnorePatterns: ["/node_modules/(?!(env-paths)/)"], + reporters: ["default", ["jest-ctrf-json-reporter", {}]], +}; diff --git a/openapi.yml b/openapi.yml index bed694d..0799f3c 100644 --- a/openapi.yml +++ b/openapi.yml @@ -550,4 +550,4 @@ paths: # TODO: /v1/service/{serviceId}/sessions # TODO: /v1/service/{serviceId}/logs -# TODO: /v1/session/{sessionId}/logs \ No newline at end of file +# TODO: /v1/session/{sessionId}/logs diff --git a/package.json b/package.json index 6ad3e3b..14a24ad 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "check-disk-space": "^3.4.0", "express-ws": "^5.0.2", "jest-ctrf-json-reporter": "^0.0.9", + "prettier": "3.8.3", "prisma": "^5.10.2", "superagent": "^9.0.2", "supertest": "^7.0.0", diff --git a/resources/config.yml b/resources/config.yml index a610785..53b08fd 100644 --- a/resources/config.yml +++ b/resources/config.yml @@ -2,13 +2,13 @@ # with CONFIG_ format. # ID of this node. Should be unique. -node_id: 'main' +node_id: "main" # Listen port. port: 3000 # Security # Supported types: 'none', 'auth_token' -auth: 'none' -docker_host: 'unix:///var/run/docker.sock' +auth: "none" +docker_host: "unix:///var/run/docker.sock" # Override resources path if needed. # By default, an explicit system-specific data path is used. -# resources_path: '/srv/resources' \ No newline at end of file +# resources_path: '/srv/resources' diff --git a/resources/template/example/example_settings.yml b/resources/template/example/example_settings.yml index 3b8ee85..f351972 100644 --- a/resources/template/example/example_settings.yml +++ b/resources/template/example/example_settings.yml @@ -1,5 +1,5 @@ -name: 'Example' -description: 'An Example template' +name: "Example" +description: "An Example template" port_range: min: 25565 max: 35565 @@ -10,8 +10,8 @@ defaults: disk: 2000000000 # bytes meta: # Stop command to be sent in stop signal endpoint - stopCmd: 'stop' + stopCmd: "stop" # Optional ENV vars, and their default values env: - STARTUP_FILE: 'server.jar' - JAVA_VERSION: '' # Required option \ No newline at end of file + STARTUP_FILE: "server.jar" + JAVA_VERSION: "" # Required option diff --git a/resources/template/test/test_settings.yml b/resources/template/test/test_settings.yml index d8e6f1d..1b222b8 100644 --- a/resources/template/test/test_settings.yml +++ b/resources/template/test/test_settings.yml @@ -1,5 +1,5 @@ -name: 'Test' -description: 'A Test template' +name: "Test" +description: "A Test template" port_range: min: 22222 max: 33333 @@ -9,6 +9,6 @@ defaults: disk: 2000000000 # bytes meta: # Stop command to be sent in stop signal endpoint - stopCmd: 'stop' + stopCmd: "stop" # Optional ENV vars, and their default values -env: {} \ No newline at end of file +env: {} diff --git a/src/addon.ts b/src/addon.ts index 9fbe55e..81f8435 100644 --- a/src/addon.ts +++ b/src/addon.ts @@ -1,112 +1,123 @@ import winston from "winston"; -import {AppContext} from "./app"; +import { AppContext } from "./app"; import * as fs from "fs"; import npm from "npm"; import * as http from "http"; -import {isDebug} from "./helpers"; -import {createLogger} from "./logger"; +import { isDebug } from "./helpers"; +import { createLogger } from "./logger"; type FunctionTypes = { - 'BEFORE_CONFIG': (ctx: { logger: winston.Logger }) => Promise; - 'BEFORE_DB': (ctx: { logger: winston.Logger, appConfig: any }) => Promise; - 'BEFORE_ENGINE': (ctx: AppContext) => Promise; - 'BEFORE_SECURITY': (ctx: AppContext) => Promise; - 'BEFORE_ROUTES': (ctx: AppContext) => Promise; - 'BEFORE_SERVER': (ctx: AppContext) => Promise; - 'BOOT': (ctx: AppContext, srv: http.Server) => Promise; - 'EXIT': (ctx: AppContext) => Promise; -} + BEFORE_CONFIG: (ctx: { logger: winston.Logger }) => Promise; + BEFORE_DB: (ctx: { logger: winston.Logger; appConfig: any }) => Promise; + BEFORE_ENGINE: (ctx: AppContext) => Promise; + BEFORE_SECURITY: (ctx: AppContext) => Promise; + BEFORE_ROUTES: (ctx: AppContext) => Promise; + BEFORE_SERVER: (ctx: AppContext) => Promise; + BOOT: (ctx: AppContext, srv: http.Server) => Promise; + EXIT: (ctx: AppContext) => Promise; +}; export type Moment = keyof FunctionTypes; export type AddonSteps = { - [key in Moment]: FunctionTypes[key]; + [key in Moment]: FunctionTypes[key]; }; export type Addon = { - name: string, - briefName?: string, - author?: string, - version?: string, - disabled?: boolean, - steps: AddonSteps, -} + name: string; + briefName?: string; + author?: string; + version?: string; + disabled?: boolean; + steps: AddonSteps; +}; async function initNpm() { - await npm.load(); - npm.config.set('save', false); - npm.config.set('save-dev', false); + await npm.load(); + npm.config.set("save", false); + npm.config.set("save-dev", false); } // Installs dependencies written in libraries.txt -async function installLibs(logger: winston.Logger, libs: { [key: string]: string }) { - const libsArray = Object.keys(libs).map((key) => key + '@' + libs[key]); - logger.info(`Installing ${libsArray.join(', ')}`); - await new Promise((resolve, reject) => { - npm.commands.install(libsArray, (err) => { - if (err) { - reject(err); - } else { - resolve(true); - } - }); +async function installLibs( + logger: winston.Logger, + libs: { [key: string]: string }, +) { + const libsArray = Object.keys(libs).map((key) => key + "@" + libs[key]); + logger.info(`Installing ${libsArray.join(", ")}`); + await new Promise((resolve, reject) => { + npm.commands.install(libsArray, (err) => { + if (err) { + reject(err); + } else { + resolve(true); + } }); + }); } // Load addons export default async function (logger: winston.Logger) { - // Load NPM client - await initNpm(); + // Load NPM client + await initNpm(); - const addons: Addon[] = []; - // Loop addon dirs - for (const dir of ( - // Directories array - fs.readdirSync(__dirname + '/../addons') - .map((dir) => __dirname + '/../addons/' + dir) - .filter((file) => fs.existsSync(file + '/index.js')) - )) { - if (dir.endsWith('example_addon')) { - // Skip default example addon - continue; - } - logger.info(`Loading addon from ${dir}`); - if (fs.existsSync(dir + '/libraries.txt')) { - await installLibs(logger, ( - // Libraries mapped - fs.readFileSync(dir + '/libraries.txt', 'utf8') - .split('\n') - .filter((lib) => lib.includes("=")) - .map((lib) => lib.split('=')) - .reduce((acc, [name, version]) => { - acc[name] = version.replace('\r', ''); - return acc; - }, {} as { [key: string]: string }) - )); - } + const addons: Addon[] = []; + // Loop addon dirs + // Directories array + for (const dir of fs + .readdirSync(__dirname + "/../addons") + .map((dir) => __dirname + "/../addons/" + dir) + .filter((file) => fs.existsSync(file + "/index.js"))) { + if (dir.endsWith("example_addon")) { + // Skip default example addon + continue; + } + logger.info(`Loading addon from ${dir}`); + if (fs.existsSync(dir + "/libraries.txt")) { + await installLibs( + logger, + // Libraries mapped + fs + .readFileSync(dir + "/libraries.txt", "utf8") + .split("\n") + .filter((lib) => lib.includes("=")) + .map((lib) => lib.split("=")) + .reduce( + (acc, [name, version]) => { + acc[name] = version.replace("\r", ""); + return acc; + }, + {} as { [key: string]: string }, + ), + ); + } - const addon = require(dir + '/index.js').default as Addon; - if (!addon.disabled) { - addons.push(addon); + const addon = require(dir + "/index.js").default as Addon; + if (!addon.disabled) { + addons.push(addon); - const { name, author, version } = addon; + const { name, author, version } = addon; - logger.info(`Loaded addon ${name}${author ? ` by ${author}` : ``}${version ? ` (v${version})` : ``}`); - } + logger.info( + `Loaded addon ${name}${author ? ` by ${author}` : ``}${version ? ` (v${version})` : ``}`, + ); } - return (step: T, ctx: any, ...args: any[]) => { - if (isDebug()) { - logger.info(`Running step ${step}`); - } - addons - .filter((addon) => addon.steps[step]) - .forEach(addon => { - const f = addon.steps[step]; - // Make temporary duplicate - const ctxAddon = { ...ctx }; - if (ctxAddon.logger) { - // Make custom logger for each addon - ctxAddon.logger = createLogger({ label: addon.briefName ?? addon.name }); - } - f.apply(f, [ctxAddon, ...args]) - }); + } + return (step: T, ctx: any, ...args: any[]) => { + if (isDebug()) { + logger.info(`Running step ${step}`); } -} \ No newline at end of file + addons + .filter((addon) => addon.steps[step]) + .forEach((addon) => { + const f = addon.steps[step]; + // Make temporary duplicate + const ctxAddon = { ...ctx }; + if (ctxAddon.logger) { + // Make custom logger for each addon + ctxAddon.logger = createLogger({ + label: addon.briefName ?? addon.name, + }); + } + f.apply(f, [ctxAddon, ...args]); + }); + }; +} diff --git a/src/app.ts b/src/app.ts index 6d4d4b5..cbc4b90 100644 --- a/src/app.ts +++ b/src/app.ts @@ -1,6 +1,10 @@ import dotenv from "dotenv"; -import {loadAppConfig} from "@nsm/config"; -import {init as initFileStructure, getResourcesTargetPath, prepareFolders} from "@nsm/filestructure"; +import { loadAppConfig } from "@nsm/config"; +import { + init as initFileStructure, + getResourcesTargetPath, + prepareFolders, +} from "@nsm/filestructure"; // Load .env dotenv.config(); @@ -9,68 +13,78 @@ dotenv.config(); const appConfig = loadAppConfig(); initFileStructure(appConfig); -import {Router} from 'express'; -import {Database} from "@nsm/database"; -import {ServiceManager} from "@nsm/engine"; +import { Router } from "express"; +import { Database } from "@nsm/database"; +import { ServiceManager } from "@nsm/engine"; import loadAddons from "./addon"; -import loadAppRoutes from '@nsm/router'; -import createDbManager from '@nsm/database'; +import loadAppRoutes from "@nsm/router"; +import createDbManager from "@nsm/database"; import loadSecurity from "@nsm/security"; import * as manager from "@nsm/engine/manager"; import * as sessionManager from "@nsm/engine/session"; import * as logging from "./logger"; import winston from "winston"; -import {Application} from "express-ws"; +import { Application } from "express-ws"; import fs from "fs"; -import {middleLayer} from "@nsm/engine/middle"; -import {SessionManager} from "@nsm/engine/session"; -import {mkdirResource, saveResource} from "@nsm/resources"; +import { middleLayer } from "@nsm/engine/middle"; +import { SessionManager } from "@nsm/engine/session"; +import { mkdirResource, saveResource } from "@nsm/resources"; import path from "path"; -import {AppConfig} from "@nsm/config"; +import { AppConfig } from "@nsm/config"; export type AppBootContext = AppContext & { steps: any }; // Passed context to the routes export type AppContext = { - router: Router; - manager: ServiceManager; - sessionManager: SessionManager; - database: Database; - appConfig: AppConfig; - logger: winston.Logger; - debug: boolean; + router: Router; + manager: ServiceManager; + sessionManager: SessionManager; + database: Database; + appConfig: AppConfig; + logger: winston.Logger; + debug: boolean; }; export type AppBootOptions = { - test?: boolean; -} + test?: boolean; +}; export let currentContext: AppContext; function initGlobalLogger() { - logging.createLatestLogFile(); + logging.createLatestLogFile(); - return logging.createLogger(); + return logging.createLogger(); } // Decorate all manager functions except those excluded to disallow using them // before manager.engine is initialized. This is necessary as the manager is being // used (mainly for expandEngine()) even before manager.init() is called. function managerForUnsafeUse() { - const excludeKeys: (keyof ServiceManager)[] = ["expandEngine", "initEngineForcibly", "engine"]; - // - const managerRef = { ...manager }; - const handler: ProxyHandler = { - get(target, prop, receiver) { - // If it's key of base manager, not expanded object and is not excluded, deny access - if ((Object.keys(managerRef) as any[]).includes(prop) && !(excludeKeys as any[]).includes(prop)) { - throw new Error("ServiceManager is not initialized yet! " + - "You can only access those members now: " + excludeKeys.join(", ")); - } - return Reflect.get(target, prop, receiver); - } - } - return new Proxy(manager, handler); + const excludeKeys: (keyof ServiceManager)[] = [ + "expandEngine", + "initEngineForcibly", + "engine", + ]; + // + const managerRef = { ...manager }; + const handler: ProxyHandler = { + get(target, prop, receiver) { + // If it's key of base manager, not expanded object and is not excluded, deny access + if ( + (Object.keys(managerRef) as any[]).includes(prop) && + !(excludeKeys as any[]).includes(prop) + ) { + throw new Error( + "ServiceManager is not initialized yet! " + + "You can only access those members now: " + + excludeKeys.join(", "), + ); + } + return Reflect.get(target, prop, receiver); + }, + }; + return new Proxy(manager, handler); } /** @@ -79,73 +93,79 @@ function managerForUnsafeUse() { * @param router The app router. * @param options The optional boot options. */ -export const init = async (router: Application, options?: AppBootOptions): Promise => { - // Prepare logging - const logger = initGlobalLogger(); - - prepareFolders(); - - // Prepare templates folder - mkdirResource("templates"); - if (options?.test === true) { - prepareTestResources(); // Copy resources for test - } - - // Load addon steps - const steps = await loadAddons(logger); - - steps('BEFORE_CONFIG', { logger }); - - // Database connection layer - steps('BEFORE_DB', { logger, appConfig }); - const database = createDbManager(); - - // Temporarily lock manager until it's initialized - const ctx = currentContext = { - router, - manager: managerForUnsafeUse(), - sessionManager, - database, - appConfig, - logger, - debug: process.env.DEBUG === 'true', - }; - - // Service (virtualization) layer - steps('BEFORE_ENGINE', ctx); - await manager.init(database, appConfig, logger); - - // Bring back original manager - ctx.manager = currentContext.manager = middleLayer(manager); - - // Load security - steps('BEFORE_SECURITY', ctx); - await loadSecurity(ctx); - - // Load HTTP routes - steps('BEFORE_ROUTES', ctx); - await loadAppRoutes(ctx); - - // Start the server - steps('BEFORE_SERVER', ctx); - - let srv = undefined; - if (options?.test == undefined || options.test == false) { - logger.info(`Starting server`); - srv = router.listen(appConfig.getPort(), () => { - logger.info(`Server started on port ${appConfig.getPort()}`); - }); - } - steps('BOOT', ctx, srv); - return { ...ctx, steps }; -} +export const init = async ( + router: Application, + options?: AppBootOptions, +): Promise => { + // Prepare logging + const logger = initGlobalLogger(); + + prepareFolders(); + + // Prepare templates folder + mkdirResource("templates"); + if (options?.test === true) { + prepareTestResources(); // Copy resources for test + } + + // Load addon steps + const steps = await loadAddons(logger); + + steps("BEFORE_CONFIG", { logger }); + + // Database connection layer + steps("BEFORE_DB", { logger, appConfig }); + const database = createDbManager(); + + // Temporarily lock manager until it's initialized + const ctx = (currentContext = { + router, + manager: managerForUnsafeUse(), + sessionManager, + database, + appConfig, + logger, + debug: process.env.DEBUG === "true", + }); + + // Service (virtualization) layer + steps("BEFORE_ENGINE", ctx); + await manager.init(database, appConfig, logger); + + // Bring back original manager + ctx.manager = currentContext.manager = middleLayer(manager); + + // Load security + steps("BEFORE_SECURITY", ctx); + await loadSecurity(ctx); + + // Load HTTP routes + steps("BEFORE_ROUTES", ctx); + await loadAppRoutes(ctx); + + // Start the server + steps("BEFORE_SERVER", ctx); + + let srv = undefined; + if (options?.test == undefined || options.test == false) { + logger.info(`Starting server`); + srv = router.listen(appConfig.getPort(), () => { + logger.info(`Server started on port ${appConfig.getPort()}`); + }); + } + steps("BOOT", ctx, srv); + return { ...ctx, steps }; +}; const prepareTestResources = () => { - if (fs.existsSync(path.join(getResourcesTargetPath(), 'templates', 'test'))) { - return; - } - - saveResource('template/test/test_settings.yml', 'templates/test/settings.yml') - saveResource('template/test/test_dockerfile', 'templates/test/Dockerfile') - saveResource('template/test/test_nsmignore', 'templates/test/.nsmignore') -} \ No newline at end of file + if (fs.existsSync(path.join(getResourcesTargetPath(), "templates", "test"))) { + return; + } + + saveResource( + "template/test/test_settings.yml", + "templates/test/settings.yml", + ); + saveResource("template/test/test_dockerfile", "templates/test/Dockerfile"); + saveResource("template/test/test_nsmignore", "templates/test/.nsmignore"); +}; diff --git a/src/cleanup.ts b/src/cleanup.ts index fd65777..7f8e095 100644 --- a/src/cleanup.ts +++ b/src/cleanup.ts @@ -1,47 +1,49 @@ -import {AppBootContext} from "@nsm/app"; -import {setStatus} from "@nsm/server"; -import {resolveSequentially} from "@nsm/util/promises"; -import {setStopping} from "@nsm/engine/asyncp"; +import { AppBootContext } from "@nsm/app"; +import { setStatus } from "@nsm/server"; +import { resolveSequentially } from "@nsm/util/promises"; +import { setStopping } from "@nsm/engine/asyncp"; let active = false; const cleanup = (ctx: AppBootContext, exit?: boolean) => { - const { manager, logger, steps } = ctx; - - if (active == true) { - return; - } - - active = true; + const { manager, logger, steps } = ctx; + + if (active == true) { + return; + } + + active = true; + if (exit == true) { + logger.info("SIGINT" + ": Executing stop sequence, please wait"); + setStatus("stopping"); + setStopping(); + } + + resolveSequentially( + ...(exit == true + ? [ + // Those steps that should only be called on exit + () => steps("EXIT", ctx), + ] + : []), + () => manager.stopRunning(), + ).then(() => { if (exit == true) { - logger.info('SIGINT' + ': Executing stop sequence, please wait'); - setStatus("stopping"); - setStopping(); + process.exit(0); } - - resolveSequentially( - ...(exit == true ? [ - // Those steps that should only be called on exit - () => steps('EXIT', ctx) - ] : []), - () => manager.stopRunning() - ).then(() => { - if (exit == true) { - process.exit(0); - } - }); -} + }); +}; export const postInit = (ctx: AppBootContext) => { - // Cleanup on start - cleanup(ctx); - - // Handle exit - process.on('exit', () => { - // Cleanup on exit - cleanup(ctx, true); - }); - - // Debug info - ctx.logger.debug('Signal handlers'); -} \ No newline at end of file + // Cleanup on start + cleanup(ctx); + + // Handle exit + process.on("exit", () => { + // Cleanup on exit + cleanup(ctx, true); + }); + + // Debug info + ctx.logger.debug("Signal handlers"); +}; diff --git a/src/config.ts b/src/config.ts index a032dbd..e4ec708 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,7 +1,7 @@ -import {loadYamlFile} from "@nsm/util/yaml"; +import { loadYamlFile } from "@nsm/util/yaml"; import path from "path"; -import {currentPaths} from "@nsm/filestructure"; -import {saveResource} from "@nsm/resources"; +import { currentPaths } from "@nsm/filestructure"; +import { saveResource } from "@nsm/resources"; import z from "zod"; export interface AppConfig { @@ -13,7 +13,7 @@ export interface AppConfig { getDockerHost(): string; - getResourcesPath(): string|undefined; + getResourcesPath(): string | undefined; } /** @@ -22,14 +22,16 @@ export interface AppConfig { * @author ZorTik */ export class YamlAppConfig implements AppConfig { - private static readonly schema: z.ZodObject = z.object({ - node_id: z.string(), - // Coerce port to auto-parse from env if overwritten - port: z.coerce.number().int().positive(), - auth: z.string(), - docker_host: z.string(), - resources_path: z.string().optional() - }).strict(); + private static readonly schema: z.ZodObject = z + .object({ + node_id: z.string(), + // Coerce port to auto-parse from env if overwritten + port: z.coerce.number().int().positive(), + auth: z.string(), + docker_host: z.string(), + resources_path: z.string().optional(), + }) + .strict(); private readonly data: any; @@ -62,19 +64,19 @@ export class YamlAppConfig implements AppConfig { private validate = () => { const result = YamlAppConfig.schema.safeParse(this.data); if (!result.success) { - throw new Error('Invalid config file. ' + result.error.toString()); + throw new Error("Invalid config file. " + result.error.toString()); } - } + }; private static loadData = () => { // Copy if it does not exist - saveResource('config.yml', 'config.yml', true, currentPaths.config); + saveResource("config.yml", "config.yml", true, currentPaths.config); - const config = loadYamlFile(path.join(currentPaths.config, 'config.yml')); + const config = loadYamlFile(path.join(currentPaths.config, "config.yml")); for (let key in YamlAppConfig.schema.shape) { // Overwrite with env variable if exists. // Sync - const envKey = 'CONFIG_' + key.toUpperCase(); + const envKey = "CONFIG_" + key.toUpperCase(); if (process.env[envKey]) { config[key] = process.env[envKey]; } else if (config[key]) { @@ -82,9 +84,9 @@ export class YamlAppConfig implements AppConfig { } } return config; - } + }; } export const loadAppConfig = (): AppConfig => { return new YamlAppConfig(); -} \ No newline at end of file +}; diff --git a/src/database/image.ts b/src/database/image.ts index 189a778..3c99fc8 100644 --- a/src/database/image.ts +++ b/src/database/image.ts @@ -1,12 +1,12 @@ -import {ImageRepository} from "@nsm/database/models"; -import {optionsDiffer} from "@nsm/engine/image"; -import {PrismaClient} from "@prisma/client"; +import { ImageRepository } from "@nsm/database/models"; +import { optionsDiffer } from "@nsm/engine/image"; +import { PrismaClient } from "@prisma/client"; let client: PrismaClient; export const init = (client_: PrismaClient) => { client = client_; -} +}; export const saveImage: ImageRepository["saveImage"] = async (info) => { const { id, templateId, hash, buildOptions } = info; @@ -19,24 +19,30 @@ export const saveImage: ImageRepository["saveImage"] = async (info) => { hash, buildOptions: { deleteMany: {}, - create: Object.entries(buildOptions).map(([key, value]) => ({ key, value })), - } + create: Object.entries(buildOptions).map(([key, value]) => ({ + key, + value, + })), + }, }, create: { id, templateId, hash, buildOptions: { - create: Object.entries(buildOptions).map(([key, value]) => ({ key, value })), - } - } + create: Object.entries(buildOptions).map(([key, value]) => ({ + key, + value, + })), + }, + }, }); return true; } catch (e) { console.log(e); return false; } -} +}; export const getImage: ImageRepository["getImage"] = async (id) => { const image = await client.image.findUnique({ @@ -44,58 +50,63 @@ export const getImage: ImageRepository["getImage"] = async (id) => { include: { buildOptions: { select: { key: true, value: true }, - } - } + }, + }, }); if (image) { const buildOptions = {}; - image.buildOptions.forEach((option) => buildOptions[option.key] = option.value); + image.buildOptions.forEach( + (option) => (buildOptions[option.key] = option.value), + ); return { ...image, buildOptions, - } + }; } else { return undefined; } -} +}; export const deleteImage: ImageRepository["deleteImage"] = async (id) => { try { await client.image.delete({ - where: { id } + where: { id }, }); return true; } catch (e) { - if (e.code !== 'P2025') { + if (e.code !== "P2025") { console.log(e); } return false; } -} +}; -export const listImagesByOptions: ImageRepository["listImagesByOptions"] = async (templateId, buildOptions) => { - return ( - client.image.findMany({ - include: { - buildOptions: { - select: { key: true, value: true }, - } - } - }) - ).then((images) => ( - images.map(image => ({ - ...image, - buildOptions: image.buildOptions.reduce((acc, option) => { - acc[option.key] = option.value; - return acc; - }, {}) - })) - )) - .then((images) => ( - images.filter( - (image) => image.templateId === templateId && !optionsDiffer(image.buildOptions, buildOptions) +export const listImagesByOptions: ImageRepository["listImagesByOptions"] = + async (templateId, buildOptions) => { + return client.image + .findMany({ + include: { + buildOptions: { + select: { key: true, value: true }, + }, + }, + }) + .then((images) => + images.map((image) => ({ + ...image, + buildOptions: image.buildOptions.reduce((acc, option) => { + acc[option.key] = option.value; + return acc; + }, {}), + })), ) - )); -} \ No newline at end of file + .then((images) => + images.filter( + (image) => + image.templateId === templateId && + !optionsDiffer(image.buildOptions, buildOptions), + ), + ); + }; diff --git a/src/database/index.ts b/src/database/index.ts index e8bbdee..dd7dfaa 100644 --- a/src/database/index.ts +++ b/src/database/index.ts @@ -1,5 +1,5 @@ -import {Database} from "./models"; -import {PrismaClient} from "@prisma/client"; +import { Database } from "./models"; +import { PrismaClient } from "@prisma/client"; import * as permaRepository from "./perma"; import * as metaRepository from "./meta"; @@ -8,31 +8,31 @@ import * as imageRepository from "./image"; import * as sessionRepository from "./session"; import * as serviceLogRepository from "./serviceLog"; -export * from './models'; +export * from "./models"; export default function (client?: PrismaClient): Database { - if (!client) { - client = new PrismaClient(); - } + if (!client) { + client = new PrismaClient(); + } - // Propagate client - ( - [ - permaRepository, - metaRepository, - serviceMetaRepository, - imageRepository, - sessionRepository, - serviceLogRepository - ] as unknown as { init: (client: PrismaClient) => void }[] - ).forEach(repository => repository.init(client)); + // Propagate client + ( + [ + permaRepository, + metaRepository, + serviceMetaRepository, + imageRepository, + sessionRepository, + serviceLogRepository, + ] as unknown as { init: (client: PrismaClient) => void }[] + ).forEach((repository) => repository.init(client)); - return { - permaRepository, - metaRepository, - serviceMetaRepository, - imageRepository, - sessionRepository, - serviceLogRepository - } -} \ No newline at end of file + return { + permaRepository, + metaRepository, + serviceMetaRepository, + imageRepository, + sessionRepository, + serviceLogRepository, + }; +} diff --git a/src/database/meta.ts b/src/database/meta.ts index 738e840..5d50753 100644 --- a/src/database/meta.ts +++ b/src/database/meta.ts @@ -1,13 +1,16 @@ -import {PrismaClient} from "@prisma/client"; -import {MetaRepository} from "@nsm/database/models"; +import { PrismaClient } from "@prisma/client"; +import { MetaRepository } from "@nsm/database/models"; let client: PrismaClient; export const init = (client_: PrismaClient) => { client = client_; -} +}; -export const getMetaVal: MetaRepository["getMetaVal"] = async (key, defaultVal) => { +export const getMetaVal: MetaRepository["getMetaVal"] = async ( + key, + defaultVal, +) => { try { let meta = await client.meta.findUnique({ where: { key } }); if (!meta) { @@ -19,6 +22,6 @@ export const getMetaVal: MetaRepository["getMetaVal"] = async (key, defaultVal) return meta.value; } catch (e) { console.log(e); - return ''; + return ""; } -} \ No newline at end of file +}; diff --git a/src/database/models.ts b/src/database/models.ts index c44f9a3..3666fa1 100644 --- a/src/database/models.ts +++ b/src/database/models.ts @@ -1,120 +1,135 @@ export interface Database { - permaRepository: PermaRepository; - metaRepository: MetaRepository; - serviceMetaRepository: ServiceMetaRepository; - imageRepository: ImageRepository; - sessionRepository: SessionRepository; - serviceLogRepository: ServiceLogRepository; + permaRepository: PermaRepository; + metaRepository: MetaRepository; + serviceMetaRepository: ServiceMetaRepository; + imageRepository: ImageRepository; + sessionRepository: SessionRepository; + serviceLogRepository: ServiceLogRepository; } export interface PermaRepository { - savePerma(info: PermaModel): Promise; - deletePerma(serviceId: string): Promise; - getPerma(serviceId: string): Promise; - listPerma(nodeId: string, page?: number, pageSize?: number, meta?: {[key: string]: any}): Promise; - listPermaUsingImage(imageId: string): Promise; - countPerma(nodeId: string): Promise; + savePerma(info: PermaModel): Promise; + deletePerma(serviceId: string): Promise; + getPerma(serviceId: string): Promise; + listPerma( + nodeId: string, + page?: number, + pageSize?: number, + meta?: { [key: string]: any }, + ): Promise; + listPermaUsingImage(imageId: string): Promise; + countPerma(nodeId: string): Promise; } export interface MetaRepository { - getMetaVal(key: string, defaultVal?: string): Promise; + getMetaVal(key: string, defaultVal?: string): Promise; } export interface ServiceMetaRepository { - setServiceMeta(serviceId: string, key: string, value: any): Promise; - getServiceMeta(serviceId: string, key: string): Promise; + setServiceMeta(serviceId: string, key: string, value: any): Promise; + getServiceMeta(serviceId: string, key: string): Promise; } export interface ImageRepository { - saveImage(info: ImageModel): Promise; - getImage(id: string): Promise; - deleteImage(id: string): Promise; - listImagesByOptions(templateId: string, buildOptions: {[key: string]: string}): Promise; + saveImage(info: ImageModel): Promise; + getImage(id: string): Promise; + deleteImage(id: string): Promise; + listImagesByOptions( + templateId: string, + buildOptions: { [key: string]: string }, + ): Promise; } export interface SessionRepository { - createSession(serviceId: string): Promise; + createSession(serviceId: string): Promise; - listSessions(args: ListSessionsArgs): Promise; + listSessions( + args: ListSessionsArgs, + ): Promise; } export type ListSessionsArgs = { - filter?: { - serviceId?: string; - } - sort?: { - by?: 'startedAt' - direction?: 'asc' | 'desc' - } - page?: { - index: number; - size: number; - } -} + filter?: { + serviceId?: string; + }; + sort?: { + by?: "startedAt"; + direction?: "asc" | "desc"; + }; + page?: { + index: number; + size: number; + }; +}; export interface ServiceLogRepository { - createRecords(records: CreateLogRecordArgs[]): Promise; + createRecords(records: CreateLogRecordArgs[]): Promise; - listRecords(args: ListRecordsArgs): Promise; + listRecords( + args: ListRecordsArgs, + ): Promise; } -export type CreateLogRecordArgs = Omit; +export type CreateLogRecordArgs = Omit< + ServiceLogRecordModel, + "id" | "timestamp" +>; export type ListRecordsArgs = { - filter?: { - sessionId?: string; - } - sort?: { - by?: 'timestamp', - direction?: 'asc' | 'desc' - } - page?: { - index: number; - size: number; - } -} + filter?: { + sessionId?: string; + }; + sort?: { + by?: "timestamp"; + direction?: "asc" | "desc"; + }; + page?: { + index: number; + size: number; + }; +}; export type PermaModel = { - serviceId: string; - template: string; - nodeId: string; - imageId?: string; - port: number; - options: { - [key: string]: any; - }; - meta?: { - stopCmd?: string; - }; - env: { - [key: string]: string; - }; - network?: { - address: string; - portsOnly: boolean; - } + serviceId: string; + template: string; + nodeId: string; + imageId?: string; + port: number; + options: { + [key: string]: any; + }; + meta?: { + stopCmd?: string; + }; + env: { + [key: string]: string; + }; + network?: { + address: string; + portsOnly: boolean; + }; }; export type ImageModel = { - id: string; - templateId: string; - hash: string; - buildOptions: { - [key: string]: string; - } -} + id: string; + templateId: string; + hash: string; + buildOptions: { + [key: string]: string; + }; +}; export type ServiceSessionModel = { - id: string; - serviceId: string; - startedAt: Date; -} + id: string; + serviceId: string; + startedAt: Date; +}; export type ServiceLogRecordModel = { - id: bigint; - sessionId: string; - source: 'ENGINE' | 'CONTAINER' - timestamp: Date; - logLevel: string; - message: string; -} \ No newline at end of file + id: bigint; + sessionId: string; + source: "ENGINE" | "CONTAINER"; + timestamp: Date; + logLevel: string; + message: string; +}; diff --git a/src/database/perma.ts b/src/database/perma.ts index 642b5bd..0daa297 100644 --- a/src/database/perma.ts +++ b/src/database/perma.ts @@ -1,11 +1,11 @@ -import {PrismaClient} from "@prisma/client"; -import {PermaModel, PermaRepository} from "@nsm/database/models"; +import { PrismaClient } from "@prisma/client"; +import { PermaModel, PermaRepository } from "@nsm/database/models"; let client: PrismaClient; export const init = (client_: PrismaClient) => { client = client_; -} +}; export const savePerma: PermaRepository["savePerma"] = async (data) => { const { serviceId } = data; @@ -13,32 +13,30 @@ export const savePerma: PermaRepository["savePerma"] = async (data) => { await client.service.upsert({ where: { serviceId }, update: data, - create: data + create: data, }); return true; } catch (e) { console.log(e); return false; } -} +}; export const deletePerma: PermaRepository["deletePerma"] = async ( - serviceId + serviceId, ) => { try { await client.service.delete({ where: { serviceId } }); return true; } catch (e) { - if (e.code !== 'P2025') { + if (e.code !== "P2025") { console.log(e); } return false; } -} +}; -export const getPerma: PermaRepository["getPerma"] = async ( - serviceId -) => { +export const getPerma: PermaRepository["getPerma"] = async (serviceId) => { try { const service = await client.service.findUnique({ where: { serviceId } }); if (!service) { @@ -49,13 +47,13 @@ export const getPerma: PermaRepository["getPerma"] = async ( console.log(e); return undefined; } -} +}; export const listPerma: PermaRepository["listPerma"] = async ( nodeId, page, pageSize, - meta + meta, ) => { try { // SELECT * FROM Service WHERE JSON_EXTRACT(Meta, "$.tag1") IS NOT NULL; @@ -86,24 +84,27 @@ export const listPerma: PermaRepository["listPerma"] = async ( } } return client - .$queryRawUnsafe(`SELECT * FROM Service${where}${pg};`, ...values) - .then(result => result as PermaModel[]); + .$queryRawUnsafe< + PermaModel[] + >(`SELECT * FROM Service${where}${pg};`, ...values) + .then((result) => result as PermaModel[]); } catch (e) { console.log(e); return []; } -} +}; -export const listPermaUsingImage: PermaRepository["listPermaUsingImage"] = async ( - imageId -) => { - try { - return await client.service.findMany({ where: { imageId } }) as PermaModel[]; - } catch (e) { - console.log(e); - return []; - } -} +export const listPermaUsingImage: PermaRepository["listPermaUsingImage"] = + async (imageId) => { + try { + return (await client.service.findMany({ + where: { imageId }, + })) as PermaModel[]; + } catch (e) { + console.log(e); + return []; + } + }; export const countPerma: PermaRepository["countPerma"] = async (nodeId) => { try { @@ -112,4 +113,4 @@ export const countPerma: PermaRepository["countPerma"] = async (nodeId) => { console.log(e); return -1; } -} +}; diff --git a/src/database/serviceLog.ts b/src/database/serviceLog.ts index f79c6d7..64bb6bb 100644 --- a/src/database/serviceLog.ts +++ b/src/database/serviceLog.ts @@ -1,14 +1,14 @@ -import {Prisma, PrismaClient} from "@prisma/client"; -import {ServiceLogRepository} from "@nsm/database/models"; +import { Prisma, PrismaClient } from "@prisma/client"; +import { ServiceLogRepository } from "@nsm/database/models"; let client: PrismaClient; export const init = (client_: PrismaClient) => { client = client_; -} +}; export const createRecords: ServiceLogRepository["createRecords"] = async ( - records + records, ) => { try { await client.serviceLogRecord.createMany({ data: records }); @@ -18,21 +18,19 @@ export const createRecords: ServiceLogRepository["createRecords"] = async ( return false; } -} +}; -export const listRecords: ServiceLogRepository["listRecords"] = async (args) => { - const { - filter, - sort, - page - } = args; +export const listRecords: ServiceLogRepository["listRecords"] = async ( + args, +) => { + const { filter, sort, page } = args; const query: Prisma.ServiceLogRecordFindManyArgs = {}; if (filter?.sessionId) { query.where = filter; } query.orderBy = { - [sort?.by ?? "timestamp"]: sort?.direction ?? "desc" + [sort?.by ?? "timestamp"]: sort?.direction ?? "desc", }; if (page) { query.skip = page.index * page.size; @@ -46,4 +44,4 @@ export const listRecords: ServiceLogRepository["listRecords"] = async (args) => return undefined; } -} \ No newline at end of file +}; diff --git a/src/database/serviceMeta.ts b/src/database/serviceMeta.ts index 8799340..3db62cf 100644 --- a/src/database/serviceMeta.ts +++ b/src/database/serviceMeta.ts @@ -1,38 +1,40 @@ -import {PrismaClient} from "@prisma/client"; -import {ServiceMetaRepository} from "@nsm/database/models"; +import { PrismaClient } from "@prisma/client"; +import { ServiceMetaRepository } from "@nsm/database/models"; let client: PrismaClient; export const init = (client_: PrismaClient) => { client = client_; -} +}; export const setServiceMeta: ServiceMetaRepository["setServiceMeta"] = async ( serviceId, key, - value + value, ) => { try { await client.serviceMeta.upsert({ where: { serviceId }, update: { serviceId, key, value }, - create: { serviceId, key, value } + create: { serviceId, key, value }, }); return true; } catch (e) { console.log(e); return false; } -} +}; export const getServiceMeta: ServiceMetaRepository["getServiceMeta"] = async ( serviceId, - key + key, ) => { - const meta = await client.serviceMeta.findUnique({ where: { serviceId, key } }); + const meta = await client.serviceMeta.findUnique({ + where: { serviceId, key }, + }); if (meta) { return meta.value; } else { return undefined; } -} \ No newline at end of file +}; diff --git a/src/database/session.ts b/src/database/session.ts index 2d33d6e..ae3db43 100644 --- a/src/database/session.ts +++ b/src/database/session.ts @@ -1,15 +1,17 @@ -import {Prisma, PrismaClient} from "@prisma/client"; -import {SessionRepository} from "@nsm/database/models"; +import { Prisma, PrismaClient } from "@prisma/client"; +import { SessionRepository } from "@nsm/database/models"; let client: PrismaClient; export const init = (client_: PrismaClient) => { client = client_; -} +}; -export const createSession: SessionRepository["createSession"] = async (serviceId) => { +export const createSession: SessionRepository["createSession"] = async ( + serviceId, +) => { const data: Prisma.ServiceSessionUncheckedCreateInput = { - serviceId + serviceId, }; try { @@ -19,21 +21,17 @@ export const createSession: SessionRepository["createSession"] = async (serviceI return undefined; } -} +}; export const listSessions: SessionRepository["listSessions"] = async (args) => { - const { - filter, - sort, - page - } = args; + const { filter, sort, page } = args; const query: Prisma.ServiceSessionFindManyArgs = {}; if (filter?.serviceId) { query.where = filter; } query.orderBy = { - [sort?.by ?? "startedAt"]: sort?.direction ?? "desc" + [sort?.by ?? "startedAt"]: sort?.direction ?? "desc", }; if (page) { query.skip = page.index * page.size; @@ -47,4 +45,4 @@ export const listSessions: SessionRepository["listSessions"] = async (args) => { return undefined; } -} \ No newline at end of file +}; diff --git a/src/depend.ts b/src/depend.ts index 63264c3..7acaf0d 100644 --- a/src/depend.ts +++ b/src/depend.ts @@ -1,15 +1,15 @@ const deps: { [id: string]: any } = {}; -export type RegType = 'engine'; // Registration types +export type RegType = "engine"; // Registration types export function setSingleton(key: RegType, obj: any) { - deps[key] = obj; + deps[key] = obj; } -export function getSingleton(key: RegType): T|undefined { - return deps[key]; +export function getSingleton(key: RegType): T | undefined { + return deps[key]; } export function getSingletonOrDef(key: RegType, def: T): T { - return deps[key] ?? def; -} \ No newline at end of file + return deps[key] ?? def; +} diff --git a/src/engine/asyncp.ts b/src/engine/asyncp.ts index 2f94191..181afc7 100644 --- a/src/engine/asyncp.ts +++ b/src/engine/asyncp.ts @@ -15,69 +15,67 @@ let stopping = false; * @returns The unlock function */ export function lockBusyAction(id: string, tp: string) { - reqNotPending(id); - statuses[id] = true; - status_types[id] = tp; // type of action + reqNotPending(id); + statuses[id] = true; + status_types[id] = tp; // type of action - return (err?: any) => { - delete statuses[id]; - delete status_types[id]; + return (err?: any) => { + delete statuses[id]; + delete status_types[id]; - (obs.get(id) ?? []).forEach(o => o(id, tp, err)); - obs.delete(id); + (obs.get(id) ?? []).forEach((o) => o(id, tp, err)); + obs.delete(id); - if (pendingCount() == 0) { - obsAll.forEach(o => o()); - obsAll.splice(0, obsAll.length); - } + if (pendingCount() == 0) { + obsAll.forEach((o) => o()); + obsAll.splice(0, obsAll.length); } + }; } export function whenUnlocked(id: string, cb: UnlockObserver) { - if (isServicePending(id)) { - obs.set(id, obs.get(id) ?? []); - obs.get(id).push(cb); - } else { - cb(id, undefined, undefined); - } + if (isServicePending(id)) { + obs.set(id, obs.get(id) ?? []); + obs.get(id).push(cb); + } else { + cb(id, undefined, undefined); + } } export function whenUnlockedAll(cb: () => void) { - if (pendingCount() > 0) { - obsAll.push(cb); - } else { - cb(); - } + if (pendingCount() > 0) { + obsAll.push(cb); + } else { + cb(); + } } export function lckStatusTp(id: string, tp: string) { - status_types[id] = tp; + status_types[id] = tp; } export function ulckStatusTp(id: string) { - delete status_types[id]; + delete status_types[id]; } export function isServicePending(id: string): boolean { - return statuses[id] || false; + return statuses[id] || false; } -export function getActionType(id: string): string|undefined { - return status_types[id] || undefined; +export function getActionType(id: string): string | undefined { + return status_types[id] || undefined; } export function reqNotPending(id: string) { - if (stopping == false && isServicePending(id)) { - throw new Error('Service is pending another action.'); - } + if (stopping == false && isServicePending(id)) { + throw new Error("Service is pending another action."); + } } export function setStopping() { - stopping = true; + stopping = true; } export function pendingCount() { - return Object.keys(statuses) - .filter(k => statuses[k]) - .length; -} \ No newline at end of file + return Object.keys(statuses).filter((k) => statuses[k]).length; +} diff --git a/src/engine/docker/action/build.ts b/src/engine/docker/action/build.ts index cee4618..125f565 100644 --- a/src/engine/docker/action/build.ts +++ b/src/engine/docker/action/build.ts @@ -2,119 +2,123 @@ import DockerClient from "dockerode"; import fs from "fs"; import path from "path"; import tar from "tar"; -import {MessageListener, ServiceEngine} from "@nsm/engine"; -import {clock} from "@nsm/util/clock"; -import {getRootFilesFiltered} from "@nsm/engine/ignore"; -import {mkdirTemp} from "@nsm/filestructure"; -import {currentContext} from "@nsm/app"; +import { MessageListener, ServiceEngine } from "@nsm/engine"; +import { clock } from "@nsm/util/clock"; +import { getRootFilesFiltered } from "@nsm/engine/ignore"; +import { mkdirTemp } from "@nsm/filestructure"; +import { currentContext } from "@nsm/app"; -async function prepareImage( - args: { - imageName: string|undefined, - client: DockerClient, - arDir: string, - buildDir: string, - env: any, - messageListener?: MessageListener +async function prepareImage(args: { + imageName: string | undefined; + client: DockerClient; + arDir: string; + buildDir: string; + env: any; + messageListener?: MessageListener; +}): Promise { + let { imageName, client, arDir, buildDir, env, messageListener } = args; + + if (!imageName) { + // Generate an unique image name + imageName = + "nsm-template-" + path.basename(buildDir) + "-" + Date.now() + ":latest"; // TODO: better unique name generation, maybe hash of the build context? } -): Promise { - let { - imageName, - client, - arDir, - buildDir, - env, - messageListener - } = args; - if (!imageName) { - // Generate an unique image name - imageName = "nsm-template-" + path.basename(buildDir) + '-' + Date.now() + ':latest'; // TODO: better unique name generation, maybe hash of the build context? + // temp archive + const archive = path.join(arDir, imageName + ".tar"); + try { + // try to delete if there is already a file + fs.unlinkSync(archive); + } catch (e) { + if (!e.message.includes("ENOENT")) { + throw e; } + } - // temp archive - const archive = path.join(arDir, imageName + '.tar'); - try { - // try to delete if there is already a file - fs.unlinkSync(archive); - } catch (e) { - if (!e.message.includes('ENOENT')) { - throw e; - } - } + await tar.c( + { + gzip: false, + file: archive, + cwd: buildDir, + }, + [...getRootFilesFiltered(buildDir)], + ); - await tar.c({ - gzip: false, - file: archive, - cwd: buildDir - }, [...getRootFilesFiltered(buildDir)]); + const imageTag = imageName; + const logs = []; + return new Promise((resolve, reject) => { + const msgHandler = (msg: any) => { + if (Array.isArray(msg)) { + msg.forEach((m) => { + // Push service log record + // TODO: publish log record using messageListener + }); + } else { + // Final message, resolve the promise with the image tag. + resolve(msg); + } + }; + // In container, worker threads are not supported. Or they + // are disabled. + client + .buildImage(archive, { t: imageTag, buildargs: env }) + .then((stream) => { + logs.push("--------- Begin Build Log ---------"); + client.modem.followProgress(stream, (err, res) => { + if (err) { + console.error(err); + } else { + let errorOccurred = false; + res.forEach((r) => { + if (r.errorDetail) { + errorOccurred = true; - const imageTag = imageName; - const logs = []; - return ( - new Promise((resolve, reject) => { - const msgHandler = (msg: any) => { - if (Array.isArray(msg)) { - msg.forEach(m => { - // Push service log record - // TODO: publish log record using messageListener - }); + reject(r.errorDetail); } else { - // Final message, resolve the promise with the image tag. - resolve(msg); - } - } - // In container, worker threads are not supported. Or they - // are disabled. - client.buildImage(archive, { t: imageTag, buildargs: env }).then(stream => { - logs.push('--------- Begin Build Log ---------'); - client.modem.followProgress(stream, (err, res) => { - if (err) { - console.error(err); - } else { - let errorOccurred = false; - res.forEach(r => { - if (r.errorDetail) { - errorOccurred = true; - - reject(r.errorDetail); - } else { - const msg = r.stream?.trim(); + const msg = r.stream?.trim(); - logs.push(msg); - } - }); - if (errorOccurred) { - return; + logs.push(msg); } - logs.push('--------- End Of Build Log ---------\n'); - fs.unlinkSync(archive); - msgHandler(logs); - msgHandler(imageTag); + }); + if (errorOccurred) { + return; } - }); - }); - }).finally(() => { - // Clean up archive file if it still exists - try { - fs.unlinkSync(archive); - } catch (e) { - if (!e.message.includes('ENOENT')) { - console.error('Error cleaning up archive file:', e); - } + logs.push("--------- End Of Build Log ---------\n"); + fs.unlinkSync(archive); + msgHandler(logs); + msgHandler(imageTag); } - }) - ); + }); + }); + }).finally(() => { + // Clean up archive file if it still exists + try { + fs.unlinkSync(archive); + } catch (e) { + if (!e.message.includes("ENOENT")) { + console.error("Error cleaning up archive file:", e); + } + } + }); } -export default function (client: DockerClient): ServiceEngine['build'] { - const arDir = mkdirTemp("archives"); +export default function (client: DockerClient): ServiceEngine["build"] { + const arDir = mkdirTemp("archives"); - return async (imageId, buildDir, options, messageListener) => { - const imageBuildClock = clock(); - const imageTag = await prepareImage({imageName: imageId, client, arDir, buildDir, env: options, messageListener}); - currentContext.logger.info('Image built in ' + imageBuildClock.durFromCreation() + 'ms'); + return async (imageId, buildDir, options, messageListener) => { + const imageBuildClock = clock(); + const imageTag = await prepareImage({ + imageName: imageId, + client, + arDir, + buildDir, + env: options, + messageListener, + }); + currentContext.logger.info( + "Image built in " + imageBuildClock.durFromCreation() + "ms", + ); - return imageTag; - } -} \ No newline at end of file + return imageTag; + }; +} diff --git a/src/engine/docker/action/calcHostUsage.ts b/src/engine/docker/action/calcHostUsage.ts index 4031091..b8b4fd0 100644 --- a/src/engine/docker/action/calcHostUsage.ts +++ b/src/engine/docker/action/calcHostUsage.ts @@ -7,7 +7,7 @@ export default function calcHostUsage(client: DockerClient) { let free_ = 0; let size_ = 0; for (const vol of Volumes) { - if (!vol.Labels || !('nsm' in vol.Labels)) { + if (!vol.Labels || !("nsm" in vol.Labels)) { // Not a NSM volume. continue; } @@ -16,5 +16,5 @@ export default function calcHostUsage(client: DockerClient) { size_ += size; } return [free_, size_]; - } -} \ No newline at end of file + }; +} diff --git a/src/engine/docker/action/cmd.ts b/src/engine/docker/action/cmd.ts index 39e4984..40477d2 100644 --- a/src/engine/docker/action/cmd.ts +++ b/src/engine/docker/action/cmd.ts @@ -1,15 +1,18 @@ -import {DockerServiceEngine, ServiceEngine} from "@nsm/engine"; +import { DockerServiceEngine, ServiceEngine } from "@nsm/engine"; import DockerClient from "dockerode"; -export default function (self: ServiceEngine, _: DockerClient): ServiceEngine['cmd'] { - return async (id, cmd) => { - const watchers = (self as DockerServiceEngine).rws; - // - if (id in watchers) { - watchers[id].write(cmd + '\n'); - return true; - } else { - return false; - } +export default function ( + self: ServiceEngine, + _: DockerClient, +): ServiceEngine["cmd"] { + return async (id, cmd) => { + const watchers = (self as DockerServiceEngine).rws; + // + if (id in watchers) { + watchers[id].write(cmd + "\n"); + return true; + } else { + return false; } -} \ No newline at end of file + }; +} diff --git a/src/engine/docker/action/deletei.ts b/src/engine/docker/action/deletei.ts index 0eceaa1..835e91c 100644 --- a/src/engine/docker/action/deletei.ts +++ b/src/engine/docker/action/deletei.ts @@ -1,10 +1,12 @@ import DockerClient from "dockerode"; -import {ServiceEngine} from "@nsm/engine"; +import { ServiceEngine } from "@nsm/engine"; -export default function deleteImage(client: DockerClient): ServiceEngine["deleteImage"] { +export default function deleteImage( + client: DockerClient, +): ServiceEngine["deleteImage"] { return async (id) => { const image = client.getImage(id); await image.remove(); - } -} \ No newline at end of file + }; +} diff --git a/src/engine/docker/action/deletev.ts b/src/engine/docker/action/deletev.ts index 12f1604..773496c 100644 --- a/src/engine/docker/action/deletev.ts +++ b/src/engine/docker/action/deletev.ts @@ -1,15 +1,18 @@ import DockerClient from "dockerode"; -import {ServiceEngine} from "../../engine"; -import {currentContext} from "../../../app"; +import { ServiceEngine } from "../../engine"; +import { currentContext } from "../../../app"; -export default function (self: ServiceEngine, client: DockerClient): ServiceEngine['deleteVolume'] { - return async (id) => { - try { - await client.getVolume(id).remove(); - return true; - } catch (e) { - currentContext.logger.error(e); - return false; - } +export default function ( + self: ServiceEngine, + client: DockerClient, +): ServiceEngine["deleteVolume"] { + return async (id) => { + try { + await client.getVolume(id).remove(); + return true; + } catch (e) { + currentContext.logger.error(e); + return false; } -} \ No newline at end of file + }; +} diff --git a/src/engine/docker/action/getLabels.ts b/src/engine/docker/action/getLabels.ts index 52f3205..c9d1ae0 100644 --- a/src/engine/docker/action/getLabels.ts +++ b/src/engine/docker/action/getLabels.ts @@ -1,12 +1,12 @@ import DockerClient from "dockerode"; -import {ServiceEngine} from "@nsm/engine"; +import { ServiceEngine } from "@nsm/engine"; -export default function (client: DockerClient): ServiceEngine['getLabels'] { +export default function (client: DockerClient): ServiceEngine["getLabels"] { return async (id) => { const container = client.getContainer(id); const inspect = await container.inspect(); return inspect.Config.Labels; - } -} \ No newline at end of file + }; +} diff --git a/src/engine/docker/action/kill.ts b/src/engine/docker/action/kill.ts index f625b28..8651ea4 100644 --- a/src/engine/docker/action/kill.ts +++ b/src/engine/docker/action/kill.ts @@ -1,20 +1,20 @@ -import {ServiceEngine} from "@nsm/engine"; +import { ServiceEngine } from "@nsm/engine"; import DockerClient from "dockerode"; -export default function (client: DockerClient): ServiceEngine['kill'] { - return async (id) => { - try { - const list = await client.listContainers(); - if (list.map(c => c.Id).includes(id)) { - await client.getContainer(id).kill(); - } +export default function (client: DockerClient): ServiceEngine["kill"] { + return async (id) => { + try { + const list = await client.listContainers(); + if (list.map((c) => c.Id).includes(id)) { + await client.getContainer(id).kill(); + } - return true; - } catch (e) { - if (!e.message.includes('container is not running')) { - console.log(e); - } - return false; - } + return true; + } catch (e) { + if (!e.message.includes("container is not running")) { + console.log(e); + } + return false; } -} \ No newline at end of file + }; +} diff --git a/src/engine/docker/action/listRunning.ts b/src/engine/docker/action/listRunning.ts index d91fff3..d110ab5 100644 --- a/src/engine/docker/action/listRunning.ts +++ b/src/engine/docker/action/listRunning.ts @@ -1,15 +1,13 @@ import DockerClient from "dockerode"; -import {ContainerFilter} from "@nsm/engine"; -import {toDockerFilters} from "@nsm/engine/docker/util/labels"; +import { ContainerFilter } from "@nsm/engine"; +import { toDockerFilters } from "@nsm/engine/docker/util/labels"; export default function listRunningFunc(client: DockerClient) { return async (filter: ContainerFilter) => { const list = await client.listContainers({ all: true, - filters: toDockerFilters(filter) + filters: toDockerFilters(filter), }); - return list - .filter(c => c.State === 'running') - .map(c => c.Id); - } -} \ No newline at end of file + return list.filter((c) => c.State === "running").map((c) => c.Id); + }; +} diff --git a/src/engine/docker/action/listc.ts b/src/engine/docker/action/listc.ts index 00b7831..3c1fa2c 100644 --- a/src/engine/docker/action/listc.ts +++ b/src/engine/docker/action/listc.ts @@ -1,19 +1,22 @@ import DockerClient from "dockerode"; -import {ServiceEngine} from "@nsm/engine"; -import {toDockerFilters} from "@nsm/engine/docker/util/labels"; +import { ServiceEngine } from "@nsm/engine"; +import { toDockerFilters } from "@nsm/engine/docker/util/labels"; -export default function (self: ServiceEngine, client: DockerClient): ServiceEngine['listContainers'] { - return async (filter) => { - try { - const containers = await client.listContainers({ - all: true, - filters: toDockerFilters(filter) - }); +export default function ( + self: ServiceEngine, + client: DockerClient, +): ServiceEngine["listContainers"] { + return async (filter) => { + try { + const containers = await client.listContainers({ + all: true, + filters: toDockerFilters(filter), + }); - return containers.map(c => c.Id); - } catch (e) { - console.log(e); - return []; - } + return containers.map((c) => c.Id); + } catch (e) { + console.log(e); + return []; } -} \ No newline at end of file + }; +} diff --git a/src/engine/docker/action/listp.ts b/src/engine/docker/action/listp.ts index 05c876f..652137e 100644 --- a/src/engine/docker/action/listp.ts +++ b/src/engine/docker/action/listp.ts @@ -1,15 +1,18 @@ import DockerClient from "dockerode"; -import {ServiceEngine} from "../../engine"; +import { ServiceEngine } from "../../engine"; -export default function (self: ServiceEngine, client: DockerClient): ServiceEngine['listAttachedPorts'] { - return async () => { - try { - return (await client.listContainers()) - .map(c => c.Ports.map(p => p.PublicPort)) - .flat(); - } catch (e) { - console.log(e); - return []; - } +export default function ( + self: ServiceEngine, + client: DockerClient, +): ServiceEngine["listAttachedPorts"] { + return async () => { + try { + return (await client.listContainers()) + .map((c) => c.Ports.map((p) => p.PublicPort)) + .flat(); + } catch (e) { + console.log(e); + return []; } -} \ No newline at end of file + }; +} diff --git a/src/engine/docker/action/reattach.ts b/src/engine/docker/action/reattach.ts index cdc9ac3..ed4b075 100644 --- a/src/engine/docker/action/reattach.ts +++ b/src/engine/docker/action/reattach.ts @@ -1,11 +1,22 @@ import DockerClient from "dockerode"; -import {DockerServiceEngine, ServiceEngine, ServiceLogRecord} from "@nsm/engine"; -import {getActionType} from "@nsm/engine/asyncp"; -import {currentContext} from "@nsm/app"; -import {deleteNetwork as doDeleteNetwork, isInNetwork} from "@nsm/networking/manager"; +import { + DockerServiceEngine, + ServiceEngine, + ServiceLogRecord, +} from "@nsm/engine"; +import { getActionType } from "@nsm/engine/asyncp"; +import { currentContext } from "@nsm/app"; +import { + deleteNetwork as doDeleteNetwork, + isInNetwork, +} from "@nsm/networking/manager"; import winston from "winston"; -async function deleteContainer(id: string, client: DockerClient, options: { deleteNetwork?: boolean }) { +async function deleteContainer( + id: string, + client: DockerClient, + options: { deleteNetwork?: boolean }, +) { try { const c = client.getContainer(id); try { @@ -18,7 +29,9 @@ async function deleteContainer(id: string, client: DockerClient, options: { dele const networkId = await isInNetwork(client, id); if (networkId) { // Disconnect this container from the attached network. - await client.getNetwork(networkId).disconnect({ Container: id, Force: true }); + await client + .getNetwork(networkId) + .disconnect({ Container: id, Force: true }); if (options.deleteNetwork == true) { // Delete network if requested. await doDeleteNetwork(client, id); @@ -26,8 +39,11 @@ async function deleteContainer(id: string, client: DockerClient, options: { dele } return true; } catch (e) { - if (e.message.includes('No such container:') || e.message.includes('removal of container')) { - currentContext?.logger.warn('Ignoring error: ' + e.message); + if ( + e.message.includes("No such container:") || + e.message.includes("removal of container") + ) { + currentContext?.logger.warn("Ignoring error: " + e.message); return true; } @@ -36,7 +52,10 @@ async function deleteContainer(id: string, client: DockerClient, options: { dele } } -export default function reattach(self: ServiceEngine, client: DockerClient): ServiceEngine["reattach"] { +export default function reattach( + self: ServiceEngine, + client: DockerClient, +): ServiceEngine["reattach"] { return async (id, listener) => { const container = client.getContainer(id); const logger = currentContext.logger; @@ -45,23 +64,31 @@ export default function reattach(self: ServiceEngine, client: DockerClient): Ser await deleteContainer(container.id, client, { deleteNetwork: true }); await listener.onClose?.(); - } + }; const info = await container.inspect(); if (!info.State.Running) { // If the container is not running, we can delete it right after await handleClosed(); - throw new Error("Container is not running. Maybe it stopped before it could be attached?"); + throw new Error( + "Container is not running. Maybe it stopped before it could be attached?", + ); } - const attachOptions = { stream: true, stdin: true, stdout: true, stderr: true, hijack: true }; + const attachOptions = { + stream: true, + stdin: true, + stdout: true, + stderr: true, + hijack: true, + }; const rws = await container.attach(attachOptions); - rws.on('data', (data) => { + rws.on("data", (data) => { try { - data = Buffer.from(data).toString('ascii'); + data = Buffer.from(data).toString("ascii"); const record: ServiceLogRecord = { - level: 'info', - message: data + level: "info", + message: data, }; listener.onMessage?.(record); @@ -69,8 +96,8 @@ export default function reattach(self: ServiceEngine, client: DockerClient): Ser logger.error("Error producing container output: " + e); } }); // no-op, keepalive - rws.on('end', async () => { - if (getActionType(container.id) != 'stop') { + rws.on("end", async () => { + if (getActionType(container.id) != "stop") { // Stopped from the inside await handleClosed(); @@ -82,6 +109,10 @@ export default function reattach(self: ServiceEngine, client: DockerClient): Ser }); (self as DockerServiceEngine).rws[container.id] = rws; - await listener.onStateChange?.({ id: 'watching_changes', description: 'Watching changes', ready: true }); - } -} \ No newline at end of file + await listener.onStateChange?.({ + id: "watching_changes", + description: "Watching changes", + ready: true, + }); + }; +} diff --git a/src/engine/docker/action/run.ts b/src/engine/docker/action/run.ts index cc22b4d..96572d9 100644 --- a/src/engine/docker/action/run.ts +++ b/src/engine/docker/action/run.ts @@ -1,21 +1,26 @@ import DockerClient from "dockerode"; -import {RunOptions, MetaStorage, ServiceEngine, ServiceState} from "@nsm/engine"; -import {accessNetwork, createNetwork} from "@nsm/networking/manager"; -import {constructObjectLabels} from "@nsm/util/services"; -import {currentContext as ctx} from "@nsm/app"; -import {propagateOptionsToEnv} from "@nsm/engine/docker/util/env"; -import {infoRecord as info} from "@nsm/engine/docker/util/logging"; +import { + RunOptions, + MetaStorage, + ServiceEngine, + ServiceState, +} from "@nsm/engine"; +import { accessNetwork, createNetwork } from "@nsm/networking/manager"; +import { constructObjectLabels } from "@nsm/util/services"; +import { currentContext as ctx } from "@nsm/app"; +import { propagateOptionsToEnv } from "@nsm/engine/docker/util/env"; +import { infoRecord as info } from "@nsm/engine/docker/util/logging"; async function prepareVolume(client: DockerClient, volumeId: string) { try { await client.getVolume(volumeId).inspect(); } catch (e) { - if (e.message.includes('No such')) { + if (e.message.includes("No such")) { await client.createVolume({ Name: volumeId, Labels: { ...constructObjectLabels({ id: volumeId }), - 'nsm.volumeId': volumeId, + "nsm.volumeId": volumeId, }, }); @@ -28,18 +33,18 @@ async function prepareVolume(client: DockerClient, volumeId: string) { async function prepareNetwork( client: DockerClient, - network: RunOptions['network'], + network: RunOptions["network"], meta: MetaStorage, - creatingContainer: boolean + creatingContainer: boolean, ) { - let net: DockerClient.Network|undefined = undefined; + let net: DockerClient.Network | undefined = undefined; if (network && !network.portsOnly) { const metaKey = "net-id"; let netId = await meta.get(metaKey); if (creatingContainer || !netId) { net = await createNetwork(client, network.address); netId = net.id; - if (!await meta.set(metaKey, netId)) { + if (!(await meta.set(metaKey, netId))) { throw new Error("Could not save network data."); } } else { @@ -54,13 +59,14 @@ async function prepareContainer( imageTag: string, volumeId: string, options: RunOptions, - net: DockerClient.Network|undefined + net: DockerClient.Network | undefined, ) { - const {ram, cpu, disk, port, network} = options; - const env = {...options.env}; + const { ram, cpu, disk, port, network } = options; + const env = { ...options.env }; propagateOptionsToEnv(options, env); - const fullPortDef = (port: number) => (network?.portsOnly ? network.address + ":" : "") + port + ""; + const fullPortDef = (port: number) => + (network?.portsOnly ? network.address + ":" : "") + port + ""; // Create container const container = await client.createContainer({ Image: imageTag, @@ -68,15 +74,15 @@ async function prepareContainer( HostConfig: { Memory: ram, CpuShares: cpu, - PortBindings: { [port + '/tcp']: [{HostPort: fullPortDef(port)}] }, + PortBindings: { [port + "/tcp"]: [{ HostPort: fullPortDef(port) }] }, DiskQuota: disk, Mounts: [ { - Type: 'volume', + Type: "volume", Source: client.getVolume(volumeId).name, - Target: '/data', + Target: "/data", ReadOnly: false, - } + }, ], }, Env: Object.entries(env).map(([k, v]) => `${k}=${v}`), @@ -90,41 +96,54 @@ async function prepareContainer( return container; } -const createState = (id: string, description: string, ready?: boolean): ServiceState => { +const createState = ( + id: string, + description: string, + ready?: boolean, +): ServiceState => { return { id, description, - ready: ready ?? false - } + ready: ready ?? false, + }; }; const createErrorState = (description: string): ServiceState => { return { - id: 'error', + id: "error", description, - ready: false - } -} + ready: false, + }; +}; -export default function run(self: ServiceEngine, client: DockerClient): ServiceEngine["run"] { +export default function run( + self: ServiceEngine, + client: DockerClient, +): ServiceEngine["run"] { return async (imageId, volumeId, options, meta, listener) => { let container: DockerClient.Container; // Prepare volume let creating = await prepareVolume(client, volumeId); - await listener.onStateChange?.(createState('preparing_network', 'Preparing network')); + await listener.onStateChange?.( + createState("preparing_network", "Preparing network"), + ); const net = await prepareNetwork(client, options.network, meta, creating); // Port decorator that takes port and according to network changes it to : or keeps the same. - await listener.onStateChange?.(createState('preparing_container', 'Preparing container')); + await listener.onStateChange?.( + createState("preparing_container", "Preparing container"), + ); container = await prepareContainer(client, imageId, volumeId, options, net); - await listener.onStateChange?.(createState('starting_container', 'Starting container')); + await listener.onStateChange?.( + createState("starting_container", "Starting container"), + ); await container.start(); const inspectInfo = await container.inspect(); if (!inspectInfo.State.Running) { // Wait a bit for logs to be available - await new Promise(r => setTimeout(r, 300)); + await new Promise((r) => setTimeout(r, 300)); // Container failed to start, try to get logs and error message // The necessary error will be thrown by reattach call @@ -137,15 +156,20 @@ export default function run(self: ServiceEngine, client: DockerClient): ServiceE }); const msg = logs.toString("utf8"); - await listener.onStateChange?.(createErrorState('Container failed to start')); + await listener.onStateChange?.( + createErrorState("Container failed to start"), + ); await listener.onMessage(info(msg)); } catch (e) { - ctx.logger.error("Error while fetching logs for failed container " + container.id, e); + ctx.logger.error( + "Error while fetching logs for failed container " + container.id, + e, + ); } } await self.reattach(container.id, listener); return container.id; - } -} \ No newline at end of file + }; +} diff --git a/src/engine/docker/action/stat.ts b/src/engine/docker/action/stat.ts index 0cf729a..4672592 100644 --- a/src/engine/docker/action/stat.ts +++ b/src/engine/docker/action/stat.ts @@ -1,10 +1,13 @@ -import {ServiceEngine} from "@nsm/engine"; +import { ServiceEngine } from "@nsm/engine"; import DockerClient from "dockerode"; -import {adaptContainerStatsFromDocker} from "@nsm/util/docker"; +import { adaptContainerStatsFromDocker } from "@nsm/util/docker"; -export default function (self: ServiceEngine, client: DockerClient): ServiceEngine['stat'] { - return async (id) => { - const stats = await client.getContainer(id).stats({ stream: false }); - return adaptContainerStatsFromDocker(id, stats); - } -} \ No newline at end of file +export default function ( + self: ServiceEngine, + client: DockerClient, +): ServiceEngine["stat"] { + return async (id) => { + const stats = await client.getContainer(id).stats({ stream: false }); + return adaptContainerStatsFromDocker(id, stats); + }; +} diff --git a/src/engine/docker/action/statall.ts b/src/engine/docker/action/statall.ts index c8320a2..3316694 100644 --- a/src/engine/docker/action/statall.ts +++ b/src/engine/docker/action/statall.ts @@ -1,9 +1,9 @@ -import {ServiceEngine} from "@nsm/engine"; +import { ServiceEngine } from "@nsm/engine"; -export default function (self: ServiceEngine): ServiceEngine['statAll'] { - return async (filter) => { - const containers = await self.listContainers(filter); +export default function (self: ServiceEngine): ServiceEngine["statAll"] { + return async (filter) => { + const containers = await self.listContainers(filter); - return Promise.all(containers.map(c => self.stat(c))); - } -} \ No newline at end of file + return Promise.all(containers.map((c) => self.stat(c))); + }; +} diff --git a/src/engine/docker/action/stop.ts b/src/engine/docker/action/stop.ts index 7366526..2f0142c 100644 --- a/src/engine/docker/action/stop.ts +++ b/src/engine/docker/action/stop.ts @@ -1,20 +1,20 @@ import DockerClient from "dockerode"; -import {ServiceEngine} from "@nsm/engine"; +import { ServiceEngine } from "@nsm/engine"; -export default function (client: DockerClient): ServiceEngine['stop'] { - return async (id) => { - try { - const list = await client.listContainers(); - if (list.map(c => c.Id).includes(id)) { - await client.getContainer(id).stop({ signal: 'SIGINT' }); - } +export default function (client: DockerClient): ServiceEngine["stop"] { + return async (id) => { + try { + const list = await client.listContainers(); + if (list.map((c) => c.Id).includes(id)) { + await client.getContainer(id).stop({ signal: "SIGINT" }); + } - return true; - } catch (e) { - if (!e.message.includes('container already stopped')) { - console.log(e); - } - return false; - } + return true; + } catch (e) { + if (!e.message.includes("container already stopped")) { + console.log(e); + } + return false; } -} \ No newline at end of file + }; +} diff --git a/src/engine/docker/client.ts b/src/engine/docker/client.ts index 6833e0f..62cf004 100644 --- a/src/engine/docker/client.ts +++ b/src/engine/docker/client.ts @@ -1,34 +1,38 @@ import DockerClient from "dockerode"; -import {AppConfig} from "@nsm/config"; +import { AppConfig } from "@nsm/config"; export function initDockerClient(appConfig: AppConfig) { - let host = appConfig.getDockerHost(); + let host = appConfig.getDockerHost(); - let client: DockerClient; - if (host && ( - host.endsWith('.sock') || - host.startsWith('\\\\.\\pipe') - )) { - client = new DockerClient({ socketPath: host }); - } else if (host) { - // http(s)://host:port - host = host.substring(0, host.lastIndexOf(':') + 1); + let client: DockerClient; + if (host && (host.endsWith(".sock") || host.startsWith("\\\\.\\pipe"))) { + client = new DockerClient({ socketPath: host }); + } else if (host) { + // http(s)://host:port + host = host.substring(0, host.lastIndexOf(":") + 1); - let port = parseInt(appConfig.getDockerHost().replace(host, '')); + let port = parseInt(appConfig.getDockerHost().replace(host, "")); - host = host.substring(0, host.length - 1); + host = host.substring(0, host.length - 1); - let protocol = host.substring(0, host.indexOf('://')) as "http" | "https" | "ssh"; + let protocol = host.substring(0, host.indexOf("://")) as + | "http" + | "https" + | "ssh"; - host = host.substring(host.indexOf('://') + 3); + host = host.substring(host.indexOf("://") + 3); - if (isNaN(port)) { - throw new Error('Docker host must be in this format: protocol://host:port'); - } - - client = new DockerClient({protocol, host, port}); - } else { - throw new Error('Docker engine configuration variable not found! Please set docker_host in resources/config.yml or override using env.'); + if (isNaN(port)) { + throw new Error( + "Docker host must be in this format: protocol://host:port", + ); } - return client; -} \ No newline at end of file + + client = new DockerClient({ protocol, host, port }); + } else { + throw new Error( + "Docker engine configuration variable not found! Please set docker_host in resources/config.yml or override using env.", + ); + } + return client; +} diff --git a/src/engine/docker/index.ts b/src/engine/docker/index.ts index 26b43a3..dae08ec 100644 --- a/src/engine/docker/index.ts +++ b/src/engine/docker/index.ts @@ -1,45 +1,45 @@ -import {DockerServiceEngine} from "@nsm/engine"; -import {initDockerClient} from "@nsm/engine/docker/client"; +import { DockerServiceEngine } from "@nsm/engine"; +import { initDockerClient } from "@nsm/engine/docker/client"; -import build from './action/build'; +import build from "./action/build"; import run from "./action/run"; -import stop from './action/stop'; -import kill from './action/kill'; +import stop from "./action/stop"; +import kill from "./action/kill"; import reattach from "./action/reattach"; -import delVolume from './action/deletev'; -import delImage from './action/deletei'; -import cmd from './action/cmd'; -import getLabels from './action/getLabels'; -import listContainers from './action/listc'; -import listAttachedPorts from './action/listp'; +import delVolume from "./action/deletev"; +import delImage from "./action/deletei"; +import cmd from "./action/cmd"; +import getLabels from "./action/getLabels"; +import listContainers from "./action/listc"; +import listAttachedPorts from "./action/listp"; import stat from "./action/stat"; import statAll from "./action/statall"; import calcHostUsage from "./action/calcHostUsage"; import listRunning from "./action/listRunning"; -import {AppConfig} from "@nsm/config"; +import { AppConfig } from "@nsm/config"; export default function buildDockerEngine(appConfig: AppConfig) { - // Default engine implementation - const client = initDockerClient(appConfig); - const engine = {} as DockerServiceEngine; - engine.name = "Docker"; - engine.dockerClient = client; - engine.rws = {}; - // engine.cast - Being replaced in manager. - engine.build = build(client); - engine.run = run(engine, client); - engine.stop = stop(client); - engine.kill = kill(client); - engine.reattach = reattach(engine, client); - engine.deleteVolume = delVolume(engine, client); - engine.deleteImage = delImage(client); - engine.cmd = cmd(engine, client); - engine.getLabels = getLabels(client); - engine.listContainers = listContainers(engine, client); - engine.listAttachedPorts = listAttachedPorts(engine, client); - engine.stat = stat(engine, client); - engine.statAll = statAll(engine); - engine.calcHostUsage = calcHostUsage(client); - engine.listRunning = listRunning(client); - return engine; -} \ No newline at end of file + // Default engine implementation + const client = initDockerClient(appConfig); + const engine = {} as DockerServiceEngine; + engine.name = "Docker"; + engine.dockerClient = client; + engine.rws = {}; + // engine.cast - Being replaced in manager. + engine.build = build(client); + engine.run = run(engine, client); + engine.stop = stop(client); + engine.kill = kill(client); + engine.reattach = reattach(engine, client); + engine.deleteVolume = delVolume(engine, client); + engine.deleteImage = delImage(client); + engine.cmd = cmd(engine, client); + engine.getLabels = getLabels(client); + engine.listContainers = listContainers(engine, client); + engine.listAttachedPorts = listAttachedPorts(engine, client); + engine.stat = stat(engine, client); + engine.statAll = statAll(engine); + engine.calcHostUsage = calcHostUsage(client); + engine.listRunning = listRunning(client); + return engine; +} diff --git a/src/engine/docker/util/env.ts b/src/engine/docker/util/env.ts index 8f455a2..780cd88 100644 --- a/src/engine/docker/util/env.ts +++ b/src/engine/docker/util/env.ts @@ -1,9 +1,9 @@ -import {RunOptions} from "@nsm/engine"; +import { RunOptions } from "@nsm/engine"; export const propagateOptionsToEnv = (options: RunOptions, env: any) => { env.SERVICE_PORT = options.port.toString(); - env.SERVICE_PORTS = options.ports.join(' '); + env.SERVICE_PORTS = options.ports.join(" "); env.SERVICE_RAM = options.ram.toString(); env.SERVICE_CPU = options.cpu.toString(); env.SERVICE_DISK = options.disk.toString(); -} \ No newline at end of file +}; diff --git a/src/engine/docker/util/labels.ts b/src/engine/docker/util/labels.ts index c9e168c..82fa424 100644 --- a/src/engine/docker/util/labels.ts +++ b/src/engine/docker/util/labels.ts @@ -1,4 +1,4 @@ -import {ContainerFilter} from "@nsm/engine"; +import { ContainerFilter } from "@nsm/engine"; /** * Convert a ContainerFilter to Docker filters format. @@ -9,8 +9,10 @@ import {ContainerFilter} from "@nsm/engine"; export const toDockerFilters = (filter: ContainerFilter) => { const dockerFilters: any = {}; if (filter.labels) { - dockerFilters.label = Object.entries(filter.labels).map(([key, value]) => `${key}=${value}`); + dockerFilters.label = Object.entries(filter.labels).map( + ([key, value]) => `${key}=${value}`, + ); } return JSON.stringify(dockerFilters); -} \ No newline at end of file +}; diff --git a/src/engine/docker/util/logging.ts b/src/engine/docker/util/logging.ts index 48ae6b1..a49cc09 100644 --- a/src/engine/docker/util/logging.ts +++ b/src/engine/docker/util/logging.ts @@ -1,15 +1,15 @@ -import {ServiceLogRecord} from "@nsm/engine"; +import { ServiceLogRecord } from "@nsm/engine"; export const infoRecord = (message: string): ServiceLogRecord => { return { - level: 'info', - message - } -} + level: "info", + message, + }; +}; export const errorRecord = (message: string): ServiceLogRecord => { return { - level: 'error', - message - } -} \ No newline at end of file + level: "error", + message, + }; +}; diff --git a/src/engine/engine.ts b/src/engine/engine.ts index dd77c09..610c22d 100644 --- a/src/engine/engine.ts +++ b/src/engine/engine.ts @@ -1,111 +1,112 @@ import DockerClient from "dockerode"; import buildDockerEngine from "./docker"; -import {getSingleton} from "../depend"; -import {MetaStorage} from "./manager"; -import {AppConfig} from "@nsm/config"; +import { getSingleton } from "../depend"; +import { MetaStorage } from "./manager"; +import { AppConfig } from "@nsm/config"; /** * The options for running a service. */ export type RunOptions = { - port: number; - ports: number[]; - ram: number; // in MB - cpu: number; // in cores - disk: number; - env: { [key: string]: string }; - network?: { - address: string, - // If only ports should be exposed to this - // IP address. - portsOnly: boolean, - }; - labels?: { - [key: string]: string; - } -} + port: number; + ports: number[]; + ram: number; // in MB + cpu: number; // in cores + disk: number; + env: { [key: string]: string }; + network?: { + address: string; + // If only ports should be exposed to this + // IP address. + portsOnly: boolean; + }; + labels?: { + [key: string]: string; + }; +}; /** * The stats of a container, used for monitoring. */ export type ContainerStat = { - id: string, - memory: { - used: number, - total: number, - percent: number - }, - cpu: { - used: number, - total: number, - percent: number - }, -} + id: string; + memory: { + used: number; + total: number; + percent: number; + }; + cpu: { + used: number; + total: number; + percent: number; + }; +}; export type ContainerFilter = { - /** - * Filter containers that have all those labels. - */ - labels?: { [key: string]: string }; -} + /** + * Filter containers that have all those labels. + */ + labels?: { [key: string]: string }; +}; export type ServiceLogRecord = { - level: 'error' | 'info'; - message: string; -} + level: "error" | "info"; + message: string; +}; export type ServiceState = { - /** - * Internal ID of the state. - */ - id: string; - /** - * A brief description of the state, for display purposes. - */ - description: string; - /** - * Whether the service is ready to accept commands and connections - * in this state, thus is running. - */ - ready: boolean; -} + /** + * Internal ID of the state. + */ + id: string; + /** + * A brief description of the state, for display purposes. + */ + description: string; + /** + * Whether the service is ready to accept commands and connections + * in this state, thus is running. + */ + ready: boolean; +}; export type MessageListener = { - /** - * Called when there is a message from the container, with the message. - * - * @param message The message from the container - */ - onMessage?: (message: ServiceLogRecord) => Promise|void; -} + /** + * Called when there is a message from the container, with the message. + * + * @param message The message from the container + */ + onMessage?: (message: ServiceLogRecord) => Promise | void; +}; export type RunListener = MessageListener & { - /** - * Called when the container progress changes state. - * - * @param state The new state. - */ - onStateChange?: (state: ServiceState) => Promise|void; + /** + * Called when the container progress changes state. + * + * @param state The new state. + */ + onStateChange?: (state: ServiceState) => Promise | void; - /** - * Called when the container is closed, either by stop or kill, or by itself. - */ - onClose?: () => Promise|void; -} + /** + * Called when the container is closed, either by stop or kill, or by itself. + */ + onClose?: () => Promise | void; +}; export type DockerServiceEngine = ServiceEngineI & { - dockerClient: DockerClient; - /** - * Map of container IDs and attached watchers. - * IMPORTANT! Don't close or modify the streams, by any means! It - * would have unexpected fatal consequences. - */ - rws: { [id: string]: NodeJS.ReadWriteStream }; -} + dockerClient: DockerClient; + /** + * Map of container IDs and attached watchers. + * IMPORTANT! Don't close or modify the streams, by any means! It + * would have unexpected fatal consequences. + */ + rws: { [id: string]: NodeJS.ReadWriteStream }; +}; -export type ServiceEngineI = ServiceEngine & { // Internal - cast(): T; -} +export type ServiceEngineI = ServiceEngine & { + // Internal + cast(): T; +}; /** * The lowest layer which manipulates containers (services) directly. @@ -113,167 +114,169 @@ export type ServiceEngineI = ServiceEngine & { // Internal * containers themselves. */ export type ServiceEngine = { - // Just for display purposes - name: string; + // Just for display purposes + name: string; - /** - * Builds an image from build dir. - * - * @param imageId The image ID to build. If this is undefined, the engine should generate a random image ID and return it. - * @param buildDir The build dir path - * @param buildOptions The build options - * @param listener The listener to use for calling back up messages from the process - */ - build( - imageId: string|undefined, - buildDir: string, - buildOptions: { [key: string]: string }, - listener?: MessageListener): Promise; + /** + * Builds an image from build dir. + * + * @param imageId The image ID to build. If this is undefined, the engine should generate a random image ID and return it. + * @param buildDir The build dir path + * @param buildOptions The build options + * @param listener The listener to use for calling back up messages from the process + */ + build( + imageId: string | undefined, + buildDir: string, + buildOptions: { [key: string]: string }, + listener?: MessageListener, + ): Promise; - /** - * Runs a container from an image, with the given options. - * - * @param imageId The ID of the image to use - * @param volumeId The ID of the volume to use - * @param options The options - * @param meta The meta storage - * @param listener An optional listener for back propagation - */ - run( - imageId: string, - volumeId: string, - options: RunOptions, - meta: MetaStorage, - listener?: RunListener): Promise; + /** + * Runs a container from an image, with the given options. + * + * @param imageId The ID of the image to use + * @param volumeId The ID of the volume to use + * @param options The options + * @param meta The meta storage + * @param listener An optional listener for back propagation + */ + run( + imageId: string, + volumeId: string, + options: RunOptions, + meta: MetaStorage, + listener?: RunListener, + ): Promise; - /** - * Stops a container. - * - * @param id Container ID - * @return Success state - */ - stop(id: string): Promise; + /** + * Stops a container. + * + * @param id Container ID + * @return Success state + */ + stop(id: string): Promise; - /** - * Kills a container. - * - * @param id Container ID - * @param meta Meta storage for this unique context - * @return Success state - */ - kill(id: string, meta: MetaStorage): Promise; + /** + * Kills a container. + * + * @param id Container ID + * @param meta Meta storage for this unique context + * @return Success state + */ + kill(id: string, meta: MetaStorage): Promise; - /** - * Reattaches to a container. - * - * When this completes, the service is up and running. - * - * @param id Container ID - * @param listener Listener for container messages and state changes - */ - reattach(id: string, listener: RunListener): Promise; + /** + * Reattaches to a container. + * + * When this completes, the service is up and running. + * + * @param id Container ID + * @param listener Listener for container messages and state changes + */ + reattach(id: string, listener: RunListener): Promise; - /** - * Deletes a volume by ID. - * This is NEVER called if ServiceEngine#useVolumes is false. - * - * @param id The volume ID. - */ - deleteVolume(id: string): Promise; + /** + * Deletes a volume by ID. + * This is NEVER called if ServiceEngine#useVolumes is false. + * + * @param id The volume ID. + */ + deleteVolume(id: string): Promise; - /** - * Deletes an image by ID. - * - * @param id The image ID. - * @throw Error if the image cannot be deleted - */ - deleteImage(id: string): Promise; + /** + * Deletes an image by ID. + * + * @param id The image ID. + * @throw Error if the image cannot be deleted + */ + deleteImage(id: string): Promise; - /** - * Send a command to the container. - * - * @param id Container ID - * @param cmd The command, without new line - */ - cmd(id: string, cmd: string): Promise; + /** + * Send a command to the container. + * + * @param id Container ID + * @param cmd The command, without new line + */ + cmd(id: string, cmd: string): Promise; - /** - * Gets the labels of a container. - * - * @param id Container ID - */ - getLabels(id: string): Promise<{ [key: string]: string }>; + /** + * Gets the labels of a container. + * + * @param id Container ID + */ + getLabels(id: string): Promise<{ [key: string]: string }>; - /** - * Lists container ids of containers by templates. - * - * @param filter The filter to apply - * @return List of container IDs - */ - listContainers(filter: ContainerFilter): Promise; + /** + * Lists container ids of containers by templates. + * + * @param filter The filter to apply + * @return List of container IDs + */ + listContainers(filter: ContainerFilter): Promise; - /** - * List running containers owned by this engine on this machine. - * - * @param filter The filter to apply - * @return List of container IDs - */ - listRunning(filter: ContainerFilter): Promise; + /** + * List running containers owned by this engine on this machine. + * + * @param filter The filter to apply + * @return List of container IDs + */ + listRunning(filter: ContainerFilter): Promise; - listAttachedPorts(): Promise; + listAttachedPorts(): Promise; - stat(id: string): Promise; + stat(id: string): Promise; - statAll(filter: ContainerFilter): Promise; + statAll(filter: ContainerFilter): Promise; - // Disk usage of all services here - // [0]: free, [1]: size - calcHostUsage(): Promise; -} + // Disk usage of all services here + // [0]: free, [1]: size + calcHostUsage(): Promise; +}; /** * Standard labels that NSM uses to identify and manage containers. * Used by the manager to keep consistency across the codebase. */ export enum StandardLabel { - // The default label identifying a NSM-managed container. - Nsm = 'nsm', - // The service ID that owns the container. - ServiceId = 'nsm.id', - // The volume ID that the container is using. - VolumeId = 'nsm.volumeId', - // The template ID that the container is created from. - TemplateId = 'nsm.templateId', - // The node ID of the managing worker. - NodeId = 'nsm.nodeId', + // The default label identifying a NSM-managed container. + Nsm = "nsm", + // The service ID that owns the container. + ServiceId = "nsm.id", + // The volume ID that the container is using. + VolumeId = "nsm.volumeId", + // The template ID that the container is created from. + TemplateId = "nsm.templateId", + // The node ID of the managing worker. + NodeId = "nsm.nodeId", } export const Filters = { - /** - * The standard filter for NSM-managed containers, which - * filters containers that have the label "nsm" with value "true". - */ - nsm() { - return { - labels: { - [StandardLabel.Nsm]: 'true' - } - } - }, - /** - * The filter for containers belonging to a node with the given node ID. - * - * @param nodeId The node ID - */ - node(nodeId: string) { - return { - labels: { - ...this.nsm().labels, - [StandardLabel.NodeId]: nodeId, - } - } - } -} + /** + * The standard filter for NSM-managed containers, which + * filters containers that have the label "nsm" with value "true". + */ + nsm() { + return { + labels: { + [StandardLabel.Nsm]: "true", + }, + }; + }, + /** + * The filter for containers belonging to a node with the given node ID. + * + * @param nodeId The node ID + */ + node(nodeId: string) { + return { + labels: { + ...this.nsm().labels, + [StandardLabel.NodeId]: nodeId, + }, + }; + }, +}; /** * Combines multiple run listeners into one, by calling them in sequence. @@ -281,39 +284,39 @@ export const Filters = { * @param listeners The listeners to combine. */ export const combineRunListeners = (listeners: RunListener[]): RunListener => { - return { - onStateChange: async (state) => { - for (let listener of listeners) { - await listener.onStateChange?.(state); - } - }, - onMessage: async (record) => { - for (let listener of listeners) { - await listener.onMessage?.(record); - } - }, - onClose: () => { - for (let listener of listeners) { - listener.onClose?.(); - } - } - } -} + return { + onStateChange: async (state) => { + for (let listener of listeners) { + await listener.onStateChange?.(state); + } + }, + onMessage: async (record) => { + for (let listener of listeners) { + await listener.onMessage?.(record); + } + }, + onClose: () => { + for (let listener of listeners) { + listener.onClose?.(); + } + }, + }; +}; export default function (appConfig: AppConfig): ServiceEngineI { - let engine = getSingleton('engine'); - if (!engine) { - const engineId = process.env.NSM_ENGINE ?? 'docker'; - switch (engineId) { - case 'docker': - engine = buildDockerEngine(appConfig); - break; - default: - throw new Error('Invalid engine ID: ' + engineId); - } + let engine = getSingleton("engine"); + if (!engine) { + const engineId = process.env.NSM_ENGINE ?? "docker"; + switch (engineId) { + case "docker": + engine = buildDockerEngine(appConfig); + break; + default: + throw new Error("Invalid engine ID: " + engineId); } - return { - cast: undefined, // Being set in manager - ...engine, - }; -} \ No newline at end of file + } + return { + cast: undefined, // Being set in manager + ...engine, + }; +} diff --git a/src/engine/ignore.ts b/src/engine/ignore.ts index 3ba09ea..59c9daf 100644 --- a/src/engine/ignore.ts +++ b/src/engine/ignore.ts @@ -3,59 +3,62 @@ import ignore from "ignore"; import path from "path"; export const getRootFilesFiltered = (dir: string) => { - let filtered = fs.readdirSync(dir); - if (fs.existsSync(path.join(dir, '.nsmignore'))) { - const ig = buildIgnore(dir); + let filtered = fs.readdirSync(dir); + if (fs.existsSync(path.join(dir, ".nsmignore"))) { + const ig = buildIgnore(dir); - filtered = ig.filter(filtered); - } + filtered = ig.filter(filtered); + } - return filtered; -} + return filtered; +}; export const getFilteredPaths = (dir: string) => { - const ig = buildIgnore(dir); + const ig = buildIgnore(dir); + + let filtered = { + files: [] as string[], + dirs: [] as string[], + }; + const walk = (currentDir: string) => { + const files = fs.readdirSync(currentDir); + for (const file of files) { + const relativePath = + currentDir === dir + ? file + : currentDir.substring(dir.length + 1) + path.sep + file; + const fullPath = currentDir + path.sep + file; - let filtered = { - files: [] as string[], - dirs: [] as string[] - }; - const walk = (currentDir: string) => { - const files = fs.readdirSync(currentDir); - for (const file of files) { - const relativePath = currentDir === dir ? file : currentDir.substring(dir.length + 1) + path.sep + file; - const fullPath = currentDir + path.sep + file; - - if (ig.ignores(relativePath)) { - const isDir = fs.statSync(fullPath).isDirectory(); - if (isDir) { - filtered.dirs.push(relativePath); - } else { - filtered.files.push(relativePath); - } - - if (isDir) { - // If it's a directory, we need to ignore all its contents as well, so we skip walking into it - continue; - } - } - - if (fs.statSync(fullPath).isDirectory()) { - walk(fullPath); - } + if (ig.ignores(relativePath)) { + const isDir = fs.statSync(fullPath).isDirectory(); + if (isDir) { + filtered.dirs.push(relativePath); + } else { + filtered.files.push(relativePath); } + + if (isDir) { + // If it's a directory, we need to ignore all its contents as well, so we skip walking into it + continue; + } + } + + if (fs.statSync(fullPath).isDirectory()) { + walk(fullPath); + } } - walk(dir); + }; + walk(dir); - return filtered; -} + return filtered; +}; const buildIgnore = (dir: string) => { - const ig = ignore(); - const ignorePath = path.join(dir, '.nsmignore'); - if (fs.existsSync(ignorePath)) { - ig.add(fs.readFileSync(ignorePath, 'utf8')); - } + const ig = ignore(); + const ignorePath = path.join(dir, ".nsmignore"); + if (fs.existsSync(ignorePath)) { + ig.add(fs.readFileSync(ignorePath, "utf8")); + } - return ig; -} \ No newline at end of file + return ig; +}; diff --git a/src/engine/image.ts b/src/engine/image.ts index 3c23818..feb3984 100644 --- a/src/engine/image.ts +++ b/src/engine/image.ts @@ -1,12 +1,12 @@ -import {Database, ImageModel} from "@nsm/database"; +import { Database, ImageModel } from "@nsm/database"; import winston from "winston"; -import {MessageListener, ServiceEngineI} from "@nsm/engine/engine"; -import {templateBuildDir} from "@nsm/engine/monitoring/util"; -import {TemplateManager} from "@nsm/engine/template"; -import {TemplateDirWatcher} from "@nsm/engine/monitoring/templateDirWatcher"; +import { MessageListener, ServiceEngineI } from "@nsm/engine/engine"; +import { templateBuildDir } from "@nsm/engine/monitoring/util"; +import { TemplateManager } from "@nsm/engine/template"; +import { TemplateDirWatcher } from "@nsm/engine/monitoring/templateDirWatcher"; type BuildOptionsMap = { - [key: string]: string + [key: string]: string; }; let engine: ServiceEngineI; @@ -20,14 +20,14 @@ export const init = ( templateManager_: TemplateManager, templateDirWatcher_: TemplateDirWatcher, db_: Database, - logger_: winston.Logger + logger_: winston.Logger, ) => { engine = engine_; templateManager = templateManager_; templateDirWatcher = templateDirWatcher_; db = db_; logger = logger_; -} +}; /** * Ensures that the image associated with the given ID is up to date and @@ -43,7 +43,9 @@ export const init = ( */ export const processImage = async ( id: string | undefined | null, - templateId: string, buildOptions: BuildOptionsMap, messageListener?: MessageListener + templateId: string, + buildOptions: BuildOptionsMap, + messageListener?: MessageListener, ) => { const template = templateManager.getTemplate(templateId); // Checks if the provided options are still compatible with the template @@ -56,21 +58,29 @@ export const processImage = async ( const imageModel = await getImage(id); if (imageModel.templateId != templateId) { - throw new Error(`Image ${id} is based on template ${imageModel.templateId}, but template ${templateId} was expected`); + throw new Error( + `Image ${id} is based on template ${imageModel.templateId}, but template ${templateId} was expected`, + ); } - const imageOutdated = imageModel.hash != templateDirWatcher.getTemplateHash(imageModel.templateId); + const imageOutdated = + imageModel.hash != + templateDirWatcher.getTemplateHash(imageModel.templateId); const optionsChanged = optionsDiffer(buildOptions, imageModel.buildOptions); if (imageOutdated || optionsChanged) { if (optionsChanged) { - logger.info(`The target options differ, finding or building a new compatible image...`); + logger.info( + `The target options differ, finding or building a new compatible image...`, + ); id = await pickImageOrBuild(templateId, buildOptions); // If the image becomes unused after the switch, delete it await deleteImageIfUnused(imageModel); } else { - logger.info(`Image ${id} is outdated due to template changes. Rebuilding...`); + logger.info( + `Image ${id} is outdated due to template changes. Rebuilding...`, + ); // Template changed, we need to rebuild the image await rebuildImage(imageModel, messageListener); @@ -78,7 +88,7 @@ export const processImage = async ( } return id; -} +}; /** * Tries to find an existing image that is compatible with the given template ID and build options. @@ -88,7 +98,10 @@ export const processImage = async ( * @param buildOptions Build options to use when finding/building the image * @returns The ID of the found or built image */ -const pickImageOrBuild = async (templateId: string, buildOptions: BuildOptionsMap) => { +const pickImageOrBuild = async ( + templateId: string, + buildOptions: BuildOptionsMap, +) => { let id = await pickImage(templateId, buildOptions); if (id == null) { logger.info(`No compatible image found for request. Building new image...`); @@ -98,9 +111,12 @@ const pickImageOrBuild = async (templateId: string, buildOptions: BuildOptionsMa } return id; -} +}; -export const optionsDiffer = (options1: BuildOptionsMap, options2: BuildOptionsMap): boolean => { +export const optionsDiffer = ( + options1: BuildOptionsMap, + options2: BuildOptionsMap, +): boolean => { const keys1 = Object.keys(options1); const keys2 = Object.keys(options2); @@ -119,7 +135,7 @@ export const optionsDiffer = (options1: BuildOptionsMap, options2: BuildOptionsM } return false; -} +}; /** * Retrieves the image information from the database for the given image ID. @@ -135,7 +151,7 @@ const getImage = async (id: string) => { } return image; -} +}; /** * Builds a new image based on the given template ID and build options, and saves it to the database. @@ -151,10 +167,15 @@ const buildImage = async ( templateId: string, options: BuildOptionsMap, imageId?: string, - messageListener?: MessageListener + messageListener?: MessageListener, ): Promise => { const hash = templateDirWatcher.getTemplateHash(templateId); - imageId = await engine.build(imageId, templateBuildDir(templateId), options, messageListener); + imageId = await engine.build( + imageId, + templateBuildDir(templateId), + options, + messageListener, + ); await db.imageRepository.saveImage({ id: imageId, @@ -163,10 +184,16 @@ const buildImage = async ( buildOptions: options, }); return imageId; -} +}; -const pickImage = async (templateId: string, options: BuildOptionsMap): Promise => { - const images = await db.imageRepository.listImagesByOptions(templateId, options); +const pickImage = async ( + templateId: string, + options: BuildOptionsMap, +): Promise => { + const images = await db.imageRepository.listImagesByOptions( + templateId, + options, + ); if (images.length == 0) { return null; } @@ -174,20 +201,32 @@ const pickImage = async (templateId: string, options: BuildOptionsMap): Promise< const image = images[Math.floor(Math.random() * images.length)]; // TODO: implement better image picking strategy (e.g. based on usage) return image.id; -} +}; -const rebuildImage = async (image: ImageModel, messageListener?: MessageListener) => { - return buildImage(image.templateId, image.buildOptions, image.id, messageListener); -} +const rebuildImage = async ( + image: ImageModel, + messageListener?: MessageListener, +) => { + return buildImage( + image.templateId, + image.buildOptions, + image.id, + messageListener, + ); +}; export const deleteImageIfUnused = async (image: ImageModel) => { - const servicesUsingImage = await db.permaRepository.listPermaUsingImage(image.id); + const servicesUsingImage = await db.permaRepository.listPermaUsingImage( + image.id, + ); if (servicesUsingImage.length > 0) { // Image is still in use, do not delete return; } - logger.debug(`Image ${image.id} is no longer used by any service. Deleting...`); + logger.debug( + `Image ${image.id} is no longer used by any service. Deleting...`, + ); try { await engine.deleteImage(image.id); @@ -195,4 +234,4 @@ export const deleteImageIfUnused = async (image: ImageModel) => { logger.error(`Failed to delete image ${image.id}`, e); } await db.imageRepository.deleteImage(image.id); -} \ No newline at end of file +}; diff --git a/src/engine/index.ts b/src/engine/index.ts index abdc154..eff7ffb 100644 --- a/src/engine/index.ts +++ b/src/engine/index.ts @@ -1,2 +1,2 @@ export * from "./manager"; -export * from "./engine"; \ No newline at end of file +export * from "./engine"; diff --git a/src/engine/manager.ts b/src/engine/manager.ts index 01f08f2..188c2c7 100644 --- a/src/engine/manager.ts +++ b/src/engine/manager.ts @@ -1,90 +1,105 @@ -import {currentContext} from "../app"; +import { currentContext } from "../app"; import createEngine, { - RunOptions, - RunListener, - ServiceEngineI, - StandardLabel, - Filters, combineRunListeners + RunOptions, + RunListener, + ServiceEngineI, + StandardLabel, + Filters, + combineRunListeners, } from "./engine"; -import {Template, getTemplate as loadTemplate, getAllTemplates} from "./template"; +import { + Template, + getTemplate as loadTemplate, + getAllTemplates, +} from "./template"; import * as templateManager from "./template"; import * as templateDirWatcher from "./monitoring/templateDirWatcher"; import crypto from "crypto"; -import {randomPort as retrieveRandomPort} from "@nsm/util/port"; -import {Database, PermaModel} from "../database"; +import { randomPort as retrieveRandomPort } from "@nsm/util/port"; +import { Database, PermaModel } from "../database"; import { - isServicePending, - lckStatusTp, - lockBusyAction, - reqNotPending, - ulckStatusTp, - UnlockObserver, - whenUnlocked, whenUnlockedAll + isServicePending, + lckStatusTp, + lockBusyAction, + reqNotPending, + ulckStatusTp, + UnlockObserver, + whenUnlocked, + whenUnlockedAll, } from "./asyncp"; import winston from "winston"; -import {isDebug} from "../helpers"; -import {resolveSequentially} from "@nsm/util/promises"; -import {watchTemplateDirChanges} from "@nsm/engine/monitoring/templateDirWatcher"; -import {processImage, init as initImageEngine, deleteImageIfUnused} from "@nsm/engine/image"; -import {propagateOptionsToEnv} from "@nsm/engine/docker/util/env"; -import {ActiveServiceSession, beginServiceSession, ServiceSession, init as initSessionEngine} from "@nsm/engine/session"; -import {AppConfig} from "@nsm/config"; +import { isDebug } from "../helpers"; +import { resolveSequentially } from "@nsm/util/promises"; +import { watchTemplateDirChanges } from "@nsm/engine/monitoring/templateDirWatcher"; +import { + processImage, + init as initImageEngine, + deleteImageIfUnused, +} from "@nsm/engine/image"; +import { propagateOptionsToEnv } from "@nsm/engine/docker/util/env"; +import { + ActiveServiceSession, + beginServiceSession, + ServiceSession, + init as initSessionEngine, +} from "@nsm/engine/session"; +import { AppConfig } from "@nsm/config"; export type Options = { + /** + * The amount of RAM that the service can allocate in MB. + * (optional) + */ + ram?: number; + /** + * The amount of CPU cores that the service can use. + * (optional) + */ + cpu?: number; + /** + * The amount of disk space that the service can use in MB. + * (optional) + */ + disk?: number; + /** + * The additional ports to expose. (optional) + * Main port will be chosen automatically. + * (optional) + */ + ports?: number[]; // Optional ports to expose + meta?: { [key: string]: any }; + /** + * The optional environment variables (template options) to set. + * These are custom variables that the specific template uses to correctly + * build its environment. + * + * Firstly, you need to specify those env variables and their defaults + * in the settings.yml file of the template, and then they can be used + * in the Dockerfile of template. Those variables can be listed by the + * lookup and will be stored for later use when resuming the service. + * (optional) + */ + env?: { [key: string]: string }; // Optional ENV, see example_settings.yml + /** + * The (optional) network settings for the service. + * This specifies fi the service will be bind to custom network interface + * in the future and how. + */ + network?: { /** - * The amount of RAM that the service can allocate in MB. - * (optional) - */ - ram?: number, - /** - * The amount of CPU cores that the service can use. - * (optional) - */ - cpu?: number, - /** - * The amount of disk space that the service can use in MB. - * (optional) - */ - disk?: number, - /** - * The additional ports to expose. (optional) - * Main port will be chosen automatically. - * (optional) + * Bind address. */ - ports?: number[], // Optional ports to expose - meta?: {[key: string]: any}, + address: string; /** - * The optional environment variables (template options) to set. - * These are custom variables that the specific template uses to correctly - * build its environment. + * If whole service interface (all ports) should be exposed to the + * interface (false), or only defined ports (true). * - * Firstly, you need to specify those env variables and their defaults - * in the settings.yml file of the template, and then they can be used - * in the Dockerfile of template. Those variables can be listed by the - * lookup and will be stored for later use when resuming the service. - * (optional) + * Defined ports are those specified in ports?: number[], and main + * service port. */ - env?: {[key: string]: string}, // Optional ENV, see example_settings.yml - /** - * The (optional) network settings for the service. - * This specifies fi the service will be bind to custom network interface - * in the future and how. - */ - network?: { - /** - * Bind address. - */ - address: string, - /** - * If whole service interface (all ports) should be exposed to the - * interface (false), or only defined ports (true). - * - * Defined ports are those specified in ports?: number[], and main - * service port. - */ - portsOnly: boolean, - } -} + portsOnly: boolean; + }; +}; /** * Per-service storage. @@ -92,229 +107,233 @@ export type Options = { * as long term data. Every key set here is per-service. */ export type MetaStorage = { - set: (key: string, value: any) => Promise; - get: (key: string, def?: T) => Promise; -} + set: (key: string, value: any) => Promise; + get: (key: string, def?: T) => Promise; +}; export type EngineExpansion = { - [k in keyof ServiceEngineI | string]: any; + [k in keyof ServiceEngineI | string]: any; }; type ServiceEvent = { - id: string; - error?: Error; -} + id: string; + error?: Error; +}; type ServiceManagerEvents = { - resume: ServiceEvent; - stop: ServiceEvent; -} + resume: ServiceEvent; + stop: ServiceEvent; +}; -type EventHandler = (event: ServiceManagerEvents[T]) => boolean|void; +type EventHandler = ( + event: ServiceManagerEvents[T], +) => boolean | void; type ServiceManagerEventBus = { - on(evt: T, h: EventHandler): void; -} - + on(evt: T, h: EventHandler): void; +}; export type ListServicesOptions = { + /** + * The page number (index). + */ + page: number; + /** + * The page size. + */ + pageSize: number; + + /** + * Filter options. + */ + filter?: { /** - * The page number (index). - */ - page: number; - /** - * The page size. - */ - pageSize: number; - - /** - * Filter options. + * Filter services by their meta attributes. */ - filter?: { - /** - * Filter services by their meta attributes. - */ - meta?: {[key: string]: any}; - } -} + meta?: { [key: string]: any }; + }; +}; export type ServiceManager = ServiceManagerEventBus & { - /** - * This NSM instance ID - */ - nodeId: string; - /** - * Internal engine implementation - */ - engine: ServiceEngineI; - - /** - * Initialize the service manager. - * - * @param db The database - * @param appConfig The app config - * @param logger The global logger - */ - init(db: Database, appConfig: any, logger: winston.Logger): Promise; - - /** - * Create a new service. - * - * @param template The template ID (folder name) to use - * @param options The options to use. Options will be stored for later use. - * @returns The service ID - */ - createService(template: string, options: Options): Promise; // Service ID - - /** - * Resume a service. - * - * @param id The service ID - * @returns Whether the service was resumed - */ - resumeService(id: string): Promise; - - /** - * Stop a service. - * - * @param id The service ID - */ - stopService(id: string): Promise; - - /** - * Stop a service forcibly (kill). - * - * @param id The service ID - */ - stopServiceForcibly(id: string): Promise; - - /** - * Send pre-configured stop signal to the service. - * - * @param id The service ID - * @returns Whether the signal has been sent - */ - sendStopSignal(id: string): Promise; - - /** - * Delete a service. - * - * @param id The service ID - */ - deleteService(id: string): Promise; - - /** - * Update the options of a service. - * - * @param id The service ID - * @param options The new options - */ - updateOptions(id: string, options: Options): Promise; - - /** - * Get the template by ID. - * - * @param id The template ID - * @returns The template wrapper - */ - getTemplate(id: string): Template|undefined; - - /** - * Get the service by ID. - * - * @param from The service ID, or model - * @param options The get options - * includeSession: Whether to include the session to result - * otherNodes: If true, we will include services on other NSM nodes to search - */ - getService(from: string|PermaModel, options?: { includeSession?: boolean, otherNodes?: boolean }): Promise; - - /** - * Get the last power error of a service. - * - * @param id The service ID - */ - getLastPowerError(id: string): Error|undefined; - - /** - * Get list of running services on this node. - */ - getRunningServices(): RunningService[]; - - /** - * Get the running service by ID. - * - * @param id The service ID - */ - getRunningService(id: string): RunningService|undefined; - - /** - * List all available services. - * - * @param options The list options - * @returns The list of service IDs - */ - listServices(options: ListServicesOptions): Promise; - - /** - * List all available templates. - * - * @returns The list of template IDs - */ - listTemplates(): Promise; - - /** - * Stop all running services on this instance. - */ - stopRunning(): Promise; - - isRunning(id: string): boolean; - - waitForBusyAction(id: string): Promise; - - // DON'T call those until you really know what you are doing. - expandEngine(exp?: T): Promise; - - initEngineForcibly(): Promise; - // + /** + * This NSM instance ID + */ + nodeId: string; + /** + * Internal engine implementation + */ + engine: ServiceEngineI; + + /** + * Initialize the service manager. + * + * @param db The database + * @param appConfig The app config + * @param logger The global logger + */ + init(db: Database, appConfig: any, logger: winston.Logger): Promise; + + /** + * Create a new service. + * + * @param template The template ID (folder name) to use + * @param options The options to use. Options will be stored for later use. + * @returns The service ID + */ + createService(template: string, options: Options): Promise; // Service ID + + /** + * Resume a service. + * + * @param id The service ID + * @returns Whether the service was resumed + */ + resumeService(id: string): Promise; + + /** + * Stop a service. + * + * @param id The service ID + */ + stopService(id: string): Promise; + + /** + * Stop a service forcibly (kill). + * + * @param id The service ID + */ + stopServiceForcibly(id: string): Promise; + + /** + * Send pre-configured stop signal to the service. + * + * @param id The service ID + * @returns Whether the signal has been sent + */ + sendStopSignal(id: string): Promise; + + /** + * Delete a service. + * + * @param id The service ID + */ + deleteService(id: string): Promise; + + /** + * Update the options of a service. + * + * @param id The service ID + * @param options The new options + */ + updateOptions(id: string, options: Options): Promise; + + /** + * Get the template by ID. + * + * @param id The template ID + * @returns The template wrapper + */ + getTemplate(id: string): Template | undefined; + + /** + * Get the service by ID. + * + * @param from The service ID, or model + * @param options The get options + * includeSession: Whether to include the session to result + * otherNodes: If true, we will include services on other NSM nodes to search + */ + getService( + from: string | PermaModel, + options?: { includeSession?: boolean; otherNodes?: boolean }, + ): Promise; + + /** + * Get the last power error of a service. + * + * @param id The service ID + */ + getLastPowerError(id: string): Error | undefined; + + /** + * Get list of running services on this node. + */ + getRunningServices(): RunningService[]; + + /** + * Get the running service by ID. + * + * @param id The service ID + */ + getRunningService(id: string): RunningService | undefined; + + /** + * List all available services. + * + * @param options The list options + * @returns The list of service IDs + */ + listServices(options: ListServicesOptions): Promise; + + /** + * List all available templates. + * + * @returns The list of template IDs + */ + listTemplates(): Promise; + + /** + * Stop all running services on this instance. + */ + stopRunning(): Promise; + + isRunning(id: string): boolean; + + waitForBusyAction(id: string): Promise; + + // DON'T call those until you really know what you are doing. + expandEngine(exp?: T): Promise; + + initEngineForcibly(): Promise; + // } & { - whenUnlocked: typeof whenUnlocked + whenUnlocked: typeof whenUnlocked; }; type RunningService = { - id: string; - session: ServiceSession; - internalSession: InternalSession; -} + id: string; + session: ServiceSession; + internalSession: InternalSession; +}; export type InternalSession = { - containerId: string; - // TODO: add more useful information? -} + containerId: string; + // TODO: add more useful information? +}; export type ServiceInfo = PermaModel & { - optionsRam: number; // From options.ram - optionsCpu: number; // From options.cpu - optionsDisk: number; // From options.disk - state: State; - session?: ServiceSession; - internalSession?: InternalSession; -} + optionsRam: number; // From options.ram + optionsCpu: number; // From options.cpu + optionsDisk: number; // From options.disk + state: State; + session?: ServiceSession; + internalSession?: InternalSession; +}; -export type State = 'RUNNING' | 'BUILDING' | 'STOPPED'; +export type State = "RUNNING" | "BUILDING" | "STOPPED"; // 1 = unknown, 2 = conflict, 3 = not found export type StatusCode = 1 | 2 | 3; export class _InternalError extends Error { - readonly code: StatusCode; - readonly msg: string; + readonly code: StatusCode; + readonly msg: string; - constructor(msg: string, code?: StatusCode) { - super(msg); - this.code = code ?? 1; - this.msg = msg; - } + constructor(msg: string, code?: StatusCode) { + super(msg); + this.code = code ?? 1; + this.msg = msg; + } } export let engine: ServiceEngineI = undefined; @@ -330,43 +349,53 @@ const started: RunningService[] = []; const startedStates: Map = new Map(); const evtHandlers: Map[]> = new Map(); -["push", "splice"].forEach(funcName => { - started[funcName] = (...args: any[]) => { - const result = Array.prototype[funcName].apply(started, args); +["push", "splice"].forEach((funcName) => { + started[funcName] = (...args: any[]) => { + const result = Array.prototype[funcName].apply(started, args); - // Emit services change within those methods - if (isDebug()) { - currentContext.logger.debug('Service registry changed'); - } + // Emit services change within those methods + if (isDebug()) { + currentContext.logger.debug("Service registry changed"); + } - return result; - }; + return result; + }; }); -export async function init(db_: Database, appConfig_: AppConfig, logger: winston.Logger) { - const nodeId_ = appConfig_.getNodeId(); +export async function init( + db_: Database, + appConfig_: AppConfig, + logger: winston.Logger, +) { + const nodeId_ = appConfig_.getNodeId(); - logger.info(`Initializing service manager for node ${nodeId_}...`); + logger.info(`Initializing service manager for node ${nodeId_}...`); - db = db_; - if (!engine) { - // Init only if it has not already been force-initialized - await initEngineForcibly(); - } - nodeId = nodeId_ as string; + db = db_; + if (!engine) { + // Init only if it has not already been force-initialized + await initEngineForcibly(); + } + nodeId = nodeId_ as string; - initImageEngine(engine, templateManager, templateDirWatcher, db_, currentContext.logger); - initSessionEngine(db_); - watchTemplateDirChanges(currentContext.logger); + initImageEngine( + engine, + templateManager, + templateDirWatcher, + db_, + currentContext.logger, + ); + initSessionEngine(db_); + watchTemplateDirChanges(currentContext.logger); - await deleteGarbage(logger); - await reattachStaleContainers(logger); + await deleteGarbage(logger); + await reattachStaleContainers(logger); - logger.info(`Using engine: ${engine.name}`); + logger.info(`Using engine: ${engine.name}`); } async function deleteGarbage(logger: winston.Logger) { - // TODO: delete containers that are not running and remained from last session + // TODO: delete containers that are not running and remained from last session } /** @@ -376,480 +405,493 @@ async function deleteGarbage(logger: winston.Logger) { * @param logger The logger to use */ async function reattachStaleContainers(logger: winston.Logger) { - const running = await engine.listRunning(Filters.node(nodeId)) - .then(containerIds => containerIds + const running = await engine + .listRunning(Filters.node(nodeId)) + .then((containerIds) => + containerIds // Filter out those that we have already started in this session, just in case // this was started more than once a session - .filter(id => !started.find(runningService => runningService.internalSession.containerId === id))); - - for (let containerId of running) { - const labels = await engine.getLabels(containerId); - if (!labels[StandardLabel.ServiceId]) { - // The container was in the running list, but does not have the required labels - // Should not happen, but just in case - logger.warn(`Found a running container with id ${containerId} that does not have a service id label, stopping.`); + .filter( + (id) => + !started.find( + (runningService) => + runningService.internalSession.containerId === id, + ), + ), + ); - await engine.stop(containerId); - } + for (let containerId of running) { + const labels = await engine.getLabels(containerId); + if (!labels[StandardLabel.ServiceId]) { + // The container was in the running list, but does not have the required labels + // Should not happen, but just in case + logger.warn( + `Found a running container with id ${containerId} that does not have a service id label, stopping.`, + ); - const serviceId = labels[StandardLabel.ServiceId]; - - // We must begin a new session since the previous was interrupted - const session = await beginServiceSession(serviceId); - // Reattach and watch the container - await engine.reattach(containerId, buildRunListener(session)); - - // Save session in-memory - const info: RunningService = { - id: serviceId, - session, - internalSession: { - containerId - } - }; - started.push(info); - logger.info(`Reattached container ${containerId} for service ${serviceId}`); + await engine.stop(containerId); } - await new Promise((resolve) => whenUnlockedAll(() => resolve(null))); -} + const serviceId = labels[StandardLabel.ServiceId]; -export async function expandEngine(exp?: T): Promise { - if (exp) { - if (!engine && (!currentContext || !currentContext.appConfig)) { - throw new Error("Engine is not yet loaded and can't be loaded forcibly!"); - } else if (!engine) { - // Engine is not initialized yet, but we want to expand it, so - // we need to force load it. - await initEngineForcibly(); - } - // An expansion is provided, so there are changes to be applied. - Object.keys(exp).forEach((expKey) => { - if (!Number.isNaN(Number(expKey))) { - throw new Error("Invalid expansion format, please replace functions within with lambda functions. " + - "Invalid: { funcName(param) {}, funcName2(param) {} }" + - "Valid: { funcName: (param) => {}, funcName2: (param) => {} }") - } - engine[expKey] = exp[expKey]; - }); - } - return engine as any; -} + // We must begin a new session since the previous was interrupted + const session = await beginServiceSession(serviceId); + // Reattach and watch the container + await engine.reattach(containerId, buildRunListener(session)); -export async function createService(template: string, options: Options) { - const { - ram, - cpu, - disk, - ports, - env, - network - } = options; - const serviceSettings = reqTemplate(template).settings; - - // Join meta supplied by user and template meta - const meta = { - ...(options.meta ?? {}), - ...(serviceSettings.meta ?? {}) + // Save session in-memory + const info: RunningService = { + id: serviceId, + session, + internalSession: { + containerId, + }, }; - if (!meta || !meta.stopCmd) { - throw new _InternalError('Invalid template meta for ' + template); + started.push(info); + logger.info(`Reattached container ${containerId} for service ${serviceId}`); + } + + await new Promise((resolve) => whenUnlockedAll(() => resolve(null))); +} + +export async function expandEngine( + exp?: T, +): Promise { + if (exp) { + if (!engine && (!currentContext || !currentContext.appConfig)) { + throw new Error("Engine is not yet loaded and can't be loaded forcibly!"); + } else if (!engine) { + // Engine is not initialized yet, but we want to expand it, so + // we need to force load it. + await initEngineForcibly(); } + // An expansion is provided, so there are changes to be applied. + Object.keys(exp).forEach((expKey) => { + if (!Number.isNaN(Number(expKey))) { + throw new Error( + "Invalid expansion format, please replace functions within with lambda functions. " + + "Invalid: { funcName(param) {}, funcName2(param) {} }" + + "Valid: { funcName: (param) => {}, funcName2: (param) => {} }", + ); + } + engine[expKey] = exp[expKey]; + }); + } + return engine as any; +} - const serviceId = crypto.randomUUID(); // Create new unique service id - // Pick random main port from the range specified in settings.yml - const portRange = serviceSettings.port_range; - const port = await retrieveRandomPort( - engine, - portRange.min as number, - portRange.max as number - ); +export async function createService(template: string, options: Options) { + const { ram, cpu, disk, ports, env, network } = options; + const serviceSettings = reqTemplate(template).settings; + + // Join meta supplied by user and template meta + const meta = { + ...(options.meta ?? {}), + ...(serviceSettings.meta ?? {}), + }; + if (!meta || !meta.stopCmd) { + throw new _InternalError("Invalid template meta for " + template); + } + + const serviceId = crypto.randomUUID(); // Create new unique service id + // Pick random main port from the range specified in settings.yml + const portRange = serviceSettings.port_range; + const port = await retrieveRandomPort( + engine, + portRange.min as number, + portRange.max as number, + ); + + const perma: PermaModel = { + serviceId, + template, + nodeId, + port, + options: { ram, cpu, disk, ports }, + meta, + env: env ?? {}, + network, + }; + let err: any; + // Save permanent info + if (!(await db.permaRepository.savePerma(perma))) { + err = new _InternalError("Failed to save perma info to database"); + } + + if (err) { + // Save to be later retrieved + errors[serviceId] = err; + currentContext.logger.error(err.message); + } + + if (err) { + throw err; + } else { + return serviceId; + } +} - const perma: PermaModel = { - serviceId, - template, - nodeId, - port, - options: {ram, cpu, disk, ports}, +export async function resumeService(id: string) { + reqNotRunning(id); + let { template, options, env, network, port } = await getPermaModel(id); + + const { defaults, env: settingsEnv } = reqTemplate(template).settings; + // Filter env to only those that are defined in settings.yml, because those are the only ones that + // we can guarantee to be used and will not make problems when handling images. + env = { + ...Object.entries(env) + .filter(([key]) => settingsEnv && key in settingsEnv) + .reduce((obj, [key, value]) => ({ ...obj, [key]: value }), {}), + }; + + const meta = metaStorageForService(id); + const unlock = lockBusyAction(id, "resume"); + + const runOptions: RunOptions = { + ram: options.ram ?? (defaults.ram as number), + cpu: options.cpu ?? (defaults.cpu as number), + disk: options.disk ?? (defaults.disk as number), + env: env ?? (defaults.env as { [key: string]: string }), + port, + ports: options.ports ?? [], + network, + labels: { + [StandardLabel.Nsm]: "true", + [StandardLabel.ServiceId]: id, + [StandardLabel.NodeId]: nodeId, + [StandardLabel.VolumeId]: id, + [StandardLabel.TemplateId]: template, + }, + }; + + const perma = await db.permaRepository.getPerma(id); + let image = perma.imageId; + + // Propagate other options to env, so they can be used in image processing and building + propagateOptionsToEnv(runOptions, runOptions.env); + // Include service ID in env + runOptions.env.SERVICE_ID = id; + + // Omit the always-changing args from build env, since they would always trigger an + // image rebuild + const { SERVICE_ID, SERVICE_PORT, SERVICE_PORTS, ...buildEnv } = + runOptions.env; + const processedImage = await processImage(image, template, buildEnv); // TODO: tato funkce má poslední parametr messageListener, vymyslet jak sem propagovat message listener z session + // If the image was changed by processing (e.g. it was built or rebuilt), update the image id in database + if (processedImage != image) { + image = processedImage; + + // Update image in database if it was changed by processing + perma.imageId = image; + await db.permaRepository.savePerma(perma); + } + + let session: ActiveServiceSession | undefined; + let containerId: string | undefined; + try { + // Run the container with the built image and save the container id for later use. + if (image) { + session = await beginServiceSession(id); + containerId = await engine.run( + image, + id, + runOptions, meta, - env: env ?? {}, - network - }; - let err: any; - // Save permanent info - if (!await db.permaRepository.savePerma(perma)) { - err = new _InternalError('Failed to save perma info to database'); + buildRunListener(session), + ); } + } catch (e) { + currentContext.logger.error("Failed to run container for service " + id); + currentContext.logger.error(e); + } + + let success: boolean = false; + if (containerId) { + const runningService: RunningService = { + id, + session, + internalSession: { + containerId, + }, + }; + started.push(runningService); + success = true; + } - if (err) { - // Save to be later retrieved - errors[serviceId] = err; - currentContext.logger.error(err.message); - } + if (success == true) { + currentContext.logger.debug("Service " + id + " resumed"); + callManagerEvent("resume", { id }); + } else { + errors[id] = new Error("Failed to resume service"); + clearRunningServiceIfExists(id); + callManagerEvent("resume", { id, error: errors[id] }); + } - if (err) { - throw err; - } else { - return serviceId; - } + unlock(); + + return true; } -export async function resumeService(id: string) { - reqNotRunning(id); - let { - template, - options, - env, - network, - port, - } = await getPermaModel(id); - - const {defaults, env: settingsEnv} = reqTemplate(template).settings; - // Filter env to only those that are defined in settings.yml, because those are the only ones that - // we can guarantee to be used and will not make problems when handling images. - env = { - ...Object.entries(env) - .filter(([key]) => settingsEnv && key in settingsEnv) - .reduce((obj, [key, value]) => ({ ...obj, [key]: value }), {}), - } +export async function stopService(id: string, force?: boolean) { + await reqExists(id); - const meta = metaStorageForService(id); - const unlock = lockBusyAction(id, 'resume'); - - const runOptions: RunOptions = { - ram: options.ram ?? defaults.ram as number, - cpu: options.cpu ?? defaults.cpu as number, - disk: options.disk ?? defaults.disk as number, - env: env ?? defaults.env as {[key: string]: string}, - port, - ports: options.ports ?? [], - network, - labels: { - [StandardLabel.Nsm]: 'true', - [StandardLabel.ServiceId]: id, - [StandardLabel.NodeId]: nodeId, - [StandardLabel.VolumeId]: id, - [StandardLabel.TemplateId]: template, - } - }; + const { internalSession } = reqRunning(id); - const perma = await db.permaRepository.getPerma(id); - let image = perma.imageId; - - // Propagate other options to env, so they can be used in image processing and building - propagateOptionsToEnv(runOptions, runOptions.env); - // Include service ID in env - runOptions.env.SERVICE_ID = id; - - // Omit the always-changing args from build env, since they would always trigger an - // image rebuild - const { SERVICE_ID, SERVICE_PORT, SERVICE_PORTS, ...buildEnv } = runOptions.env; - const processedImage = await processImage(image, template, buildEnv); // TODO: tato funkce má poslední parametr messageListener, vymyslet jak sem propagovat message listener z session - // If the image was changed by processing (e.g. it was built or rebuilt), update the image id in database - if (processedImage != image) { - image = processedImage; - - // Update image in database if it was changed by processing - perma.imageId = image; - await db.permaRepository.savePerma(perma); - } + lckStatusTp(internalSession.containerId, "stop"); + const unlock = lockBusyAction(id, "stop"); - let session: ActiveServiceSession|undefined; - let containerId: string|undefined; - try { - // Run the container with the built image and save the container id for later use. - if (image) { - session = await beginServiceSession(id); - containerId = await engine.run( - image, - id, - runOptions, - meta, - buildRunListener(session) - ); - } - } catch (e) { - currentContext.logger.error('Failed to run container for service ' + id); - currentContext.logger.error(e); - } + try { + on("stop", ({ id: stoppedId, error }) => { + if (stoppedId !== id) { + // This call is not for me + return false; + } - let success: boolean = false; - if (containerId) { - const runningService: RunningService = { - id, - session, - internalSession: { - containerId - } - }; - started.push(runningService); - success = true; - } + if (isServicePending(id)) { + unlock(error); + } + ulckStatusTp(internalSession.containerId); + return true; + }); - if (success == true) { - currentContext.logger.debug('Service ' + id + ' resumed'); - callManagerEvent('resume', { id }); + const meta = metaStorageForService(id); + if (force) { + await engine.kill(internalSession.containerId, meta); } else { - errors[id] = new Error('Failed to resume service'); - clearRunningServiceIfExists(id); - callManagerEvent('resume', { id, error: errors[id] }); + await engine.stop(internalSession.containerId); } + } catch (e) { + currentContext.logger.error(e); - unlock(); - - return true; -} - -export async function stopService(id: string, force?: boolean) { - await reqExists(id); - - const { internalSession } = reqRunning(id); - - lckStatusTp(internalSession.containerId, 'stop'); - const unlock = lockBusyAction(id, 'stop'); - - try { - on("stop", ({ id: stoppedId, error }) => { - if (stoppedId !== id) { - // This call is not for me - return false; - } - - if (isServicePending(id)) { - unlock(error); - } - ulckStatusTp(internalSession.containerId); - return true; - }) - - const meta = metaStorageForService(id); - if (force) { - await engine.kill(internalSession.containerId, meta); - } else { - await engine.stop(internalSession.containerId); - } - } catch (e) { - currentContext.logger.error(e); - - callManagerEvent('stop', { id, error: e }); - } + callManagerEvent("stop", { id, error: e }); + } } export async function stopServiceForcibly(id: string) { - return stopService(id, true); + return stopService(id, true); } export async function sendStopSignal(id: string) { - const perma = await reqExists(id); - const { internalSession } = reqRunning(id); + const perma = await reqExists(id); + const { internalSession } = reqRunning(id); - const stopCmd = perma.meta?.stopCmd; - if (!stopCmd) { - throw new _InternalError('Service does not have stop command set.'); - } + const stopCmd = perma.meta?.stopCmd; + if (!stopCmd) { + throw new _InternalError("Service does not have stop command set."); + } - await engine.cmd(internalSession.containerId, stopCmd); - return true; + await engine.cmd(internalSession.containerId, stopCmd); + return true; } export async function deleteService(id: string) { - try { - await stopService(id, true); - } catch (e) { - // Skip not running error - if (!(e.code && e.code == 2)) { - throw e; - } + try { + await stopService(id, true); + } catch (e) { + // Skip not running error + if (!(e.code && e.code == 2)) { + throw e; } - - const unlockHandler: UnlockObserver = (_, __, ___) => { - const resolveDeleteImageFunc = async () => { - const image = await db.permaRepository.getPerma(id) - .then((perma) => perma.imageId - ? db.imageRepository.getImage(perma.imageId) - : undefined); - - return async () => { - if (image) { - // If the image becomes unused after service deletion, delete it - await deleteImageIfUnused(image); - } - } - }; - - resolveDeleteImageFunc() - .then((deleteImageFunc) => ( - resolveSequentially( - async () => engine.deleteVolume(id), - async () => db.permaRepository.deletePerma(id), - deleteImageFunc, - ) - )) - .then(() => { - currentContext.logger.debug(`Service ${id} deleted`); - }); + } + + const unlockHandler: UnlockObserver = (_, __, ___) => { + const resolveDeleteImageFunc = async () => { + const image = await db.permaRepository + .getPerma(id) + .then((perma) => + perma.imageId + ? db.imageRepository.getImage(perma.imageId) + : undefined, + ); + + return async () => { + if (image) { + // If the image becomes unused after service deletion, delete it + await deleteImageIfUnused(image); + } + }; }; - whenUnlocked(id, unlockHandler); + resolveDeleteImageFunc() + .then((deleteImageFunc) => + resolveSequentially( + async () => engine.deleteVolume(id), + async () => db.permaRepository.deletePerma(id), + deleteImageFunc, + ), + ) + .then(() => { + currentContext.logger.debug(`Service ${id} deleted`); + }); + }; + + whenUnlocked(id, unlockHandler); } export async function updateOptions(id: string, options: Options) { - reqNotPending(id); - const perma = await db.permaRepository.getPerma(id); - const data: PermaModel = { - ...perma, - ...options, - meta: { - ...perma.meta, - ...options.meta, - }, - env: { - ...perma.env, - ...options.env, - }, - }; - return db.permaRepository.savePerma(data); + reqNotPending(id); + const perma = await db.permaRepository.getPerma(id); + const data: PermaModel = { + ...perma, + ...options, + meta: { + ...perma.meta, + ...options.meta, + }, + env: { + ...perma.env, + ...options.env, + }, + }; + return db.permaRepository.savePerma(data); } export function getTemplate(id: string) { - return loadTemplate(id); -} - -export async function getService(from: string, options?: { includeSession?: boolean, otherNodes?: boolean }): ReturnType { - const data = typeof from === 'string' ? await db.permaRepository.getPerma(from) : from; - if (data && (data.nodeId == nodeId || options?.otherNodes === true)) { - let session = undefined; - let internalSession = undefined; - if (options?.includeSession === true) { - const runningService = getRunningService(data.serviceId); - if (runningService) { - session = runningService.session; - internalSession = runningService.internalSession; - } - } - - return { - ...data, - optionsRam: data.env.SERVICE_RAM ? Number(data.env.SERVICE_RAM) : 0, - optionsCpu: data.env.SERVICE_CPU ? Number(data.env.SERVICE_CPU) : 0, - optionsDisk: data.env.SERVICE_DISK ? Number(data.env.SERVICE_DISK) : 0, - state: getServiceState(data.serviceId), - session, - internalSession - } - } else { - return undefined; + return loadTemplate(id); +} + +export async function getService( + from: string, + options?: { includeSession?: boolean; otherNodes?: boolean }, +): ReturnType { + const data = + typeof from === "string" ? await db.permaRepository.getPerma(from) : from; + if (data && (data.nodeId == nodeId || options?.otherNodes === true)) { + let session = undefined; + let internalSession = undefined; + if (options?.includeSession === true) { + const runningService = getRunningService(data.serviceId); + if (runningService) { + session = runningService.session; + internalSession = runningService.internalSession; + } } + + return { + ...data, + optionsRam: data.env.SERVICE_RAM ? Number(data.env.SERVICE_RAM) : 0, + optionsCpu: data.env.SERVICE_CPU ? Number(data.env.SERVICE_CPU) : 0, + optionsDisk: data.env.SERVICE_DISK ? Number(data.env.SERVICE_DISK) : 0, + state: getServiceState(data.serviceId), + session, + internalSession, + }; + } else { + return undefined; + } } export function getLastPowerError(id: string) { - return errors[id]; + return errors[id]; } export async function listServices(options: ListServicesOptions) { - const meta = options.filter?.meta; - return db.permaRepository - .listPerma(nodeId, options.page, options.pageSize, meta) - .then(list => list.map(d => d.serviceId)); + const meta = options.filter?.meta; + return db.permaRepository + .listPerma(nodeId, options.page, options.pageSize, meta) + .then((list) => list.map((d) => d.serviceId)); } export async function listTemplates(): Promise { - return getAllTemplates().map(template => template.id); + return getAllTemplates().map((template) => template.id); } export async function stopRunning() { - const tasks = started.map(({id}) => ( + const tasks = started.map( + ({ id }) => new Promise((resolve) => { - whenUnlocked(id, () => { - stopService(id) - .catch(e => currentContext.logger.error(e)) - .then(() => { - whenUnlocked(id, () => resolve(null)); - }); - }); - }) - )); + whenUnlocked(id, () => { + stopService(id) + .catch((e) => currentContext.logger.error(e)) + .then(() => { + whenUnlocked(id, () => resolve(null)); + }); + }); + }), + ); - await Promise.all(tasks); + await Promise.all(tasks); } export async function waitForBusyAction(id: string) { - return new Promise( - (resolve, reject) => { - whenUnlocked(id, (_, __, err) => err ? reject(err) : resolve(null)); - } - ); + return new Promise((resolve, reject) => { + whenUnlocked(id, (_, __, err) => (err ? reject(err) : resolve(null))); + }); } export function isRunning(id: string) { - return getRunningService(id) != undefined; + return getRunningService(id) != undefined; } export function getRunningService(id: string) { - return started.find(service => service.id === id); + return started.find((service) => service.id === id); } -function metaStorageForService(id: string): MetaStorage { // service id - return { - set: async (key, value) => { - return db.serviceMetaRepository.setServiceMeta(id, key, value); - }, - get: async (key, def) => { - const meta = await db.serviceMetaRepository.getServiceMeta(id, key); - - return meta ?? def; - }, - }; +function metaStorageForService(id: string): MetaStorage { + // service id + return { + set: async (key, value) => { + return db.serviceMetaRepository.setServiceMeta(id, key, value); + }, + get: async (key, def) => { + const meta = await db.serviceMetaRepository.getServiceMeta(id, key); + + return meta ?? def; + }, + }; } export async function initEngineForcibly() { - if (engine) { - throw new Error("Engine is already loaded."); - } - if (!currentContext || !currentContext.appConfig) { - throw new Error("Engine can't be loaded forcibly!"); - } - engine = createEngine(currentContext.appConfig); - // I set it here to keep the exact reference if the engine - // is changed in the future. - engine.cast = () => engine as any; + if (engine) { + throw new Error("Engine is already loaded."); + } + if (!currentContext || !currentContext.appConfig) { + throw new Error("Engine can't be loaded forcibly!"); + } + engine = createEngine(currentContext.appConfig); + // I set it here to keep the exact reference if the engine + // is changed in the future. + engine.cast = () => engine as any; } export function getRunningServices() { - return [...started]; + return [...started]; } -export function on(evt: T, h: EventHandler) { - if (!evtHandlers.has(evt)) { - evtHandlers.set(evt, []); - } - evtHandlers.get(evt).push(h); +export function on( + evt: T, + h: EventHandler, +) { + if (!evtHandlers.has(evt)) { + evtHandlers.set(evt, []); + } + evtHandlers.get(evt).push(h); } -export { - whenUnlocked -} +export { whenUnlocked }; function clearRunningServiceIfExists(id: string) { - const service = getRunningService(id); + const service = getRunningService(id); - if (service) { - started.splice(started.indexOf(service, 1)); - } + if (service) { + started.splice(started.indexOf(service, 1)); + } } -function callManagerEvent(e: T, event: ServiceManagerEvents[T]) { - if (!evtHandlers.has(e)) { - return; - } - const newArray = evtHandlers.get(e) - .filter(handler => { - // Filter out those who returned true, which means they want to be unsubscribed after this call. - const result = handler(event); +function callManagerEvent( + e: T, + event: ServiceManagerEvents[T], +) { + if (!evtHandlers.has(e)) { + return; + } + const newArray = evtHandlers.get(e).filter((handler) => { + // Filter out those who returned true, which means they want to be unsubscribed after this call. + const result = handler(event); - return typeof result != 'boolean' || !result; - }); - evtHandlers.set(e, newArray); + return typeof result != "boolean" || !result; + }); + evtHandlers.set(e, newArray); } /** @@ -859,32 +901,30 @@ function callManagerEvent(e: T, event: Ser * @param session The session for whom to create the session. */ function buildRunListener(session: ActiveServiceSession): RunListener { - const { - serviceId - } = session; - - // The internal run listener of this manager - const internalRunListener: RunListener = { - onStateChange: (state) => { - startedStates.set(serviceId, state.ready ? 'RUNNING' : 'BUILDING'); - }, - onClose: async () => { - clearRunningServiceIfExists(serviceId); - startedStates.delete(serviceId); - - // Call stop event on the manager for the stopService() to potentially - // unlock a busy action - callManagerEvent("stop", { id: serviceId }); - - currentContext.logger.debug("Service " + serviceId + " stopped"); - } - }; - // Combine collected listeners - return combineRunListeners([ - internalRunListener, - // Add listener from the session - session.runListener - ]) + const { serviceId } = session; + + // The internal run listener of this manager + const internalRunListener: RunListener = { + onStateChange: (state) => { + startedStates.set(serviceId, state.ready ? "RUNNING" : "BUILDING"); + }, + onClose: async () => { + clearRunningServiceIfExists(serviceId); + startedStates.delete(serviceId); + + // Call stop event on the manager for the stopService() to potentially + // unlock a busy action + callManagerEvent("stop", { id: serviceId }); + + currentContext.logger.debug("Service " + serviceId + " stopped"); + }, + }; + // Combine collected listeners + return combineRunListeners([ + internalRunListener, + // Add listener from the session + session.runListener, + ]); } /** @@ -894,51 +934,50 @@ function buildRunListener(session: ActiveServiceSession): RunListener { * @returns The state of the service */ function getServiceState(id: string) { - return startedStates.get(id) ?? 'STOPPED'; + return startedStates.get(id) ?? "STOPPED"; } // --------------------------------------------------------------------------------------- async function getPermaModel(id: string) { - const perma_ = await db.permaRepository.getPerma(id); - if (!perma_) { - // Service does not exist - throw new _InternalError('Not found.', 3); - } + const perma_ = await db.permaRepository.getPerma(id); + if (!perma_) { + // Service does not exist + throw new _InternalError("Not found.", 3); + } - return perma_; + return perma_; } async function reqExists(id: string) { - const perma = await db.permaRepository.getPerma(id); - if (!perma) { - throw new _InternalError("Service not found.", 3); - } + const perma = await db.permaRepository.getPerma(id); + if (!perma) { + throw new _InternalError("Service not found.", 3); + } - return perma; + return perma; } function reqRunning(id: string) { - const session = getRunningService(id); - if (!session) { - throw new _InternalError("This service is not running.", 2); - } + const session = getRunningService(id); + if (!session) { + throw new _InternalError("This service is not running.", 2); + } - return session; + return session; } function reqNotRunning(id: string) { - if (isRunning(id)) { - throw new _InternalError('Already running.', 2); - } + if (isRunning(id)) { + throw new _InternalError("Already running.", 2); + } } function reqTemplate(id: string) { - const template = getTemplate(id); - if (!template) { - throw new _InternalError('' + - 'Template not found.', 3); - } + const template = getTemplate(id); + if (!template) { + throw new _InternalError("" + "Template not found.", 3); + } - return template; -} \ No newline at end of file + return template; +} diff --git a/src/engine/middle.ts b/src/engine/middle.ts index f29c912..f2cfd0e 100644 --- a/src/engine/middle.ts +++ b/src/engine/middle.ts @@ -1,7 +1,13 @@ -import {_InternalError, ServiceManager} from "@nsm/engine/manager"; -import {currentContext} from "@nsm/app"; +import { _InternalError, ServiceManager } from "@nsm/engine/manager"; +import { currentContext } from "@nsm/app"; -export type ServiceActionType = 'create' | 'resume' | 'stop' | 'forceStop' | 'sendStopSignal' | 'delete'; +export type ServiceActionType = + | "create" + | "resume" + | "stop" + | "forceStop" + | "sendStopSignal" + | "delete"; /** * Represents an error that occurred during a service action. @@ -13,7 +19,6 @@ export interface ServiceActionError { } export interface ErrorPublisher { - /** * Publishes an error that occurred during a service action. * @@ -35,15 +40,15 @@ const publishers: ErrorPublisher[] = [ */ export const registerErrorPublisher = (publisher: ErrorPublisher) => { publishers.push(publisher); -} +}; const publishError = async (action: ServiceActionError) => { try { - await Promise.all(publishers.map(p => p.publishError(action))); + await Promise.all(publishers.map((p) => p.publishError(action))); } catch (e) { - currentContext.logger.error('Failed to publish service action error', e); + currentContext.logger.error("Failed to publish service action error", e); } -} +}; /** * Decorates an asynchronous function to allow for additional behavior, such as error handling or logging. @@ -56,7 +61,7 @@ const publishError = async (action: ServiceActionError) => { const decorateFunc = ) => Promise>( fn: F, actionType: ServiceActionType, - serviceIdExtractor: (args: Parameters) => string = (args) => args[0] as string, + serviceIdExtractor?: (args: Parameters) => string, ) => { return async (...args: Parameters) => { try { @@ -70,16 +75,53 @@ const decorateFunc = ) => Promise>( await publishError(action); // don't log stack trace of known errors - const errorMeta: any[] = e instanceof _InternalError && e.code != 1 ? [] : [e]; + const errorMeta: any[] = + e instanceof _InternalError && e.code != 1 ? [] : [e]; currentContext.logger.error( `${action.serviceId ? `Service ${action.serviceId} f` : "F"}ailed action ${action.type}: ${action.message}`, - ...errorMeta + ...errorMeta, ); throw e; } - } -} + }; +}; + +/** + * Creates a service ID extractor function that extracts the service ID from the + * specified argument index of the function arguments. + * + * @param argIndex The index of the argument from which to extract the service ID. + * @returns A function that takes the function arguments and returns the extracted service ID. + */ +const argServiceIdExtractor = ( + argIndex: number, +): () => Promise>( + args: Parameters, +) => string) => { + return (args) => { + if (!Array.isArray(args)) { + throw new Error("Expected function call arguments to be an array"); + } + + const argsArray = args as unknown[]; + // Check if the argument index is within bounds + if (argsArray.length <= argIndex) { + throw new Error( + `Expected at least ${argIndex + 1} arguments, but got ${argsArray.length}`, + ); + } + + const serviceId = argsArray[argIndex]; + if (typeof serviceId !== "string") { + throw new Error( + `Expected service ID argument to be a string, but got ${typeof serviceId}`, + ); + } + + return serviceId; + }; +}; /** * Wraps a {@link ServiceManager} instance with additional capabilities. @@ -93,16 +135,36 @@ export const middleLayer = (manager: ServiceManager): ServiceManager => { return { ...manager, - createService: decorateFunc(manager.createService, "create", null), - - resumeService: decorateFunc(manager.resumeService, "resume"), - - stopService: decorateFunc(manager.stopService, "stop"), - - stopServiceForcibly: decorateFunc(manager.stopServiceForcibly, "forceStop"), - - sendStopSignal: decorateFunc(manager.sendStopSignal, "sendStopSignal"), - - deleteService: decorateFunc(manager.deleteService, "delete"), - } -} \ No newline at end of file + createService: decorateFunc(manager.createService, "create"), + + resumeService: decorateFunc( + manager.resumeService, + "resume", + argServiceIdExtractor(0), + ), + + stopService: decorateFunc( + manager.stopService, + "stop", + argServiceIdExtractor(0), + ), + + stopServiceForcibly: decorateFunc( + manager.stopServiceForcibly, + "forceStop", + argServiceIdExtractor(0), + ), + + sendStopSignal: decorateFunc( + manager.sendStopSignal, + "sendStopSignal", + argServiceIdExtractor(0), + ), + + deleteService: decorateFunc( + manager.deleteService, + "delete", + argServiceIdExtractor(0), + ), + }; +}; diff --git a/src/engine/monitoring/templateDirWatcher.ts b/src/engine/monitoring/templateDirWatcher.ts index 4cda3b5..6e82d43 100644 --- a/src/engine/monitoring/templateDirWatcher.ts +++ b/src/engine/monitoring/templateDirWatcher.ts @@ -1,14 +1,13 @@ -import {templateBuildDir, debounce} from "@nsm/engine/monitoring/util"; -import {hashElement} from "folder-hash"; -import {getFilteredPaths} from "@nsm/engine/ignore"; -import {getAllTemplates} from "@nsm/engine/template"; +import { templateBuildDir, debounce } from "@nsm/engine/monitoring/util"; +import { hashElement } from "folder-hash"; +import { getFilteredPaths } from "@nsm/engine/ignore"; +import { getAllTemplates } from "@nsm/engine/template"; import winston from "winston"; -import chokidar, {FSWatcher} from "chokidar"; +import chokidar, { FSWatcher } from "chokidar"; import path from "path"; -import {getTemplatesPath} from "@nsm/filestructure"; +import { getTemplatesPath } from "@nsm/filestructure"; export type TemplateDirWatcher = { - /** * Starts watching the template directories for changes. * When a change is detected, the template hash is updated and cached. @@ -33,12 +32,12 @@ export const watchTemplateDirChanges = (logger: winston.Logger) => { const templates = getAllTemplates(); // Populate on startup - templates.forEach(template => watchTemplateDir(template.id)); + templates.forEach((template) => watchTemplateDir(template.id)); // Watch the base directory for new templates watchBaseDir(logger); logger.info("Watching template directories for changes..."); -} +}; /** * Watches the base templates directory for new template directories being added or removed. @@ -56,7 +55,9 @@ const watchBaseDir = (logger: winston.Logger) => { watcher.on("addDir", async (path_) => { const template = path.basename(path_); if (template && !watchers.has(template)) { - logger.debug(`New template directory detected: ${template}. Starting to watch for changes...`); + logger.debug( + `New template directory detected: ${template}. Starting to watch for changes...`, + ); await watchTemplateDir(template); } @@ -67,7 +68,8 @@ const watchBaseDir = (logger: winston.Logger) => { const tWatcher = watchers.get(template); if (tWatcher) { logger.debug( - `Template directory removed: ${template}. Stopping watch and removing hash from cache...`); + `Template directory removed: ${template}. Stopping watch and removing hash from cache...`, + ); await tWatcher.close(); } @@ -75,8 +77,8 @@ const watchBaseDir = (logger: winston.Logger) => { watchers.delete(template); hashCache.delete(template); } - }) -} + }); +}; /** * Watches a specific template directory for changes and updates the hash cache when a change is detected. @@ -103,13 +105,13 @@ const watchTemplateDir = async (template: string) => { // and unnecessary rehashing. awaitWriteFinish: { stabilityThreshold: 500, - pollInterval: 100 - } + pollInterval: 100, + }, }); watcher.on("all", recalc); watchers.set(template, watcher); -} +}; /** * Recalculates the hash of a template directory and updates the cache. @@ -128,19 +130,19 @@ const recalculateTemplateHash = async (template: string) => { try { const hash = await hashElement(dir, { - encoding: 'hex', + encoding: "hex", folders: { - exclude: excluded.dirs + exclude: excluded.dirs, }, files: { - exclude: excluded.files - } + exclude: excluded.files, + }, }); hashCache.set(template, hash.hash); } finally { hashingInProgress.delete(template); } -} +}; export const getTemplateHash = (template: string): string => { const hash = hashCache.get(template); @@ -149,4 +151,4 @@ export const getTemplateHash = (template: string): string => { } return hash; -} \ No newline at end of file +}; diff --git a/src/engine/monitoring/util.ts b/src/engine/monitoring/util.ts index 112e2c7..f171755 100644 --- a/src/engine/monitoring/util.ts +++ b/src/engine/monitoring/util.ts @@ -1,5 +1,5 @@ -import path from 'path'; -import {getTemplatesPath} from "@nsm/filestructure"; +import path from "path"; +import { getTemplatesPath } from "@nsm/filestructure"; // Returns the build directory for the template export function templateBuildDir(template: string) { @@ -27,4 +27,4 @@ export const debounce = (fn: () => void | Promise, ms: number) => { fn(); }, ms); }; -}; \ No newline at end of file +}; diff --git a/src/engine/session.ts b/src/engine/session.ts index 20eb06c..44b8a55 100644 --- a/src/engine/session.ts +++ b/src/engine/session.ts @@ -1,11 +1,11 @@ -import {RunListener} from "@nsm/engine/engine"; +import { RunListener } from "@nsm/engine/engine"; import { CreateLogRecordArgs, Database, ListRecordsArgs, ListSessionsArgs, ServiceLogRecordModel, - ServiceSessionModel + ServiceSessionModel, } from "@nsm/database"; export interface SessionManager { @@ -13,8 +13,12 @@ export interface SessionManager { beginServiceSession(serviceId: string): Promise; - listSessions(args: ListSessionsArgs): Promise; - listSessionLogs(args: ListRecordsArgs): Promise; + listSessions( + args: ListSessionsArgs, + ): Promise; + listSessionLogs( + args: ListRecordsArgs, + ): Promise; } export interface ServiceSession { @@ -35,7 +39,7 @@ let db: Database; export const init = (db_: Database) => { db = db_; -} +}; /** * Begins a new service session for the given service ID. @@ -43,47 +47,43 @@ export const init = (db_: Database) => { * @param serviceId The ID of the service for which to begin a session. * @return An object representing the active service session. */ -export const beginServiceSession: SessionManager["beginServiceSession"] = async ( - serviceId: string -): Promise => { - let session = await db.sessionRepository.createSession(serviceId); - - // Debounce the push in bulk to prevent database overhead - const { - flush: flushRecords, - debounce: pushRecord - } = debounceBulkPush(); - - const runListener: RunListener = { - onStateChange: async (state) => { - pushRecord({ - sessionId: session.id, - source: 'ENGINE', - logLevel: 'INFO', - message: state.description - }); - }, - onMessage: async (record) => { - pushRecord({ - sessionId: session.id, - source: 'CONTAINER', - logLevel: record.level.toUpperCase(), - message: record.message - }); - }, - onClose: async () => { - // Push remaining logs now - await flushRecords(); - - // TODO: mark session as closed - } - } - - return { - ...session, - runListener - } -} +export const beginServiceSession: SessionManager["beginServiceSession"] = + async (serviceId: string): Promise => { + let session = await db.sessionRepository.createSession(serviceId); + + // Debounce the push in bulk to prevent database overhead + const { flush: flushRecords, debounce: pushRecord } = debounceBulkPush(); + + const runListener: RunListener = { + onStateChange: async (state) => { + pushRecord({ + sessionId: session.id, + source: "ENGINE", + logLevel: "INFO", + message: state.description, + }); + }, + onMessage: async (record) => { + pushRecord({ + sessionId: session.id, + source: "CONTAINER", + logLevel: record.level.toUpperCase(), + message: record.message, + }); + }, + onClose: async () => { + // Push remaining logs now + await flushRecords(); + + // TODO: mark session as closed + }, + }; + + return { + ...session, + runListener, + }; + }; /** * Creates a debounced function for pushing log records in bulk to the database. @@ -94,12 +94,12 @@ export const beginServiceSession: SessionManager["beginServiceSession"] = async * @return A function that can be called to push a log record, which will be debounced and pushed in bulk. */ const debounceBulkPush = () => { - const logRecordsBulk: Omit[] = []; + const logRecordsBulk: Omit[] = []; const MAX_BATCH_SIZE = 50; const DEBOUNCE_MS = 500; - let timeout: NodeJS.Timeout|null = null; + let timeout: NodeJS.Timeout | null = null; let isFlushing = false; const flush = async () => { @@ -163,16 +163,20 @@ const debounceBulkPush = () => { // Renew timer renew(); - } - } -} + }, + }; +}; // TODO: get service session -export const listSessions: SessionManager["listSessions"] = async (args: ListSessionsArgs) => { +export const listSessions: SessionManager["listSessions"] = async ( + args: ListSessionsArgs, +) => { return db.sessionRepository.listSessions(args); -} +}; -export const listSessionLogs: SessionManager["listSessionLogs"] = async (args: ListRecordsArgs) => { +export const listSessionLogs: SessionManager["listSessionLogs"] = async ( + args: ListRecordsArgs, +) => { return db.serviceLogRepository.listRecords(args); -} \ No newline at end of file +}; diff --git a/src/engine/template.ts b/src/engine/template.ts index f6b7828..02b20a1 100644 --- a/src/engine/template.ts +++ b/src/engine/template.ts @@ -1,110 +1,122 @@ -import {loadYamlFile} from "@nsm/util/yaml"; +import { loadYamlFile } from "@nsm/util/yaml"; import * as fs from "fs"; import path from "path"; -import {getTemplatesPath} from "@nsm/filestructure"; +import { getTemplatesPath } from "@nsm/filestructure"; export type Template = { - /** - * The unique ID of the template. - */ - id: string, - /** - * The display name of the template, used for display purposes. - */ - name: string; - /** - * A short description of the template, used for display purposes. - */ - description: string; - /** - * The settings (definitions) object. - */ - settings: any; -} + /** + * The unique ID of the template. + */ + id: string; + /** + * The display name of the template, used for display purposes. + */ + name: string; + /** + * A short description of the template, used for display purposes. + */ + description: string; + /** + * The settings (definitions) object. + */ + settings: any; +}; export type TemplateManager = { + /** + * Prepares the environment variables for a template by validating the provided env object against + * the template's settings and filling in default values where necessary. It checks for required options, validates + * types, and returns a new env object that can be used when creating a service from the template. + * + * @param template The template or template ID for which to prepare the environment variables + * @param env The environment variables provided by the user, which may be incomplete or have incorrect types + * @return A new env object that has been validated and filled with default values according to the template's settings + * @throws Error if a required option is missing or if an option has an invalid type + */ + prepareEnvForTemplate(template: Template | string, env: any): any; - /** - * Prepares the environment variables for a template by validating the provided env object against - * the template's settings and filling in default values where necessary. It checks for required options, validates - * types, and returns a new env object that can be used when creating a service from the template. - * - * @param template The template or template ID for which to prepare the environment variables - * @param env The environment variables provided by the user, which may be incomplete or have incorrect types - * @return A new env object that has been validated and filled with default values according to the template's settings - * @throws Error if a required option is missing or if an option has an invalid type - */ - prepareEnvForTemplate(template: Template | string, env: any): any; + /** + * Returns a template by ID. + * + * @param id The ID of the template + * @return The template, or null if not exists + */ + getTemplate(id: string): Template | null; - /** - * Returns a template by ID. - * - * @param id The ID of the template - * @return The template, or null if not exists - */ - getTemplate(id: string): Template|null; - - getAllTemplates(): Template[]; -} + getAllTemplates(): Template[]; +}; const templateCache = {}; -export const getTemplate = (id: string): Template|null => { - if (templateCache[id]) { - return templateCache[id]; - } - const settingsPath = path.join(getTemplatesPath(), id, 'settings.yml'); - if (!fs.existsSync(settingsPath)) { - return null; - } - const settings = loadYamlFile(settingsPath); - const template = { - id, - name: settings.name, - description: settings.description, - settings - }; - templateCache[id] = template; - return template; -} +export const getTemplate = (id: string): Template | null => { + if (templateCache[id]) { + return templateCache[id]; + } + const settingsPath = path.join(getTemplatesPath(), id, "settings.yml"); + if (!fs.existsSync(settingsPath)) { + return null; + } + const settings = loadYamlFile(settingsPath); + const template = { + id, + name: settings.name, + description: settings.description, + settings, + }; + templateCache[id] = template; + return template; +}; export const getAllTemplates = () => { - if (!fs.existsSync(getTemplatesPath())) { - return []; - } + if (!fs.existsSync(getTemplatesPath())) { + return []; + } - return fs - .readdirSync(getTemplatesPath()) - .filter(file => fs.statSync(path.join(getTemplatesPath(), file)).isDirectory()) - .map(id => getTemplate(id)) - .filter(template => template !== null); -} + return fs + .readdirSync(getTemplatesPath()) + .filter((file) => + fs.statSync(path.join(getTemplatesPath(), file)).isDirectory(), + ) + .map((id) => getTemplate(id)) + .filter((template) => template !== null); +}; -export const prepareEnvForTemplate = (template: Template | string, env: any) => { - env = { ...env }; // Shallow copy to avoid mutating the original object - if (typeof template === 'string') { - template = getTemplate(template); // Load the template if ID provided - } +export const prepareEnvForTemplate = ( + template: Template | string, + env: any, +) => { + env = { ...env }; // Shallow copy to avoid mutating the original object + if (typeof template === "string") { + template = getTemplate(template); // Load the template if ID provided + } - for (const key of Object.keys(template.settings['env'])) { - if (env[key] && typeof env[key] == typeof template.settings['env'][key]) { - // Keep the value - } else if (env[key]) { - throw new Error('Invalid option type for ' + key + '. Got ' + typeof env[key] + ' but expected ' + typeof template.settings['env'][key] + '.'); - } else if (isRequiredOption(template.settings['env'][key])) { - throw new Error('Missing required option ' + key); - } else { - // Set default - env[key] = template.settings['env'][key]; - } + for (const key of Object.keys(template.settings["env"])) { + if (env[key] && typeof env[key] == typeof template.settings["env"][key]) { + // Keep the value + } else if (env[key]) { + throw new Error( + "Invalid option type for " + + key + + ". Got " + + typeof env[key] + + " but expected " + + typeof template.settings["env"][key] + + ".", + ); + } else if (isRequiredOption(template.settings["env"][key])) { + throw new Error("Missing required option " + key); + } else { + // Set default + env[key] = template.settings["env"][key]; } - return env; -} + } + return env; +}; // Defines if the value represents required option. const isRequiredOption = (value: any) => { - return ( - (typeof value == "string" && value === "") || - (typeof value === "number" && value == -1) - ) -} \ No newline at end of file + return ( + (typeof value == "string" && value === "") || + (typeof value === "number" && value == -1) + ); +}; diff --git a/src/filestructure.ts b/src/filestructure.ts index 682bf6f..214bd70 100644 --- a/src/filestructure.ts +++ b/src/filestructure.ts @@ -1,6 +1,6 @@ import path from "path"; -import envPaths, {Paths} from "env-paths"; -import {AppConfig} from "@nsm/config"; +import envPaths, { Paths } from "env-paths"; +import { AppConfig } from "@nsm/config"; import fs from "fs"; export const currentPaths: Paths = envPaths("nsm"); @@ -9,7 +9,7 @@ let appConfig: AppConfig; export const init = (appConfig_: AppConfig) => { appConfig = appConfig_; -} +}; // The local resources dir (not the source of truth) export const resourcesPath = path.join(process.cwd(), "resources"); @@ -17,28 +17,30 @@ export const resourcesPath = path.join(process.cwd(), "resources"); // The target (platform-agnostic) resources dir (the source of truth) export const getResourcesTargetPath = () => { return appConfig.getResourcesPath() ?? path.join(currentPaths.data); -} +}; export const getTemplatesPath = () => { - return path.join(getResourcesTargetPath(), 'templates') -} + return path.join(getResourcesTargetPath(), "templates"); +}; export const getTempPath = () => { return currentPaths.temp; -} +}; export const mkdirTemp = (...p: string[]) => { const dir = path.join(getTempPath(), ...p); if (fs.existsSync(dir)) { if (!fs.statSync(dir).isDirectory()) { - throw new Error('Temp path already exists and is not a directory: ' + dir); + throw new Error( + "Temp path already exists and is not a directory: " + dir, + ); } } else { fs.mkdirSync(dir, { recursive: true }); } return dir; -} +}; export const prepareFolders = () => { const resourcesTargetPath = getResourcesTargetPath(); @@ -55,4 +57,4 @@ export const prepareFolders = () => { if (!fs.existsSync(tempPath)) { fs.mkdirSync(tempPath, { recursive: true }); } -} \ No newline at end of file +}; diff --git a/src/helpers.ts b/src/helpers.ts index 8a6da96..688a40a 100644 --- a/src/helpers.ts +++ b/src/helpers.ts @@ -1,10 +1,10 @@ export function isDebug() { - return process.env.DEBUG === 'true'; + return process.env.DEBUG === "true"; } export function consumeEnginePowerAction(action: () => Promise) { - action().catch((e) => { - // Manager service power action errors are ignored since - // they are handled by the middle-layer defined in engine/middle.ts - }); -} \ No newline at end of file + action().catch((e) => { + // Manager service power action errors are ignored since + // they are handled by the middle-layer defined in engine/middle.ts + }); +} diff --git a/src/logger.ts b/src/logger.ts index 9e88488..522ce15 100644 --- a/src/logger.ts +++ b/src/logger.ts @@ -1,37 +1,48 @@ import winston from "winston"; import fs from "fs"; import path from "path"; -import {getResourcesTargetPath} from "@nsm/filestructure"; +import { getResourcesTargetPath } from "@nsm/filestructure"; const { combine, timestamp, label, errors, printf } = winston.format; export function createLatestLogFile() { - if (fs.existsSync(path.join(getResourcesTargetPath(), 'logs', 'latest.log'))) { - const date = new Date(Date.now()).toJSON().slice(2, 10) + '.' - + new Date(Date.now()).getHours() + '.' - + new Date(Date.now()).getMinutes(); + if ( + fs.existsSync(path.join(getResourcesTargetPath(), "logs", "latest.log")) + ) { + const date = + new Date(Date.now()).toJSON().slice(2, 10) + + "." + + new Date(Date.now()).getHours() + + "." + + new Date(Date.now()).getMinutes(); - fs.renameSync(path.join(getResourcesTargetPath(), 'logs', 'latest.log'), path.join(getResourcesTargetPath(), 'logs', date + '.log')); - } + fs.renameSync( + path.join(getResourcesTargetPath(), "logs", "latest.log"), + path.join(getResourcesTargetPath(), "logs", date + ".log"), + ); + } } export function createLogger(options?: { label?: string }) { - const debug = process.env.DEBUG === 'true'; - return winston.createLogger({ - level: debug ? 'debug' : 'info', - format: combine( - errors({ stack: true }), - label({ label: options?.label ?? 'NSM' }), - timestamp(), - printf(({ level, message, label, timestamp, stack }) => { - let row = `${timestamp} [${label}] ${level}: ${message}`; + const debug = process.env.DEBUG === "true"; + return winston.createLogger({ + level: debug ? "debug" : "info", + format: combine( + errors({ stack: true }), + label({ label: options?.label ?? "NSM" }), + timestamp(), + printf(({ level, message, label, timestamp, stack }) => { + let row = `${timestamp} [${label}] ${level}: ${message}`; - return stack ? row + `\n${stack}` : row; - }) - ), - transports: [ - new winston.transports.Console(), - new winston.transports.File({dirname: path.join(getResourcesTargetPath(), 'logs'), filename: 'latest.log'}) - ] - }); -} \ No newline at end of file + return stack ? row + `\n${stack}` : row; + }), + ), + transports: [ + new winston.transports.Console(), + new winston.transports.File({ + dirname: path.join(getResourcesTargetPath(), "logs"), + filename: "latest.log", + }), + ], + }); +} diff --git a/src/networking/manager.ts b/src/networking/manager.ts index e900fbd..eb4ec4f 100644 --- a/src/networking/manager.ts +++ b/src/networking/manager.ts @@ -1,57 +1,64 @@ import DockerClient from "dockerode"; -export async function accessNetwork(client: DockerClient, ip: string, id: string) { - let net = client.getNetwork(id); - try { - await net.inspect(); - } catch (e) { - if (e.message.includes('not found')) { - net = await createNetwork(client, ip); - } else { - // Something unexpected occurred here. - throw e; - } +export async function accessNetwork( + client: DockerClient, + ip: string, + id: string, +) { + let net = client.getNetwork(id); + try { + await net.inspect(); + } catch (e) { + if (e.message.includes("not found")) { + net = await createNetwork(client, ip); + } else { + // Something unexpected occurred here. + throw e; } - return net; + } + return net; } export async function createNetwork(client: DockerClient, ip: string) { - const uuid = crypto.randomUUID(); - return client.createNetwork({ - Name: uuid, - Driver: 'bridge', - Options: { - 'com.docker.network.bridge.enable_icc': 'true', // Inter-container connectivity, may disable - 'com.docker.network.bridge.enable_ip_masquerade': 'true', - 'com.docker.network.bridge.host_binding_ipv4': ip, - 'com.docker.network.bridge.name': uuid, - 'com.docker.network.driver.mtu': '1500' - }, - Labels: { - 'nsm': 'true', - } - }); + const uuid = crypto.randomUUID(); + return client.createNetwork({ + Name: uuid, + Driver: "bridge", + Options: { + "com.docker.network.bridge.enable_icc": "true", // Inter-container connectivity, may disable + "com.docker.network.bridge.enable_ip_masquerade": "true", + "com.docker.network.bridge.host_binding_ipv4": ip, + "com.docker.network.bridge.name": uuid, + "com.docker.network.driver.mtu": "1500", + }, + Labels: { + nsm: "true", + }, + }); } export async function deleteNetwork(client: DockerClient, id: string) { - try { - await client.getNetwork(id).remove(); - } catch (e) { - if (!e.message.toLowerCase().includes('no such network')) { - console.log(e); - } + try { + await client.getNetwork(id).remove(); + } catch (e) { + if (!e.message.toLowerCase().includes("no such network")) { + console.log(e); } + } } // Returns network id, or undef if not in net -export async function isInNetwork(client: DockerClient, containerId: string): Promise { - try { - await client.getNetwork(containerId).inspect(); - return containerId; - } catch (e) { - if (!e.message.includes('not found')) { - console.log(e); - } - return undefined; +export async function isInNetwork( + client: DockerClient, + containerId: string, +): Promise { + try { + await client.getNetwork(containerId).inspect(); + return containerId; + } catch (e) { + if (!e.message.includes("not found")) { + console.log(e); } -} \ No newline at end of file + return undefined; + } +} diff --git a/src/profiler/index.ts b/src/profiler/index.ts index e2f5f4c..66bb0f6 100644 --- a/src/profiler/index.ts +++ b/src/profiler/index.ts @@ -1,7 +1,7 @@ export function measureEventLoop() { - var time = process.hrtime(); - process.nextTick(function() { - var diff = process.hrtime(time); - console.log('event loop took %d nanoseconds', diff[0] * 1e9 + diff[1]); - }); -} \ No newline at end of file + var time = process.hrtime(); + process.nextTick(function () { + var diff = process.hrtime(time); + console.log("event loop took %d nanoseconds", diff[0] * 1e9 + diff[1]); + }); +} diff --git a/src/resources.ts b/src/resources.ts index d735a3c..e0d7672 100644 --- a/src/resources.ts +++ b/src/resources.ts @@ -1,6 +1,6 @@ import path from "path"; import fs from "fs"; -import {getResourcesTargetPath, resourcesPath} from "@nsm/filestructure"; +import { getResourcesTargetPath, resourcesPath } from "@nsm/filestructure"; /** * Reads resource from target dir. @@ -10,8 +10,8 @@ import {getResourcesTargetPath, resourcesPath} from "@nsm/filestructure"; export const readResource = (name: string) => { const p = path.join(getResourcesTargetPath(), name); - return fs.readFileSync(p, 'utf8'); -} + return fs.readFileSync(p, "utf8"); +}; /** * Creates a directory in the target dir. Creates parent dirs if missing. @@ -22,7 +22,7 @@ export const mkdirResource = (name: string) => { const p = path.join(getResourcesTargetPath(), name); fs.mkdirSync(p, { recursive: true }); -} +}; /** * Saves resource to target dir. Creates parent dirs if missing. @@ -36,7 +36,7 @@ export const saveResource = ( name: string, targetName: string, skipIfExists: boolean = false, - targetDirPath: string = getResourcesTargetPath() + targetDirPath: string = getResourcesTargetPath(), ) => { const targetPath = path.join(targetDirPath, targetName); // Create parent dirs if missing @@ -46,7 +46,7 @@ export const saveResource = ( return; } fs.writeFileSync(targetPath, readCwdResource(name)); -} +}; /** * Reads resource from resources dir. @@ -56,5 +56,5 @@ export const saveResource = ( export const readCwdResource = (name: string) => { const p = path.join(resourcesPath, name); - return fs.readFileSync(p, 'utf8'); -} \ No newline at end of file + return fs.readFileSync(p, "utf8"); +}; diff --git a/src/router/index.ts b/src/router/index.ts index 30e9a95..5419d4b 100644 --- a/src/router/index.ts +++ b/src/router/index.ts @@ -1,57 +1,61 @@ -import {AppContext} from "../app"; -import {json, RequestHandler, Router} from "express"; +import { AppContext } from "../app"; +import { json, RequestHandler, Router } from "express"; import v1Routes from "./v1"; -import {measureEventLoop} from "@nsm/profiler"; +import { measureEventLoop } from "@nsm/profiler"; export type RouterHandler = { - url: string; - routes: {[method: string]: RequestHandler}; + url: string; + routes: { [method: string]: RequestHandler }; }; type RouterInit = (context: AppContext) => Promise; // Load API by version async function api(ver: string, context: AppContext, routes: RouterInit[]) { - const router = Router(); - router.use(json()); - if (context.debug) { - router.use((req, res, next) => { - if (req.body) { - context.logger.debug(`Body: ${JSON.stringify(req.body)}`); - } else { - context.logger.debug('No body'); - } - next(); - }); - // Measure event loop process time if in debug mode - router.use((_, __, next) => { - measureEventLoop(); + const router = Router(); + router.use(json()); + if (context.debug) { + router.use((req, res, next) => { + if (req.body) { + context.logger.debug(`Body: ${JSON.stringify(req.body)}`); + } else { + context.logger.debug("No body"); + } + next(); + }); + // Measure event loop process time if in debug mode + router.use((_, __, next) => { + measureEventLoop(); + next(); + }); + } + for (let init of routes) { + // Create handler with changed router to the sub-router that will be + // used specifically for this API version + const handler = await init({ ...context, router }); + let reg = false; + + for (const method of ["get", "post", "put", "delete"]) { + if (handler.routes[method]) { + // Register handler to express + router[method]( + handler.url, + (req, res, next) => { + context.logger.debug(`${method.toUpperCase()} ${req.url}`); next(); - }); + }, + handler.routes[method], + ); + reg = true; + } } - for (let init of routes) { - // Create handler with changed router to the sub-router that will be - // used specifically for this API version - const handler = await init({ ...context, router }); - let reg = false; - - for (const method of ['get', 'post', 'put', 'delete']) { - if (handler.routes[method]) { - // Register handler to express - router[method](handler.url, (req, res, next) => { - context.logger.debug(`${method.toUpperCase()} ${req.url}`); - next(); - }, handler.routes[method]); - reg = true; - } - } - if (reg) { - context.logger.debug(`Registered route ${handler.url}`); - } + if (reg) { + context.logger.debug(`Registered route ${handler.url}`); } - context.router.use(`/${ver}`, router); + } + context.router.use(`/${ver}`, router); } export default async function (context: AppContext) { - await api('v1', context, v1Routes); // v1 -} \ No newline at end of file + await api("v1", context, v1Routes); // v1 +} diff --git a/src/router/util/preconditions.ts b/src/router/util/preconditions.ts index b9e7b3a..d471544 100644 --- a/src/router/util/preconditions.ts +++ b/src/router/util/preconditions.ts @@ -1,25 +1,35 @@ import express from "express"; -import {isServicePending} from "@nsm/engine/asyncp"; -import {handleErrorMessage} from "@nsm/util/routes"; -import {ServiceManager} from "@nsm/engine"; +import { isServicePending } from "@nsm/engine/asyncp"; +import { handleErrorMessage } from "@nsm/util/routes"; +import { ServiceManager } from "@nsm/engine"; export const checkServiceExists = async ( - serviceId: string, manager: ServiceManager, res: express.Response) => { - if (!await manager.getService(serviceId)) { - handleErrorMessage(404, 'Service not found.', res); + serviceId: string, + manager: ServiceManager, + res: express.Response, +) => { + if (!(await manager.getService(serviceId))) { + handleErrorMessage(404, "Service not found.", res); return false; } return true; -} +}; -export const checkServicePending = (serviceId: string, res: express.Response) => { +export const checkServicePending = ( + serviceId: string, + res: express.Response, +) => { if (isServicePending(serviceId)) { - handleErrorMessage(409, 'Service is pending another action. Please wait a moment.', res); + handleErrorMessage( + 409, + "Service is pending another action. Please wait a moment.", + res, + ); return false; } return true; -}; \ No newline at end of file +}; diff --git a/src/router/v1/index.ts b/src/router/v1/index.ts index 2534223..2e657bd 100644 --- a/src/router/v1/index.ts +++ b/src/router/v1/index.ts @@ -28,5 +28,5 @@ export default [ listRoute, sessionsRoute, logsRoute, - sessionLogsRoute -] \ No newline at end of file + sessionLogsRoute, +]; diff --git a/src/router/v1/service/createRoute.ts b/src/router/v1/service/createRoute.ts index 49e634b..8549f27 100644 --- a/src/router/v1/service/createRoute.ts +++ b/src/router/v1/service/createRoute.ts @@ -1,54 +1,66 @@ -import {RouterHandler} from "../../index"; -import {AppContext} from "@nsm/app"; -import {Options} from "@nsm/engine"; -import {clock} from "@nsm/util/clock"; -import {prepareEnvForTemplate} from "@nsm/engine/template"; -import {consumeEnginePowerAction} from "@nsm/helpers"; +import { RouterHandler } from "../../index"; +import { AppContext } from "@nsm/app"; +import { Options } from "@nsm/engine"; +import { clock } from "@nsm/util/clock"; +import { prepareEnvForTemplate } from "@nsm/engine/template"; +import { consumeEnginePowerAction } from "@nsm/helpers"; -export default async function ({manager}: AppContext): Promise { - return { - url: '/service/create', - routes: { - post: async (req, res) => { - const clk = clock(); - if (!req.body || !req.body.template) { - res.status(400).json({status: 400, message: 'Missing body or template key.'}).end(); - return; - } - const template = manager.getTemplate(req.body.template); - if (!template) { - res.status(400).json({status: 400, message: 'Invalid template ID.'}).end(); - return; - } - let env = req.body.env ?? {}; - try { - env = prepareEnvForTemplate(template, env); - } catch (e) { - res.status(400).json({status: 400, message: e.message}).end(); - return; - } +export default async function ({ + manager, +}: AppContext): Promise { + return { + url: "/service/create", + routes: { + post: async (req, res) => { + const clk = clock(); + if (!req.body || !req.body.template) { + res + .status(400) + .json({ status: 400, message: "Missing body or template key." }) + .end(); + return; + } + const template = manager.getTemplate(req.body.template); + if (!template) { + res + .status(400) + .json({ status: 400, message: "Invalid template ID." }) + .end(); + return; + } + let env = req.body.env ?? {}; + try { + env = prepareEnvForTemplate(template, env); + } catch (e) { + res.status(400).json({ status: 400, message: e.message }).end(); + return; + } - // Build options - const options: Options = req.body; - options.env = env; - // Create the service - try { - const serviceId = await manager.createService(template.id, options); + // Build options + const options: Options = req.body; + options.env = env; + // Create the service + try { + const serviceId = await manager.createService(template.id, options); - // Resume right afterward - consumeEnginePowerAction(() => manager.resumeService(serviceId)); + // Resume right afterward + consumeEnginePowerAction(() => manager.resumeService(serviceId)); - res.status(200).json({ - status: 200, - message: 'Service create action successfully registered to be completed in a moment.', - serviceId, - statusPath: '/v1/service/' + serviceId + '/powerstatus', - time: clk.durFromCreation() - }).end(); - } catch (e) { - res.status(500).json({status: 500, message: e.message}).end(); - } - } - }, - } -} \ No newline at end of file + res + .status(200) + .json({ + status: 200, + message: + "Service create action successfully registered to be completed in a moment.", + serviceId, + statusPath: "/v1/service/" + serviceId + "/powerstatus", + time: clk.durFromCreation(), + }) + .end(); + } catch (e) { + res.status(500).json({ status: 500, message: e.message }).end(); + } + }, + }, + }; +} diff --git a/src/router/v1/service/deleteRoute.ts b/src/router/v1/service/deleteRoute.ts index 86bc7e8..63b2f3c 100644 --- a/src/router/v1/service/deleteRoute.ts +++ b/src/router/v1/service/deleteRoute.ts @@ -1,29 +1,36 @@ -import {AppContext} from "@nsm/app"; -import {RouterHandler} from "../../index"; -import {handleErr} from "@nsm/util/routes"; -import {checkServiceExists} from "@nsm/router/util/preconditions"; +import { AppContext } from "@nsm/app"; +import { RouterHandler } from "../../index"; +import { handleErr } from "@nsm/util/routes"; +import { checkServiceExists } from "@nsm/router/util/preconditions"; -export default async function ({manager}: AppContext): Promise { - return { - url: '/service/:id/delete', - routes: { - post: async (req, res) => { - const id = req.params.id; - if (!id) { - res.status(400).json({status: 400, message: 'Required \'id\' field not present in the body.'}); - return; - } - if (!await checkServiceExists(id, manager, res)) { - return; - } - try { - await manager.deleteService(id); +export default async function ({ + manager, +}: AppContext): Promise { + return { + url: "/service/:id/delete", + routes: { + post: async (req, res) => { + const id = req.params.id; + if (!id) { + res + .status(400) + .json({ + status: 400, + message: "Required 'id' field not present in the body.", + }); + return; + } + if (!(await checkServiceExists(id, manager, res))) { + return; + } + try { + await manager.deleteService(id); - res.status(200).json({status: 200, message: 'Service deleted.'}); - } catch (e) { - handleErr(e, res); - } - } - }, - } -} \ No newline at end of file + res.status(200).json({ status: 200, message: "Service deleted." }); + } catch (e) { + handleErr(e, res); + } + }, + }, + }; +} diff --git a/src/router/v1/service/listRoute.ts b/src/router/v1/service/listRoute.ts index 27cb1f2..4b06c19 100644 --- a/src/router/v1/service/listRoute.ts +++ b/src/router/v1/service/listRoute.ts @@ -1,62 +1,83 @@ -import {AppContext} from "../../../app"; -import {RouterHandler} from "../../index"; -import {ListServicesOptions} from "@nsm/engine"; +import { AppContext } from "../../../app"; +import { RouterHandler } from "../../index"; +import { ListServicesOptions } from "@nsm/engine"; import z from "zod"; -export default async function ({manager, database}: AppContext): Promise { - return { - url: '/servicelist', - routes: { - post: async (req, res) => { - const page = req.body.page ?? 0; - const pageSize = req.body.pageSize ?? 10; - const meta = req.body.meta; - if (typeof page !== 'number' || typeof pageSize !== 'number' || page < 0 || pageSize < 1) { - res.status(400).json({status: 400, message: 'Invalid page or pageSize.'}).end(); - return; - } +export default async function ({ + manager, + database, +}: AppContext): Promise { + return { + url: "/servicelist", + routes: { + post: async (req, res) => { + const page = req.body.page ?? 0; + const pageSize = req.body.pageSize ?? 10; + const meta = req.body.meta; + if ( + typeof page !== "number" || + typeof pageSize !== "number" || + page < 0 || + pageSize < 1 + ) { + res + .status(400) + .json({ status: 400, message: "Invalid page or pageSize." }) + .end(); + return; + } - // Options for the query - const listOptions: ListServicesOptions = { - page, - pageSize, - }; + // Options for the query + const listOptions: ListServicesOptions = { + page, + pageSize, + }; - // Meta is optional in req body - if (meta) { - // Validate and parse meta - const metaParse = z - .object({}) - // Pass unrecognized keys - .passthrough() - .refine((data) => { - // Allow only primitives (no nested objects) - return Object.keys(data).every((key) => (typeof data[key]) !== "object") - }, { - message: "Meta should contain only primitives." - }) - .safeParse(meta); - if (metaParse.success) { - listOptions.filter = { meta: metaParse.data }; - } else { - res.status(400) - .json({status: 400, message: 'Invalid meta filter format.', error: metaParse.error}) - .end(); - return; - } - } + // Meta is optional in req body + if (meta) { + // Validate and parse meta + const metaParse = z + .object({}) + // Pass unrecognized keys + .passthrough() + .refine( + (data) => { + // Allow only primitives (no nested objects) + return Object.keys(data).every( + (key) => typeof data[key] !== "object", + ); + }, + { + message: "Meta should contain only primitives.", + }, + ) + .safeParse(meta); + if (metaParse.success) { + listOptions.filter = { meta: metaParse.data }; + } else { + res + .status(400) + .json({ + status: 400, + message: "Invalid meta filter format.", + error: metaParse.error, + }) + .end(); + return; + } + } - // Response body - const data = { - services: await manager.listServices(listOptions), - meta: { - ...listOptions, - // Total num of services on this node - total: await database.permaRepository.countPerma(manager.nodeId), - } - }; - res.status(200).json(data).end(); - } - }, - } -} \ No newline at end of file + // Response body + const data = { + services: await manager.listServices(listOptions), + meta: { + ...listOptions, + // Total num of services on this node + total: await database.permaRepository.countPerma(manager.nodeId), + }, + }; + res.status(200).json(data).end(); + }, + }, + }; +} diff --git a/src/router/v1/service/logsRoute.ts b/src/router/v1/service/logsRoute.ts index 61ea880..55aa338 100644 --- a/src/router/v1/service/logsRoute.ts +++ b/src/router/v1/service/logsRoute.ts @@ -1,19 +1,24 @@ -import {AppContext} from "@nsm/app"; -import {RouterHandler} from "@nsm/router"; -import {ListRecordsArgs} from "@nsm/database"; -import {checkServiceExists} from "@nsm/router/util/preconditions"; +import { AppContext } from "@nsm/app"; +import { RouterHandler } from "@nsm/router"; +import { ListRecordsArgs } from "@nsm/database"; +import { checkServiceExists } from "@nsm/router/util/preconditions"; -export default async function(ctx: AppContext): Promise { +export default async function (ctx: AppContext): Promise { return { - url: '/service/:id/logs', + url: "/service/:id/logs", routes: { get: async (req, res) => { const id = req.params.id; if (!id) { - res.status(400).json({status: 400, message: 'Required \'id\' field not present in the body.'}); + res + .status(400) + .json({ + status: 400, + message: "Required 'id' field not present in the body.", + }); return; } - if (!await checkServiceExists(id, ctx.manager, res)) { + if (!(await checkServiceExists(id, ctx.manager, res))) { return; } @@ -28,7 +33,7 @@ export default async function(ctx: AppContext): Promise { const lastSession = await ctx.sessionManager.listSessions({ filter: { serviceId: id }, sort: { by: "startedAt", direction: "desc" }, - page: { index: 0, size: 1 } + page: { index: 0, size: 1 }, }); if (lastSession && lastSession.length > 0) { sessionId = lastSession[0].id; @@ -36,7 +41,9 @@ export default async function(ctx: AppContext): Promise { } if (!sessionId) { - res.status(400).json({status: 400, message: 'Service was never active.'}); + res + .status(400) + .json({ status: 400, message: "Service was never active." }); return; } @@ -44,29 +51,28 @@ export default async function(ctx: AppContext): Promise { const pageSize = req.query.pageSize ? Number(req.query.pageSize) : 10; // Use pagination only if it was requested by params - const page = req.query.pageIndex || req.query.pageSize - ? ( - { - index: pageIndex, - size: pageSize - } - ) - : undefined; + const page = + req.query.pageIndex || req.query.pageSize + ? { + index: pageIndex, + size: pageSize, + } + : undefined; const args: ListRecordsArgs = { filter: { - sessionId + sessionId, }, sort: { by: "timestamp", - direction: "asc" + direction: "asc", }, - page + page, }; const logs = await ctx.sessionManager.listSessionLogs(args); res.status(200).json({ logs }); - } - } - } -} \ No newline at end of file + }, + }, + }; +} diff --git a/src/router/v1/service/lookupRoute.ts b/src/router/v1/service/lookupRoute.ts index f9d10ec..3c18a82 100644 --- a/src/router/v1/service/lookupRoute.ts +++ b/src/router/v1/service/lookupRoute.ts @@ -1,45 +1,50 @@ -import {AppContext} from "@nsm/app"; -import {RouterHandler} from "../../index"; +import { AppContext } from "@nsm/app"; +import { RouterHandler } from "../../index"; -export default async function ({manager}: AppContext): Promise { - return { - url: '/service/:id', - routes: { - get: async (req, res) => { - const id = req.params.id; +export default async function ({ + manager, +}: AppContext): Promise { + return { + url: "/service/:id", + routes: { + get: async (req, res) => { + const id = req.params.id; - const service = await manager.getService(id, { includeSession: true }); - if (!service) { - res.status(404).json({status: 404, message: 'Invalid service ID.'}).end(); - return; - } + const service = await manager.getService(id, { includeSession: true }); + if (!service) { + res + .status(404) + .json({ status: 404, message: "Invalid service ID." }) + .end(); + return; + } - const session = service.internalSession; - let stats: any; - if (session && req.query.stats === 'true') { - stats = await manager.engine.stat(session.containerId); - } else { - stats = null; - } + const session = service.internalSession; + let stats: any; + if (session && req.query.stats === "true") { + stats = await manager.engine.stat(session.containerId); + } else { + stats = null; + } - const data: any = { - id: service.serviceId, - templateId: service.template, - state: service.state, - port: service.port, - options: service.options, - env: service.env - }; - if (session) { - data.session = { - id: service.session.id, - startedAt: service.session.startedAt.getTime(), - stats, - }; - } + const data: any = { + id: service.serviceId, + templateId: service.template, + state: service.state, + port: service.port, + options: service.options, + env: service.env, + }; + if (session) { + data.session = { + id: service.session.id, + startedAt: service.session.startedAt.getTime(), + stats, + }; + } - res.json(data).end(); - }, - }, - } -} \ No newline at end of file + res.json(data).end(); + }, + }, + }; +} diff --git a/src/router/v1/service/optionsRoute.ts b/src/router/v1/service/optionsRoute.ts index dfa4401..105aeba 100644 --- a/src/router/v1/service/optionsRoute.ts +++ b/src/router/v1/service/optionsRoute.ts @@ -1,31 +1,50 @@ -import {AppContext} from "../../../app"; -import {RouterHandler} from "../../index"; +import { AppContext } from "../../../app"; +import { RouterHandler } from "../../index"; -export default async function ({manager}: AppContext): Promise { - return { - url: '/service/:id/options', - routes: { - post: async (req, res) => { - const id = req.params.id; - if (!id) { - res.status(400).json({status: 400, message: 'Required \'id\' field not present in the body.'}); - return; - } - const options = req.body; - if (!options) { - res.status(400).json({status: 400, message: 'Body is required.'}); - return; - } - if (Object.keys(options).includes('port') || Object.keys(options).includes('ports')) { - res.status(400).json({status: 400, message: 'Port(s) cannot be changed yet.'}); - return; - } - if (await manager.updateOptions(id, options)) { - res.status(200).json({status: 200, message: 'Service options updated.'}); - } else { - res.status(404).json({status: 404, message: 'Service not found or unknown error occured.'}); - } - } - }, - } -} \ No newline at end of file +export default async function ({ + manager, +}: AppContext): Promise { + return { + url: "/service/:id/options", + routes: { + post: async (req, res) => { + const id = req.params.id; + if (!id) { + res + .status(400) + .json({ + status: 400, + message: "Required 'id' field not present in the body.", + }); + return; + } + const options = req.body; + if (!options) { + res.status(400).json({ status: 400, message: "Body is required." }); + return; + } + if ( + Object.keys(options).includes("port") || + Object.keys(options).includes("ports") + ) { + res + .status(400) + .json({ status: 400, message: "Port(s) cannot be changed yet." }); + return; + } + if (await manager.updateOptions(id, options)) { + res + .status(200) + .json({ status: 200, message: "Service options updated." }); + } else { + res + .status(404) + .json({ + status: 404, + message: "Service not found or unknown error occured.", + }); + } + }, + }, + }; +} diff --git a/src/router/v1/service/powerStatusRoute.ts b/src/router/v1/service/powerStatusRoute.ts index c2cce21..e3db385 100644 --- a/src/router/v1/service/powerStatusRoute.ts +++ b/src/router/v1/service/powerStatusRoute.ts @@ -1,30 +1,37 @@ -import {AppContext} from "../../../app"; -import {RouterHandler} from "../../index"; -import {isServicePending} from "@nsm/engine/asyncp"; +import { AppContext } from "../../../app"; +import { RouterHandler } from "../../index"; +import { isServicePending } from "@nsm/engine/asyncp"; -export default async function ({manager}: AppContext): Promise { - return { - url: '/service/:id/powerstatus', - routes: { - get: async (req, res) => { - const id = req.params.id; - if (!id) { - res.status(400).json({status: 400, message: 'Required \'id\' field not present in the body.'}); - return; - } - let status = 'IDLE'; - let error = undefined; - if (isServicePending(id)) { - status = 'PENDING'; - } else { - const err = manager.getLastPowerError(id); - if (err) { - status = 'ERROR'; - error = err; - } - } - res.status(200).json({ id, status, error }).end(); - } - }, - } -} \ No newline at end of file +export default async function ({ + manager, +}: AppContext): Promise { + return { + url: "/service/:id/powerstatus", + routes: { + get: async (req, res) => { + const id = req.params.id; + if (!id) { + res + .status(400) + .json({ + status: 400, + message: "Required 'id' field not present in the body.", + }); + return; + } + let status = "IDLE"; + let error = undefined; + if (isServicePending(id)) { + status = "PENDING"; + } else { + const err = manager.getLastPowerError(id); + if (err) { + status = "ERROR"; + error = err; + } + } + res.status(200).json({ id, status, error }).end(); + }, + }, + }; +} diff --git a/src/router/v1/service/rebootRoute.ts b/src/router/v1/service/rebootRoute.ts index 9820372..af6961f 100644 --- a/src/router/v1/service/rebootRoute.ts +++ b/src/router/v1/service/rebootRoute.ts @@ -1,45 +1,56 @@ -import {AppContext} from "@nsm/app"; -import {RouterHandler} from "../../index"; -import {checkServiceExists, checkServicePending} from "@nsm/router/util/preconditions"; -import {consumeEnginePowerAction} from "@nsm/helpers"; +import { AppContext } from "@nsm/app"; +import { RouterHandler } from "../../index"; +import { + checkServiceExists, + checkServicePending, +} from "@nsm/router/util/preconditions"; +import { consumeEnginePowerAction } from "@nsm/helpers"; -export default async function ({manager, logger}: AppContext): Promise { - return { - url: '/service/:id/reboot', - routes: { - post: async (req, res) => { - const id = req.params.id; - if (!id) { - res.status(400).json({status: 400, message: 'Required \'id\' field not present in the body.'}); - return; - } - if (!await checkServiceExists(id, manager, res)) { - return; - } - if (!checkServicePending(id, res)) { - return; - } +export default async function ({ + manager, + logger, +}: AppContext): Promise { + return { + url: "/service/:id/reboot", + routes: { + post: async (req, res) => { + const id = req.params.id; + if (!id) { + res + .status(400) + .json({ + status: 400, + message: "Required 'id' field not present in the body.", + }); + return; + } + if (!(await checkServiceExists(id, manager, res))) { + return; + } + if (!checkServicePending(id, res)) { + return; + } - consumeEnginePowerAction(() => ( - manager.stopService(id) - .then(() => { - // Service stopped successfully, now wait for it to be unlocked before resuming. + consumeEnginePowerAction(() => + manager.stopService(id).then(() => { + // Service stopped successfully, now wait for it to be unlocked before resuming. - manager.whenUnlocked(id, (_, __, err) => { - if (err) { - logger.error(err); - } else { - manager.resumeService(id); - } - }); - }) - )); + manager.whenUnlocked(id, (_, __, err) => { + if (err) { + logger.error(err); + } else { + manager.resumeService(id); + } + }); + }), + ); - res.status(200).json({ - status: 200, - message: 'Service reboot action successfully registered to be completed in a moment.' - }); - } - }, - } -} \ No newline at end of file + res.status(200).json({ + status: 200, + message: + "Service reboot action successfully registered to be completed in a moment.", + }); + }, + }, + }; +} diff --git a/src/router/v1/service/resumeRoute.ts b/src/router/v1/service/resumeRoute.ts index 54e8e9f..91e0c0f 100644 --- a/src/router/v1/service/resumeRoute.ts +++ b/src/router/v1/service/resumeRoute.ts @@ -1,37 +1,50 @@ -import {AppContext} from "@nsm/app"; -import {RouterHandler} from "../../index"; -import {checkServiceExists, checkServicePending} from "@nsm/router/util/preconditions"; -import {consumeEnginePowerAction} from "@nsm/helpers"; +import { AppContext } from "@nsm/app"; +import { RouterHandler } from "../../index"; +import { + checkServiceExists, + checkServicePending, +} from "@nsm/router/util/preconditions"; +import { consumeEnginePowerAction } from "@nsm/helpers"; -export default async function ({manager}: AppContext): Promise { - return { - url: '/service/:id/resume', - routes: { - post: async (req, res) => { - const id = req.params.id; - if (!id) { - res.status(400).json({status: 400, message: 'Required \'id\' field not present in the body.'}); - return; - } - if (!await checkServiceExists(id, manager, res)) { - return; - } - if (!checkServicePending(id, res)) { - return; - } - if (manager.isRunning(id)) { - res.status(409).json({status: 400, message: 'Service is already running.'}); - return; - } +export default async function ({ + manager, +}: AppContext): Promise { + return { + url: "/service/:id/resume", + routes: { + post: async (req, res) => { + const id = req.params.id; + if (!id) { + res + .status(400) + .json({ + status: 400, + message: "Required 'id' field not present in the body.", + }); + return; + } + if (!(await checkServiceExists(id, manager, res))) { + return; + } + if (!checkServicePending(id, res)) { + return; + } + if (manager.isRunning(id)) { + res + .status(409) + .json({ status: 400, message: "Service is already running." }); + return; + } - consumeEnginePowerAction(() => manager.resumeService(id)); + consumeEnginePowerAction(() => manager.resumeService(id)); - res.status(200).json({ - status: 200, - message: 'Service resume action successfully registered to be completed in a moment.', - statusPath: '/v1/service/' + id + '/powerstatus', - }); - } - }, - } -} \ No newline at end of file + res.status(200).json({ + status: 200, + message: + "Service resume action successfully registered to be completed in a moment.", + statusPath: "/v1/service/" + id + "/powerstatus", + }); + }, + }, + }; +} diff --git a/src/router/v1/service/sessionsRoute.ts b/src/router/v1/service/sessionsRoute.ts index 8426652..8d8a854 100644 --- a/src/router/v1/service/sessionsRoute.ts +++ b/src/router/v1/service/sessionsRoute.ts @@ -1,11 +1,11 @@ -import {AppContext} from "@nsm/app"; -import {RouterHandler} from "@nsm/router"; -import {checkServiceExists} from "@nsm/router/util/preconditions"; -import {ListSessionsArgs} from "@nsm/database"; +import { AppContext } from "@nsm/app"; +import { RouterHandler } from "@nsm/router"; +import { checkServiceExists } from "@nsm/router/util/preconditions"; +import { ListSessionsArgs } from "@nsm/database"; -export default async function(ctx: AppContext): Promise { +export default async function (ctx: AppContext): Promise { return { - url: '/service/:id/sessions', + url: "/service/:id/sessions", routes: { get: async (req, res) => { const id = req.params.id; @@ -13,29 +13,29 @@ export default async function(ctx: AppContext): Promise { const pageIndex = req.query.pageIndex ? Number(req.query.pageIndex) : 0; const pageSize = req.query.pageSize ? Number(req.query.pageSize) : 10; - if (!await checkServiceExists(id, ctx.manager, res)) { + if (!(await checkServiceExists(id, ctx.manager, res))) { return; } const args: ListSessionsArgs = { filter: { - serviceId: id + serviceId: id, }, sort: { by: "startedAt", - direction: "desc" + direction: "desc", }, page: { index: pageIndex, - size: pageSize - } + size: pageSize, + }, }; const sessionIds = await ctx.sessionManager .listSessions(args) - .then(sessions => sessions.map(session => session.id)); + .then((sessions) => sessions.map((session) => session.id)); res.status(200).json({ sessions: sessionIds }); - } - } - } -} \ No newline at end of file + }, + }, + }; +} diff --git a/src/router/v1/service/stopCmdRoute.ts b/src/router/v1/service/stopCmdRoute.ts index f11b90e..21860b7 100644 --- a/src/router/v1/service/stopCmdRoute.ts +++ b/src/router/v1/service/stopCmdRoute.ts @@ -1,37 +1,51 @@ -import {AppContext} from "@nsm/app"; -import {RouterHandler} from "@nsm/router"; -import {isServicePending} from "@nsm/engine/asyncp"; -import {handleErr} from "@nsm/util/routes"; -import {checkServicePending} from "@nsm/router/util/preconditions"; +import { AppContext } from "@nsm/app"; +import { RouterHandler } from "@nsm/router"; +import { isServicePending } from "@nsm/engine/asyncp"; +import { handleErr } from "@nsm/util/routes"; +import { checkServicePending } from "@nsm/router/util/preconditions"; -export default async function ({manager}: AppContext): Promise { - return { - url: '/service/:id/stopcmd', - routes: { - post: async (req, res) => { - const id = req.params.id; - if (!id) { - res.status(400).json({status: 400, message: 'Required \'id\' field not present in the body.'}); - return; - } - if (!checkServicePending(id, res)) { - return; - } - if (!await manager.getService(id)) { - res.status(404).json({status: 404, message: 'Service not found.'}); - return; - } - try { - const result = await manager.sendStopSignal(id); - if (result) { - res.status(200).json({status: 200, message: 'Service stop signal sent.'}); - } else { - res.status(404).json({status: 404, message: 'Service not found or unknown error occured.'}); - } - } catch (e) { - handleErr(e, res); - } - } - }, - } -} \ No newline at end of file +export default async function ({ + manager, +}: AppContext): Promise { + return { + url: "/service/:id/stopcmd", + routes: { + post: async (req, res) => { + const id = req.params.id; + if (!id) { + res + .status(400) + .json({ + status: 400, + message: "Required 'id' field not present in the body.", + }); + return; + } + if (!checkServicePending(id, res)) { + return; + } + if (!(await manager.getService(id))) { + res.status(404).json({ status: 404, message: "Service not found." }); + return; + } + try { + const result = await manager.sendStopSignal(id); + if (result) { + res + .status(200) + .json({ status: 200, message: "Service stop signal sent." }); + } else { + res + .status(404) + .json({ + status: 404, + message: "Service not found or unknown error occured.", + }); + } + } catch (e) { + handleErr(e, res); + } + }, + }, + }; +} diff --git a/src/router/v1/service/stopRoute.ts b/src/router/v1/service/stopRoute.ts index dd782a8..488127f 100644 --- a/src/router/v1/service/stopRoute.ts +++ b/src/router/v1/service/stopRoute.ts @@ -1,43 +1,57 @@ -import {AppContext} from "@nsm/app"; -import {RouterHandler} from "../../index"; -import {checkServiceExists, checkServicePending} from "@nsm/router/util/preconditions"; -import {consumeEnginePowerAction} from "@nsm/helpers"; +import { AppContext } from "@nsm/app"; +import { RouterHandler } from "../../index"; +import { + checkServiceExists, + checkServicePending, +} from "@nsm/router/util/preconditions"; +import { consumeEnginePowerAction } from "@nsm/helpers"; -export default async function ({manager, logger}: AppContext): Promise { - return { - url: '/service/:id/stop', - routes: { - post: async (req, res) => { - const id = req.params.id; - if (!id) { - res.status(400).json({status: 400, message: 'Required \'id\' field not present in the body.'}); - return; - } - if (!await checkServiceExists(id, manager, res)) { - return; - } - if (!checkServicePending(id, res)) { - return; - } - if (!manager.isRunning(id)) { - res.status(409).json({status: 400, message: 'Service is not running.'}); - return; - } +export default async function ({ + manager, + logger, +}: AppContext): Promise { + return { + url: "/service/:id/stop", + routes: { + post: async (req, res) => { + const id = req.params.id; + if (!id) { + res + .status(400) + .json({ + status: 400, + message: "Required 'id' field not present in the body.", + }); + return; + } + if (!(await checkServiceExists(id, manager, res))) { + return; + } + if (!checkServicePending(id, res)) { + return; + } + if (!manager.isRunning(id)) { + res + .status(409) + .json({ status: 400, message: "Service is not running." }); + return; + } - consumeEnginePowerAction(async () => { - if (req.query.force === 'true') { - await manager.stopServiceForcibly(id); - } else { - await manager.stopService(id) - } - }); + consumeEnginePowerAction(async () => { + if (req.query.force === "true") { + await manager.stopServiceForcibly(id); + } else { + await manager.stopService(id); + } + }); - res.status(200).json({ - status: 200, - message: 'Service stop action successfully registered to be completed in a moment.', - statusPath: '/v1/service/' + id + '/powerstatus', - }); - } - }, - } -} \ No newline at end of file + res.status(200).json({ + status: 200, + message: + "Service stop action successfully registered to be completed in a moment.", + statusPath: "/v1/service/" + id + "/powerstatus", + }); + }, + }, + }; +} diff --git a/src/router/v1/session/sessionLogsRoute.ts b/src/router/v1/session/sessionLogsRoute.ts index dc111bc..423294e 100644 --- a/src/router/v1/session/sessionLogsRoute.ts +++ b/src/router/v1/session/sessionLogsRoute.ts @@ -1,10 +1,10 @@ -import {AppContext} from "@nsm/app"; -import {RouterHandler} from "@nsm/router"; -import {ListRecordsArgs} from "@nsm/database"; +import { AppContext } from "@nsm/app"; +import { RouterHandler } from "@nsm/router"; +import { ListRecordsArgs } from "@nsm/database"; -export default async function(ctx: AppContext): Promise { +export default async function (ctx: AppContext): Promise { return { - url: '/session/:id/logs', + url: "/session/:id/logs", routes: { get: async (req, res) => { const id = req.params.id; @@ -13,29 +13,28 @@ export default async function(ctx: AppContext): Promise { const pageSize = req.query.pageSize ? Number(req.query.pageSize) : 10; // Use pagination only if it was requested by params - const page = req.query.pageIndex || req.query.pageSize - ? ( - { - index: pageIndex, - size: pageSize - } - ) - : undefined; + const page = + req.query.pageIndex || req.query.pageSize + ? { + index: pageIndex, + size: pageSize, + } + : undefined; const args: ListRecordsArgs = { filter: { - sessionId: id + sessionId: id, }, sort: { by: "timestamp", - direction: "asc" + direction: "asc", }, - page + page, }; const logs = await ctx.sessionManager.listSessionLogs(args); res.status(200).json({ logs }); - } - } - } -} \ No newline at end of file + }, + }, + }; +} diff --git a/src/router/v1/status/index.ts b/src/router/v1/status/index.ts index aa8497d..3388a02 100644 --- a/src/router/v1/status/index.ts +++ b/src/router/v1/status/index.ts @@ -1,48 +1,52 @@ -import {AppContext} from "@nsm/app"; -import {RouterHandler} from "../../index"; +import { AppContext } from "@nsm/app"; +import { RouterHandler } from "../../index"; import * as os from "os"; -import {Filters, ServiceManager} from "@nsm/engine"; -import {Database} from "@nsm/database"; +import { Filters, ServiceManager } from "@nsm/engine"; +import { Database } from "@nsm/database"; async function checkNsmResources(engine: ServiceManager, db: Database) { - const stats = await engine.engine.statAll(Filters.node(engine.nodeId)); - const servicesGlobal = await db.permaRepository.listPerma(engine.nodeId); - const res = stats.reduce((acc, s) => { - acc.memory.used += s.memory.used; - acc.memory.total += s.memory.total; - acc.cpu.used += s.cpu.used; - acc.cpu.total += s.cpu.total; - return acc; - }, { - memory: { - used: 0, - total: 0, - percent: 0, - }, - cpu: { - used: 0, - total: 0, - percent: 0, - }, - services: { // TODO: Ukazuje stále 0??? - memTotal: BigInt(0), - cpuTotal: BigInt(0), - diskTotal: BigInt(0), - }, - }); - for (const s of servicesGlobal) { - const service = await engine.getService(s); - res.services.memTotal += BigInt(service.optionsRam); - res.services.cpuTotal += BigInt(service.optionsCpu); - res.services.diskTotal += BigInt(service.optionsDisk); - } - if (res.memory.total > 0) { - res.memory.percent = res.memory.used / res.memory.total; - } - if (res.cpu.total > 0) { - res.cpu.percent = res.cpu.used / res.cpu.total; - } - return res; + const stats = await engine.engine.statAll(Filters.node(engine.nodeId)); + const servicesGlobal = await db.permaRepository.listPerma(engine.nodeId); + const res = stats.reduce( + (acc, s) => { + acc.memory.used += s.memory.used; + acc.memory.total += s.memory.total; + acc.cpu.used += s.cpu.used; + acc.cpu.total += s.cpu.total; + return acc; + }, + { + memory: { + used: 0, + total: 0, + percent: 0, + }, + cpu: { + used: 0, + total: 0, + percent: 0, + }, + services: { + // TODO: Ukazuje stále 0??? + memTotal: BigInt(0), + cpuTotal: BigInt(0), + diskTotal: BigInt(0), + }, + }, + ); + for (const s of servicesGlobal) { + const service = await engine.getService(s); + res.services.memTotal += BigInt(service.optionsRam); + res.services.cpuTotal += BigInt(service.optionsCpu); + res.services.diskTotal += BigInt(service.optionsDisk); + } + if (res.memory.total > 0) { + res.memory.percent = res.memory.used / res.memory.total; + } + if (res.cpu.total > 0) { + res.cpu.percent = res.cpu.used / res.cpu.total; + } + return res; } /** @@ -51,29 +55,36 @@ async function checkNsmResources(engine: ServiceManager, db: Database) { * * @param context The app context */ -export default async function ({manager, appConfig, database}: AppContext): Promise { - return { - url: '/status', - routes: { - get: async (req, res) => { - const nodeId = appConfig.getNodeId(); - const all = await database.permaRepository.listPerma(nodeId); - const [free, size] = await manager.engine.calcHostUsage(); - const system = { - totalmem: os.totalmem(), - freemem: os.freemem(), - totaldisk: size, - freedisk: free, - } - res.json({ - nodeId, - running: manager.getRunningServices() - .map(s => s.id), - all: all.length, - system, - ...(req.query.stats === 'true' ? { stats: await checkNsmResources(manager, database) } : {}) - }).end(); - }, - }, - } -} \ No newline at end of file +export default async function ({ + manager, + appConfig, + database, +}: AppContext): Promise { + return { + url: "/status", + routes: { + get: async (req, res) => { + const nodeId = appConfig.getNodeId(); + const all = await database.permaRepository.listPerma(nodeId); + const [free, size] = await manager.engine.calcHostUsage(); + const system = { + totalmem: os.totalmem(), + freemem: os.freemem(), + totaldisk: size, + freedisk: free, + }; + res + .json({ + nodeId, + running: manager.getRunningServices().map((s) => s.id), + all: all.length, + system, + ...(req.query.stats === "true" + ? { stats: await checkNsmResources(manager, database) } + : {}), + }) + .end(); + }, + }, + }; +} diff --git a/src/security/index.ts b/src/security/index.ts index ee20b05..d46fa04 100644 --- a/src/security/index.ts +++ b/src/security/index.ts @@ -1,11 +1,11 @@ -import {AppContext} from "../app"; -import token from './token'; +import { AppContext } from "../app"; +import token from "./token"; export default async function (ctx: AppContext) { - // This code block is initialized before app routes. - if (ctx.appConfig.getAuth() == 'auth_token') { - // Basic credentials auth type - await token(ctx); - } - ctx.logger.info('Using ' + ctx.appConfig.getAuth() + ' auth.'); -} \ No newline at end of file + // This code block is initialized before app routes. + if (ctx.appConfig.getAuth() == "auth_token") { + // Basic credentials auth type + await token(ctx); + } + ctx.logger.info("Using " + ctx.appConfig.getAuth() + " auth."); +} diff --git a/src/security/token/index.ts b/src/security/token/index.ts index 0cd367e..62d7c35 100644 --- a/src/security/token/index.ts +++ b/src/security/token/index.ts @@ -1,29 +1,36 @@ -import {AppContext} from "../../app"; +import { AppContext } from "../../app"; import crypto from "crypto"; -export default async function ({database, router, logger}: AppContext) { - const token_new = crypto.randomBytes(30).toString('hex'); - const token = process.env.NSM_TOKEN ?? await database.metaRepository.getMetaVal('auth:basic_token', token_new); - router.use((req, res, next) => { - if (!req.header('Authorization') || req.header('Authorization') != token) { - res.status(401).json({ status: 401, message: 'Unauthorized. Invalid \'Authorization\' header.' }); - return; - } - next(); - }); - if (token == token_new) { - setTimeout(() => { - logger.info('=============================================='); - logger.info('Your Authorization token has been generated'); - logger.info('since you enabled \'auth_token\' authorization'); - logger.info('for the first time. Please copy it and keep safe.'); - logger.info('You will need to use it while requesting NSM.'); - logger.info(''); - logger.info('Token: ' + token); - logger.info('=============================================='); - }, 500); +export default async function ({ database, router, logger }: AppContext) { + const token_new = crypto.randomBytes(30).toString("hex"); + const token = + process.env.NSM_TOKEN ?? + (await database.metaRepository.getMetaVal("auth:basic_token", token_new)); + router.use((req, res, next) => { + if (!req.header("Authorization") || req.header("Authorization") != token) { + res + .status(401) + .json({ + status: 401, + message: "Unauthorized. Invalid 'Authorization' header.", + }); + return; } - if (process.env.NSM_TOKEN) { - logger.info('Authorization token loaded from env'); - } -} \ No newline at end of file + next(); + }); + if (token == token_new) { + setTimeout(() => { + logger.info("=============================================="); + logger.info("Your Authorization token has been generated"); + logger.info("since you enabled 'auth_token' authorization"); + logger.info("for the first time. Please copy it and keep safe."); + logger.info("You will need to use it while requesting NSM."); + logger.info(""); + logger.info("Token: " + token); + logger.info("=============================================="); + }, 500); + } + if (process.env.NSM_TOKEN) { + logger.info("Authorization token loaded from env"); + } +} diff --git a/src/server.ts b/src/server.ts index 714751a..49d7de1 100644 --- a/src/server.ts +++ b/src/server.ts @@ -6,8 +6,8 @@ import temp from "temp"; // Pre // toJSON() for BigInt to avoid JSON.stringify() errors (BigInt.prototype as any).toJSON = function () { - return this.toString(); -} + return this.toString(); +}; const server = ws(express()).app; @@ -21,7 +21,7 @@ server.use(cors()); temp.track(); export function setStatus(status_: string) { - status = status_; + status = status_; } -export default server; \ No newline at end of file +export default server; diff --git a/src/util/clock.ts b/src/util/clock.ts index c25f247..877fc88 100644 --- a/src/util/clock.ts +++ b/src/util/clock.ts @@ -1,10 +1,10 @@ export function clock() { - const creationDate = Date.now(); + const creationDate = Date.now(); - function durFromCreation() { - return Date.now() - creationDate; - } - return { - durFromCreation - }; -} \ No newline at end of file + function durFromCreation() { + return Date.now() - creationDate; + } + return { + durFromCreation, + }; +} diff --git a/src/util/docker.ts b/src/util/docker.ts index 4256748..b9f0b32 100644 --- a/src/util/docker.ts +++ b/src/util/docker.ts @@ -1,32 +1,38 @@ -import DockerClient, {ContainerStats} from "dockerode"; +import DockerClient, { ContainerStats } from "dockerode"; -function calcCpuUsage(precpu: DockerClient.CPUStats, cpu: DockerClient.CPUStats) { - const cpu_delta = cpu?.cpu_usage.total_usage - precpu?.cpu_usage.total_usage; - const system_cpu_delta = cpu?.system_cpu_usage - precpu?.system_cpu_usage; - const number_cpus = cpu?.online_cpus; - const result = (cpu_delta / system_cpu_delta) * number_cpus * 100.0; - if (result == null) { - return 0.0; - } else { - return result; - } +function calcCpuUsage( + precpu: DockerClient.CPUStats, + cpu: DockerClient.CPUStats, +) { + const cpu_delta = cpu?.cpu_usage.total_usage - precpu?.cpu_usage.total_usage; + const system_cpu_delta = cpu?.system_cpu_usage - precpu?.system_cpu_usage; + const number_cpus = cpu?.online_cpus; + const result = (cpu_delta / system_cpu_delta) * number_cpus * 100.0; + if (result == null) { + return 0.0; + } else { + return result; + } } -export function adaptContainerStatsFromDocker(id: string, stats: ContainerStats) { - const { memory_stats, precpu_stats, cpu_stats } = stats; +export function adaptContainerStatsFromDocker( + id: string, + stats: ContainerStats, +) { + const { memory_stats, precpu_stats, cpu_stats } = stats; - return { - id, - memory: { - used: memory_stats?.usage, - total: memory_stats?.limit, - percent: memory_stats?.usage / memory_stats?.limit, - }, - cpu: { - used: cpu_stats?.cpu_usage.total_usage, - total: cpu_stats?.system_cpu_usage, - //percent: cpu_stats.cpu_usage.total_usage / cpu_stats.system_cpu_usage, - percent: calcCpuUsage(precpu_stats, cpu_stats), - }, - } -} \ No newline at end of file + return { + id, + memory: { + used: memory_stats?.usage, + total: memory_stats?.limit, + percent: memory_stats?.usage / memory_stats?.limit, + }, + cpu: { + used: cpu_stats?.cpu_usage.total_usage, + total: cpu_stats?.system_cpu_usage, + //percent: cpu_stats.cpu_usage.total_usage / cpu_stats.system_cpu_usage, + percent: calcCpuUsage(precpu_stats, cpu_stats), + }, + }; +} diff --git a/src/util/env.ts b/src/util/env.ts index 59e8075..7d5b049 100644 --- a/src/util/env.ts +++ b/src/util/env.ts @@ -1,12 +1,14 @@ export function env(env: string[], options?: { required?: boolean }) { - const values = env.map(k => k in process.env ? process.env[k] : undefined); - const missing = values - .map((v, i) => [v, i]) - .filter(([v]) => !v) - .map(([_, i]) => env[i]); - const required = options?.required ?? true; - if (missing.length > 0 && required == true) { - throw new Error('Missing env variables: ' + missing.join(', ')); - } - return values; -} \ No newline at end of file + const values = env.map((k) => + k in process.env ? process.env[k] : undefined, + ); + const missing = values + .map((v, i) => [v, i]) + .filter(([v]) => !v) + .map(([_, i]) => env[i]); + const required = options?.required ?? true; + if (missing.length > 0 && required == true) { + throw new Error("Missing env variables: " + missing.join(", ")); + } + return values; +} diff --git a/src/util/port.ts b/src/util/port.ts index 94a0063..fbe04e1 100644 --- a/src/util/port.ts +++ b/src/util/port.ts @@ -1,40 +1,48 @@ -import net from 'net'; -import {ServiceEngine} from "@nsm/engine"; +import net from "net"; +import { ServiceEngine } from "@nsm/engine"; -export async function isPortAvailable(engine: ServiceEngine, port: number, a_ports: number[] = undefined) { - if (a_ports === undefined) { - a_ports = await engine.listAttachedPorts(); - } - if (a_ports.includes(port)) { - return false; - } - const server = net.createServer(); - return new Promise(resolve => { - server.once('error', () => { - resolve(false); - }); - server.once('listening', () => { - server.close(); - resolve(true); - }); - server.listen(port); +export async function isPortAvailable( + engine: ServiceEngine, + port: number, + a_ports: number[] = undefined, +) { + if (a_ports === undefined) { + a_ports = await engine.listAttachedPorts(); + } + if (a_ports.includes(port)) { + return false; + } + const server = net.createServer(); + return new Promise((resolve) => { + server.once("error", () => { + resolve(false); + }); + server.once("listening", () => { + server.close(); + resolve(true); }); + server.listen(port); + }); } -export async function randomPort(engine: ServiceEngine, from: number, to: number) { - const checked = []; - const all = await engine.listAttachedPorts(); - while (true) { - const port = Math.floor(Math.random() * (to - from) + from); - if (checked.includes(port)) { - continue; - } - if (await isPortAvailable(engine, port, all)) { - return port; - } - if (checked.length === to - from) { - throw new Error('No available ports'); - } - checked.push(port); +export async function randomPort( + engine: ServiceEngine, + from: number, + to: number, +) { + const checked = []; + const all = await engine.listAttachedPorts(); + while (true) { + const port = Math.floor(Math.random() * (to - from) + from); + if (checked.includes(port)) { + continue; } -} \ No newline at end of file + if (await isPortAvailable(engine, port, all)) { + return port; + } + if (checked.length === to - from) { + throw new Error("No available ports"); + } + checked.push(port); + } +} diff --git a/src/util/promises.ts b/src/util/promises.ts index f59b74a..4b75219 100644 --- a/src/util/promises.ts +++ b/src/util/promises.ts @@ -1,9 +1,9 @@ export async function resolveSequentially(...funcs: any[]) { - for (const func of funcs) { - if (typeof func == "function") { - await (func()); - } else { - await (func as Promise); - } + for (const func of funcs) { + if (typeof func == "function") { + await func(); + } else { + await (func as Promise); } -} \ No newline at end of file + } +} diff --git a/src/util/routes.ts b/src/util/routes.ts index 8d94f88..8e33693 100644 --- a/src/util/routes.ts +++ b/src/util/routes.ts @@ -1,17 +1,17 @@ export function handleErr(e: any, res: any) { - if (e.code) { - switch (e.code) { - case 2: - res.status(409).json({status: 409, message: e.message}); - return; - case 3: - res.status(404).json({status: 404, message: e.message}); - return; - } + if (e.code) { + switch (e.code) { + case 2: + res.status(409).json({ status: 409, message: e.message }); + return; + case 3: + res.status(404).json({ status: 404, message: e.message }); + return; } - res.status(500).json({status: 500, message: e.message}); + } + res.status(500).json({ status: 500, message: e.message }); } export function handleErrorMessage(status: number, message: string, res: any) { - res.status(status).json({status, message}); -} \ No newline at end of file + res.status(status).json({ status, message }); +} diff --git a/src/util/services.ts b/src/util/services.ts index 127294e..c5a8425 100644 --- a/src/util/services.ts +++ b/src/util/services.ts @@ -1,11 +1,11 @@ export type NSMObjectLabels = { - id: string, -} + id: string; +}; // Default labels to use in docker engine objects produced by NSM export function constructObjectLabels({ id }: NSMObjectLabels) { - return { - 'nsm': 'true', - 'nsm.id': id, - } -} \ No newline at end of file + return { + nsm: "true", + "nsm.id": id, + }; +} diff --git a/src/util/yaml.ts b/src/util/yaml.ts index f735d38..3943001 100644 --- a/src/util/yaml.ts +++ b/src/util/yaml.ts @@ -2,5 +2,5 @@ import * as fs from "fs"; import YAML from "yaml"; export function loadYamlFile(path: string) { - return YAML.parse(fs.readFileSync(path, 'utf8')); -} \ No newline at end of file + return YAML.parse(fs.readFileSync(path, "utf8")); +} diff --git a/tests/api/api.test.ts b/tests/api/api.test.ts index 25abb8b..301bb1f 100644 --- a/tests/api/api.test.ts +++ b/tests/api/api.test.ts @@ -1,222 +1,231 @@ import server from "@nsm/server"; -import {init as boot, AppBootContext, AppBootOptions} from "@nsm/app"; +import { init as boot, AppBootContext, AppBootOptions } from "@nsm/app"; import request from "supertest"; -import {afterAll, beforeAll, describe, expect, test} from "@jest/globals"; -import {isServicePending} from "@nsm/engine/asyncp"; -import {log} from "console"; +import { afterAll, beforeAll, describe, expect, test } from "@jest/globals"; +import { isServicePending } from "@nsm/engine/asyncp"; +import { log } from "console"; function expectProps(obj: any, model: any[]) { - for (let i = 0; i < model.length; i += 2) { - if (model[i + 1]) { - expect(obj).toHaveProperty(model[i], model[i + 1]); - } else { - expect(obj).toHaveProperty(model[i]); - } + for (let i = 0; i < model.length; i += 2) { + if (model[i + 1]) { + expect(obj).toHaveProperty(model[i], model[i + 1]); + } else { + expect(obj).toHaveProperty(model[i]); } + } } async function miniService(ctx: AppBootContext) { - const id = await ctx.manager.createService("test", {}); - await ctx.manager.resumeService(id); - - do { - await new Promise((resolve) => { - setTimeout(() => resolve(null), 300); - }); - } while (isServicePending(id)); - // Status check - if (ctx.manager.getLastPowerError(id)) { - return undefined; - } else { - return id; - } + const id = await ctx.manager.createService("test", {}); + await ctx.manager.resumeService(id); + + do { + await new Promise((resolve) => { + setTimeout(() => resolve(null), 300); + }); + } while (isServicePending(id)); + // Status check + if (ctx.manager.getLastPowerError(id)) { + return undefined; + } else { + return id; + } } async function stopMini(ctx: AppBootContext, id: string) { - await ctx.manager.stopService(id); - await ctx.manager.waitForBusyAction(id); // Await stop + await ctx.manager.stopService(id); + await ctx.manager.waitForBusyAction(id); // Await stop } describe("Test v1 API models", () => { - let ctx: AppBootContext|undefined = undefined; - - beforeAll((done) => { - const options: AppBootOptions = { - test: true, - }; - boot(server, options).then((ctx_) => { - ctx = ctx_; - done(); - }).catch(err => { - done(err); - }); - }, 20000); - - test("Test /v1/status", async () => { - const res = await request(server).get("/v1/status"); - expect(res.status).toBe(200); - expectProps(res.body, [ - 'nodeId', undefined, - 'running', undefined, - 'all', undefined, - 'system.totalmem', undefined, - 'system.freemem', undefined, - 'system.totaldisk', undefined, - 'system.freedisk', undefined, - ]); - }); - - test("Test /v1/status to have service in running", async () => { - const id = await miniService(ctx); - const res = await request(server).get("/v1/status"); - expect(res.status).toBe(200); - expect(res.body.running).toContain(id); - }, 60000); - - test("Test /v1/status?stats=true", async () => { - const res = await request(server).get("/v1/status?stats=true"); - expect(res.status).toBe(200); - expectProps(res.body, [ - 'stats.memory.used', undefined, - 'stats.memory.total', undefined, - 'stats.memory.percent', undefined, - 'stats.cpu.used', undefined, - 'stats.cpu.total', undefined, - 'stats.cpu.percent', undefined, - 'stats.services.memTotal', undefined, - 'stats.services.cpuTotal', undefined, - 'stats.services.diskTotal', undefined, - ]); - }, 60000); - - test("Test /v1/servicelist", async () => { - const res = await request(server) - .post("/v1/servicelist") - .send({ page: 0, pageSize: 10 }); - expect(res.status).toBe(200); - expectProps(res.body, [ - 'services', undefined, - 'meta.page', 0, - 'meta.pageSize', 10, - 'meta.total', 0, - ]); - }); - - test("Test /v1/servicelist right size", async () => { - await miniService(ctx); - await miniService(ctx); - const res = await request(server) - .post("/v1/servicelist") - .send({ page: 0, pageSize: 1 }); - expect(res.status).toBe(200); - expect(res.body.services).toHaveLength(1); - }); - - test("Test /v1/service/{serviceId}", async () => { - const id = await miniService(ctx); - log(id); - const res = await request(server).get("/v1/service/" + id); - expect(res.status).toBe(200); - expectProps(res.body, [ - "id", id, - "templateId", "test", - "port", undefined, - "options", undefined, - "env", undefined, - "session.id", undefined, - "session.startedAt", undefined, - ]); - }, 20000); - - test("Test /v1/service/{serviceId}/resume", async () => { - const id = await miniService(ctx); - log(id); - await stopMini(ctx, id); - const res = await request(server) - .post("/v1/service/" + id + "/resume"); - expect(res.status).toBe(200); - expectProps(res.body, [ - "status", 200, - "message", undefined, - ]); - }, 30000); - - // TODO: /v1/service//resume - - test("Test /v1/service/{serviceId}/stop", async () => { - const id = await miniService(ctx); - log(id); - const res = await request(server) - .post("/v1/service/" + id + "/stop"); - expect(res.status).toBe(200); - expectProps(res.body, [ - "status", 200, - "message", undefined, - ]); - }, 20000); - - test("Test /v1/service/{serviceId}/delete", async () => { - const id = await miniService(ctx); - log(id); - const res = await request(server) - .post("/v1/service/" + id + "/delete"); - expect(res.status).toBe(200); - expectProps(res.body, [ - "status", 200, - "message", undefined, - ]); - }, 20000); - - test("Test /v1/service/{serviceId}/reboot", async () => { - const id = await miniService(ctx); - log(id); - const res = await request(server) - .post("/v1/service/" + id + "/reboot"); - expect(res.status).toBe(200); - expectProps(res.body, [ - "status", 200, - "message", undefined, - ]); - // Wait for it to be started - await new Promise((resolve, reject) => { - ctx.manager.on('resume', (event) => { - if (event.id == id) { - if (event.error) { - reject(event.error); - } else { - resolve(null); - } - return true; - } - }); - }); - }, 20000); - - test("Test /v1/service/{serviceId}/powerstatus", async () => { - const id = await miniService(ctx); - log(id); - const res = await request(server) - .get("/v1/service/" + id + "/powerstatus"); - expect(res.status).toBe(200); - expectProps(res.body, [ - "id", id, - "status", "IDLE", - ]); - }, 20000); - - afterAll(() => { - if (!ctx) { - return; + let ctx: AppBootContext | undefined = undefined; + + beforeAll((done) => { + const options: AppBootOptions = { + test: true, + }; + boot(server, options) + .then((ctx_) => { + ctx = ctx_; + done(); + }) + .catch((err) => { + done(err); + }); + }, 20000); + + test("Test /v1/status", async () => { + const res = await request(server).get("/v1/status"); + expect(res.status).toBe(200); + expectProps(res.body, [ + "nodeId", + undefined, + "running", + undefined, + "all", + undefined, + "system.totalmem", + undefined, + "system.freemem", + undefined, + "system.totaldisk", + undefined, + "system.freedisk", + undefined, + ]); + }); + + test("Test /v1/status to have service in running", async () => { + const id = await miniService(ctx); + const res = await request(server).get("/v1/status"); + expect(res.status).toBe(200); + expect(res.body.running).toContain(id); + }, 60000); + + test("Test /v1/status?stats=true", async () => { + const res = await request(server).get("/v1/status?stats=true"); + expect(res.status).toBe(200); + expectProps(res.body, [ + "stats.memory.used", + undefined, + "stats.memory.total", + undefined, + "stats.memory.percent", + undefined, + "stats.cpu.used", + undefined, + "stats.cpu.total", + undefined, + "stats.cpu.percent", + undefined, + "stats.services.memTotal", + undefined, + "stats.services.cpuTotal", + undefined, + "stats.services.diskTotal", + undefined, + ]); + }, 60000); + + test("Test /v1/servicelist", async () => { + const res = await request(server) + .post("/v1/servicelist") + .send({ page: 0, pageSize: 10 }); + expect(res.status).toBe(200); + expectProps(res.body, [ + "services", + undefined, + "meta.page", + 0, + "meta.pageSize", + 10, + "meta.total", + 0, + ]); + }); + + test("Test /v1/servicelist right size", async () => { + await miniService(ctx); + await miniService(ctx); + const res = await request(server) + .post("/v1/servicelist") + .send({ page: 0, pageSize: 1 }); + expect(res.status).toBe(200); + expect(res.body.services).toHaveLength(1); + }); + + test("Test /v1/service/{serviceId}", async () => { + const id = await miniService(ctx); + log(id); + const res = await request(server).get("/v1/service/" + id); + expect(res.status).toBe(200); + expectProps(res.body, [ + "id", + id, + "templateId", + "test", + "port", + undefined, + "options", + undefined, + "env", + undefined, + "session.id", + undefined, + "session.startedAt", + undefined, + ]); + }, 20000); + + test("Test /v1/service/{serviceId}/resume", async () => { + const id = await miniService(ctx); + log(id); + await stopMini(ctx, id); + const res = await request(server).post("/v1/service/" + id + "/resume"); + expect(res.status).toBe(200); + expectProps(res.body, ["status", 200, "message", undefined]); + }, 30000); + + // TODO: /v1/service//resume + + test("Test /v1/service/{serviceId}/stop", async () => { + const id = await miniService(ctx); + log(id); + const res = await request(server).post("/v1/service/" + id + "/stop"); + expect(res.status).toBe(200); + expectProps(res.body, ["status", 200, "message", undefined]); + }, 20000); + + test("Test /v1/service/{serviceId}/delete", async () => { + const id = await miniService(ctx); + log(id); + const res = await request(server).post("/v1/service/" + id + "/delete"); + expect(res.status).toBe(200); + expectProps(res.body, ["status", 200, "message", undefined]); + }, 20000); + + test("Test /v1/service/{serviceId}/reboot", async () => { + const id = await miniService(ctx); + log(id); + const res = await request(server).post("/v1/service/" + id + "/reboot"); + expect(res.status).toBe(200); + expectProps(res.body, ["status", 200, "message", undefined]); + // Wait for it to be started + await new Promise((resolve, reject) => { + ctx.manager.on("resume", (event) => { + if (event.id == id) { + if (event.error) { + reject(event.error); + } else { + resolve(null); + } + return true; } + }); + }); + }, 20000); + + test("Test /v1/service/{serviceId}/powerstatus", async () => { + const id = await miniService(ctx); + log(id); + const res = await request(server).get("/v1/service/" + id + "/powerstatus"); + expect(res.status).toBe(200); + expectProps(res.body, ["id", id, "status", "IDLE"]); + }, 20000); + + afterAll(() => { + if (!ctx) { + return; + } - return ctx.manager.stopRunning(); - }, 60000); + return ctx.manager.stopRunning(); + }, 60000); - // TODO: /v1/service//options - // TODO: /v1/service//stopcmd - // TODO: /v1/service//stop?force=true + // TODO: /v1/service//options + // TODO: /v1/service//stopcmd + // TODO: /v1/service//stop?force=true - // TODO: Add missing API tests + // TODO: Add missing API tests }); -// TODO: Test v1 API in-depth \ No newline at end of file +// TODO: Test v1 API in-depth diff --git a/tests/database/manager.test.ts b/tests/database/manager.test.ts index 881edd4..2c3ccd7 100644 --- a/tests/database/manager.test.ts +++ b/tests/database/manager.test.ts @@ -1,22 +1,21 @@ -import {afterEach, beforeEach, expect, it} from "@jest/globals"; -import {StartedMariaDbContainer} from "@testcontainers/mariadb"; -import getDb, {Database} from "@nsm/database"; -import {PrismaClient} from "@prisma/client"; -import {initDbContainerForTest} from "../testUtils"; +import { afterEach, beforeEach, expect, it } from "@jest/globals"; +import { StartedMariaDbContainer } from "@testcontainers/mariadb"; +import getDb, { Database } from "@nsm/database"; +import { PrismaClient } from "@prisma/client"; +import { initDbContainerForTest } from "../testUtils"; let container: StartedMariaDbContainer; let db: Database; beforeEach(async () => { - const [ - container_, - dbUrl_, - ] = await initDbContainerForTest(); + const [container_, dbUrl_] = await initDbContainerForTest(); container = container_; - db = getDb(new PrismaClient({ - datasourceUrl: dbUrl_, - })); + db = getDb( + new PrismaClient({ + datasourceUrl: dbUrl_, + }), + ); }, 20000); afterEach(async () => { @@ -59,24 +58,33 @@ it("finds image by options", async () => { }, }); - let imagesByOptions = await db.imageRepository.listImagesByOptions("test-template", { - option1: "value1", - option2: "value2", - }); + let imagesByOptions = await db.imageRepository.listImagesByOptions( + "test-template", + { + option1: "value1", + option2: "value2", + }, + ); expect(imagesByOptions).toHaveLength(1); expect(imagesByOptions[0]?.id).toBe("test-image"); - imagesByOptions = await db.imageRepository.listImagesByOptions("test-template", { - option2: "value2", - option1: "value1", - }); + imagesByOptions = await db.imageRepository.listImagesByOptions( + "test-template", + { + option2: "value2", + option1: "value1", + }, + ); expect(imagesByOptions).toHaveLength(1); expect(imagesByOptions[0]?.id).toBe("test-image"); - imagesByOptions = await db.imageRepository.listImagesByOptions("test-template", { - option1: "value1", - option2: "value2", - option3: "value3", - }); + imagesByOptions = await db.imageRepository.listImagesByOptions( + "test-template", + { + option1: "value1", + option2: "value2", + option3: "value3", + }, + ); expect(imagesByOptions).toHaveLength(0); -}); \ No newline at end of file +}); diff --git a/tests/engine/image.test.ts b/tests/engine/image.test.ts index 9a2bc0d..b4eb40b 100644 --- a/tests/engine/image.test.ts +++ b/tests/engine/image.test.ts @@ -1,19 +1,19 @@ -import {afterAll, beforeAll, expect, it} from "@jest/globals"; -import {ServiceEngineI} from "@nsm/engine"; +import { afterAll, beforeAll, expect, it } from "@jest/globals"; +import { ServiceEngineI } from "@nsm/engine"; import createEngine from "@nsm/engine/engine"; -import {init as initImageEngine} from "@nsm/engine/image"; +import { init as initImageEngine } from "@nsm/engine/image"; import getDb from "@nsm/database"; -import {Database} from "@nsm/database"; -import {StartedMariaDbContainer} from "@testcontainers/mariadb"; -import {initDbContainerForTest} from "../testUtils"; -import {PrismaClient} from "@prisma/client"; -import {processImage} from "@nsm/engine/image"; -import {createLogger} from "@nsm/logger"; -import {Template, TemplateManager} from "@nsm/engine/template"; -import {TemplateDirWatcher} from "@nsm/engine/monitoring/templateDirWatcher"; +import { Database } from "@nsm/database"; +import { StartedMariaDbContainer } from "@testcontainers/mariadb"; +import { initDbContainerForTest } from "../testUtils"; +import { PrismaClient } from "@prisma/client"; +import { processImage } from "@nsm/engine/image"; +import { createLogger } from "@nsm/logger"; +import { Template, TemplateManager } from "@nsm/engine/template"; +import { TemplateDirWatcher } from "@nsm/engine/monitoring/templateDirWatcher"; import * as templateManager from "@nsm/engine/template"; import * as templateDirWatcher from "@nsm/engine/monitoring/templateDirWatcher"; -import {YamlAppConfig} from "@nsm/config"; +import { YamlAppConfig } from "@nsm/config"; let container: StartedMariaDbContainer; @@ -21,19 +21,18 @@ let engine: ServiceEngineI; let db: Database; beforeAll(async () => { - const [ - container_, - dbUrl_ - ] = await initDbContainerForTest(); + const [container_, dbUrl_] = await initDbContainerForTest(); container = container_; engine = createEngine( // Just to prevent assertion errors - new YamlAppConfig() + new YamlAppConfig(), + ); + db = getDb( + new PrismaClient({ + datasourceUrl: dbUrl_, + }), ); - db = getDb(new PrismaClient({ - datasourceUrl: dbUrl_, - })); }, 20000); afterAll(async () => { @@ -51,22 +50,28 @@ it("reuses image with same options", async () => { env: { option1: "", option2: "", - } - } - } + }, + }, + }; let buildCount = 0; const customEngine: ServiceEngineI = { ...engine, - build(imageId: string | undefined, _: string | undefined, __: { - [p: string]: string - }): Promise { + build( + imageId: string | undefined, + _: string | undefined, + __: { + [p: string]: string; + }, + ): Promise { buildCount++; - return Promise.resolve(imageId ?? "generated-image-id-" + (Math.random() * 1000000).toFixed(0)); - } - } + return Promise.resolve( + imageId ?? "generated-image-id-" + (Math.random() * 1000000).toFixed(0), + ); + }, + }; const customTemplateManager: TemplateManager = { ...templateManager, getTemplate(id: string): Template | null { @@ -75,7 +80,7 @@ it("reuses image with same options", async () => { } return null; - } + }, }; const customTemplateDirWatcher: TemplateDirWatcher = { ...templateDirWatcher, @@ -85,10 +90,16 @@ it("reuses image with same options", async () => { } throw new Error(`Unknown template ${template}`); - } + }, }; - initImageEngine(customEngine, customTemplateManager, customTemplateDirWatcher, db, createLogger()); + initImageEngine( + customEngine, + customTemplateManager, + customTemplateDirWatcher, + db, + createLogger(), + ); const buildOptions = { option1: "value1", @@ -102,4 +113,4 @@ it("reuses image with same options", async () => { expect(imageId2).toEqual(imageId); expect(buildCount).toBe(1); -}); \ No newline at end of file +}); diff --git a/tests/engine/middle.test.ts b/tests/engine/middle.test.ts index d9627fc..696ada9 100644 --- a/tests/engine/middle.test.ts +++ b/tests/engine/middle.test.ts @@ -1,7 +1,11 @@ -import {expect, it} from "@jest/globals"; -import {middleLayer, registerErrorPublisher, ServiceActionError} from "@nsm/engine/middle"; +import { expect, it } from "@jest/globals"; +import { + middleLayer, + registerErrorPublisher, + ServiceActionError, +} from "@nsm/engine/middle"; import * as manager from "@nsm/engine/manager"; -import {Options, ServiceManager} from "@nsm/engine/manager"; +import { Options, ServiceManager } from "@nsm/engine/manager"; it("test receives action error", async () => { let receivedError: ServiceActionError | null = null; @@ -10,14 +14,14 @@ it("test receives action error", async () => { receivedError = action; return Promise.resolve(); - } + }, }); let customManager: ServiceManager = { ...manager, async createService(_: string, __: Options) { throw new Error("Failed to create service"); - } + }, }; customManager = middleLayer(customManager); @@ -43,14 +47,14 @@ it("test sets service id in action error", async () => { receivedError = action; return Promise.resolve(); - } + }, }); let customManager: ServiceManager = { ...manager, async resumeService(_: string) { throw new Error("Failed to resume service"); - } + }, }; customManager = middleLayer(customManager); @@ -67,4 +71,4 @@ it("test sets service id in action error", async () => { expect(receivedError?.serviceId).toEqual("test-service-id"); expect(receivedError?.type).toEqual("resume"); expect(receivedError?.message).toEqual("Failed to resume service"); -}); \ No newline at end of file +}); diff --git a/tests/testUtils.ts b/tests/testUtils.ts index 391926c..b40cc9a 100644 --- a/tests/testUtils.ts +++ b/tests/testUtils.ts @@ -1,17 +1,21 @@ -import {MariaDbContainer, StartedMariaDbContainer} from "@testcontainers/mariadb"; -import {execSync} from "child_process"; +import { + MariaDbContainer, + StartedMariaDbContainer, +} from "@testcontainers/mariadb"; +import { execSync } from "child_process"; -export const initDbContainerForTest = async (): Promise<[StartedMariaDbContainer, string]> => { +export const initDbContainerForTest = async (): Promise< + [StartedMariaDbContainer, string] +> => { const container = await new MariaDbContainer("mariadb:10.4") .withRootPassword("test") .withDatabase("nsm") .start(); - const dbUrl = container.getConnectionUri() - .replace("mariadb://", "mysql://"); + const dbUrl = container.getConnectionUri().replace("mariadb://", "mysql://"); execSync("npx prisma migrate deploy", { stdio: "inherit", env: { ...process.env, DATABASE_URL: dbUrl }, }); return [container, dbUrl]; -} \ No newline at end of file +}; diff --git a/tsconfig.json b/tsconfig.json index 2d3d05a..133c53c 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -1,16 +1,16 @@ { - "compilerOptions": { - "module": "commonjs", - "esModuleInterop": true, - "target": "es6", - "moduleResolution": "node", - "sourceMap": true, - "rootDir": "./", - "outDir": "dist", - "baseUrl": ".", - "paths": { - "@nsm/*": ["src/*"] - } - }, - "lib": ["es2015"] -} \ No newline at end of file + "compilerOptions": { + "module": "commonjs", + "esModuleInterop": true, + "target": "es6", + "moduleResolution": "node", + "sourceMap": true, + "rootDir": "./", + "outDir": "dist", + "baseUrl": ".", + "paths": { + "@nsm/*": ["src/*"] + } + }, + "lib": ["es2015"] +} From 701fe96da65ac0a6e6541c9a6b85fb3c526da238 Mon Sep 17 00:00:00 2001 From: ZorTik Date: Thu, 11 Jun 2026 01:26:06 +0200 Subject: [PATCH 04/53] refactor(compose): dev environment --- .gitignore | 1 - .../example/example_nsmignore => dev/templates/test/.nsmignore | 0 .../test/test_dockerfile => dev/templates/test/Dockerfile | 0 .../test/test_settings.yml => dev/templates/test/settings.yml | 0 docker-compose.yml | 2 ++ resources/{template => templates}/example/example_dockerfile | 0 .../test/test_nsmignore => templates/example/example_nsmignore} | 0 resources/{template => templates}/example/example_settings.yml | 0 8 files changed, 2 insertions(+), 1 deletion(-) rename resources/template/example/example_nsmignore => dev/templates/test/.nsmignore (100%) rename resources/template/test/test_dockerfile => dev/templates/test/Dockerfile (100%) rename resources/template/test/test_settings.yml => dev/templates/test/settings.yml (100%) rename resources/{template => templates}/example/example_dockerfile (100%) rename resources/{template/test/test_nsmignore => templates/example/example_nsmignore} (100%) rename resources/{template => templates}/example/example_settings.yml (100%) diff --git a/.gitignore b/.gitignore index c45b6ef..36b7834 100644 --- a/.gitignore +++ b/.gitignore @@ -131,7 +131,6 @@ dist .pnp.* # Other -templates volumes # Exclude all addons except the example one diff --git a/resources/template/example/example_nsmignore b/dev/templates/test/.nsmignore similarity index 100% rename from resources/template/example/example_nsmignore rename to dev/templates/test/.nsmignore diff --git a/resources/template/test/test_dockerfile b/dev/templates/test/Dockerfile similarity index 100% rename from resources/template/test/test_dockerfile rename to dev/templates/test/Dockerfile diff --git a/resources/template/test/test_settings.yml b/dev/templates/test/settings.yml similarity index 100% rename from resources/template/test/test_settings.yml rename to dev/templates/test/settings.yml diff --git a/docker-compose.yml b/docker-compose.yml index fe72ad5..55ffce6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,12 +3,14 @@ services: build: . volumes: - "/var/run/docker.sock:/var/run/docker.sock" + - "./dev/templates/test:/data/resources/templates/test:ro" ports: - "3000:3000" extra_hosts: - "docker.host.internal:host-gateway" environment: - "CONFIG_DOCKER_HOST=///var/run/docker.sock" + - "CONFIG_RESOURCES_PATH=/data/resources" - "DATABASE_URL=mysql://root:test@db:3306/nsm" depends_on: db: diff --git a/resources/template/example/example_dockerfile b/resources/templates/example/example_dockerfile similarity index 100% rename from resources/template/example/example_dockerfile rename to resources/templates/example/example_dockerfile diff --git a/resources/template/test/test_nsmignore b/resources/templates/example/example_nsmignore similarity index 100% rename from resources/template/test/test_nsmignore rename to resources/templates/example/example_nsmignore diff --git a/resources/template/example/example_settings.yml b/resources/templates/example/example_settings.yml similarity index 100% rename from resources/template/example/example_settings.yml rename to resources/templates/example/example_settings.yml From 30ec614ccbc40acfaf0d3e9f8bd126ae7aeb9310 Mon Sep 17 00:00:00 2001 From: ZorTik Date: Thu, 11 Jun 2026 01:49:25 +0200 Subject: [PATCH 05/53] refactor: stop flow (use stop cmd if present) --- src/engine/manager.ts | 47 +++++++++++++++++------------- src/router/v1/service/stopRoute.ts | 3 +- 2 files changed, 29 insertions(+), 21 deletions(-) diff --git a/src/engine/manager.ts b/src/engine/manager.ts index 188c2c7..1480ba1 100644 --- a/src/engine/manager.ts +++ b/src/engine/manager.ts @@ -636,29 +636,36 @@ export async function stopService(id: string, force?: boolean) { await reqExists(id); const { internalSession } = reqRunning(id); - - lckStatusTp(internalSession.containerId, "stop"); - const unlock = lockBusyAction(id, "stop"); - try { - on("stop", ({ id: stoppedId, error }) => { - if (stoppedId !== id) { - // This call is not for me - return false; - } - - if (isServicePending(id)) { - unlock(error); - } - ulckStatusTp(internalSession.containerId); - return true; - }); - - const meta = metaStorageForService(id); if (force) { - await engine.kill(internalSession.containerId, meta); + await engine.kill(internalSession.containerId, metaStorageForService(id)); } else { - await engine.stop(internalSession.containerId); + // lock only on soft stop, to allow hard-killing if any issues happen during stopping + const unlock = lockBusyAction(id, "stop"); + // wait for stop + on("stop", ({ id: stoppedId, error }) => { + if (stoppedId !== id) { + // This call is not for me + return false; + } + + if (isServicePending(id)) { + unlock(error); + } + ulckStatusTp(internalSession.containerId); + return true; + }); + + // TODO: stop strategy + const service = await getService(id); + const stopCmd = service.meta?.stopCmd; + if (stopCmd) { + // send stop cmd if set + await engine.cmd(internalSession.containerId, stopCmd); + } else { + // send stop signal + await engine.stop(internalSession.containerId); + } } } catch (e) { currentContext.logger.error(e); diff --git a/src/router/v1/service/stopRoute.ts b/src/router/v1/service/stopRoute.ts index 488127f..34295a0 100644 --- a/src/router/v1/service/stopRoute.ts +++ b/src/router/v1/service/stopRoute.ts @@ -15,6 +15,7 @@ export default async function ({ routes: { post: async (req, res) => { const id = req.params.id; + const isForce = req.query.force === "true"; if (!id) { res .status(400) @@ -27,7 +28,7 @@ export default async function ({ if (!(await checkServiceExists(id, manager, res))) { return; } - if (!checkServicePending(id, res)) { + if (!isForce && !checkServicePending(id, res)) { return; } if (!manager.isRunning(id)) { From 655fc2e4690556f23c3529a7159117cebc454de9 Mon Sep 17 00:00:00 2001 From: ZorTik Date: Thu, 11 Jun 2026 03:37:34 +0200 Subject: [PATCH 06/53] fix: test env in gh actions --- .github/workflows/jest.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/jest.yml b/.github/workflows/jest.yml index e068de1..71437e1 100644 --- a/.github/workflows/jest.yml +++ b/.github/workflows/jest.yml @@ -15,6 +15,7 @@ jobs: env: DATABASE_URL: "mysql://test:test@localhost:3306/test" CONFIG_DOCKER_HOST: "///var/run/docker.sock" + CONFIG_RESOURCES_PATH: "/resources" DEBUG: "true" steps: - name: Checkout @@ -26,6 +27,12 @@ jobs: echo "This PR is from the dev branch. Exiting..." exit 0 fi + - name: Prepare resources folder for tests + run: | + # Create test template folder + mkdir -p /resources/templates/test + # Copy test template files to the resources folder + cp -r ./dev/templates/test/* /resources/templates/test/ - name: Shutdown default MySQL run: sudo service mysql stop - name: Setup MySQL From 2f85ae9c34348965b65a7c098016c9a20bc86a8e Mon Sep 17 00:00:00 2001 From: ZorTik Date: Thu, 11 Jun 2026 03:39:34 +0200 Subject: [PATCH 07/53] fix: test env in gh actions --- .github/workflows/jest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/jest.yml b/.github/workflows/jest.yml index 71437e1..ff0977f 100644 --- a/.github/workflows/jest.yml +++ b/.github/workflows/jest.yml @@ -30,7 +30,7 @@ jobs: - name: Prepare resources folder for tests run: | # Create test template folder - mkdir -p /resources/templates/test + mkdir -p ./resources/templates/test # Copy test template files to the resources folder cp -r ./dev/templates/test/* /resources/templates/test/ - name: Shutdown default MySQL From 77374bf4cee1636b0a02dbec5955e915b154eeb5 Mon Sep 17 00:00:00 2001 From: ZorTik Date: Thu, 11 Jun 2026 03:44:16 +0200 Subject: [PATCH 08/53] fix: test env in gh actions --- .github/workflows/jest.yml | 4 ++-- src/filestructure.ts | 7 ++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/.github/workflows/jest.yml b/.github/workflows/jest.yml index ff0977f..c270c15 100644 --- a/.github/workflows/jest.yml +++ b/.github/workflows/jest.yml @@ -15,7 +15,7 @@ jobs: env: DATABASE_URL: "mysql://test:test@localhost:3306/test" CONFIG_DOCKER_HOST: "///var/run/docker.sock" - CONFIG_RESOURCES_PATH: "/resources" + CONFIG_RESOURCES_PATH: "./resources" DEBUG: "true" steps: - name: Checkout @@ -32,7 +32,7 @@ jobs: # Create test template folder mkdir -p ./resources/templates/test # Copy test template files to the resources folder - cp -r ./dev/templates/test/* /resources/templates/test/ + cp -r ./dev/templates/test/* ./resources/templates/test/ - name: Shutdown default MySQL run: sudo service mysql stop - name: Setup MySQL diff --git a/src/filestructure.ts b/src/filestructure.ts index 214bd70..2ae1213 100644 --- a/src/filestructure.ts +++ b/src/filestructure.ts @@ -16,7 +16,12 @@ export const resourcesPath = path.join(process.cwd(), "resources"); // The target (platform-agnostic) resources dir (the source of truth) export const getResourcesTargetPath = () => { - return appConfig.getResourcesPath() ?? path.join(currentPaths.data); + const result = appConfig.getResourcesPath(); + if (result) { + return path.resolve(result); + } else { + return path.join(currentPaths.data); + } }; export const getTemplatesPath = () => { From 36311cd5fe3edc36fe131c0c265a142c9d953f07 Mon Sep 17 00:00:00 2001 From: ZorTik Date: Thu, 11 Jun 2026 13:42:07 +0200 Subject: [PATCH 09/53] feat: Makefile, fix tests --- docker-compose.yml | 1 + jest.config.js | 16 ++++++++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 55ffce6..3a43b62 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,6 +4,7 @@ services: volumes: - "/var/run/docker.sock:/var/run/docker.sock" - "./dev/templates/test:/data/resources/templates/test:ro" + - "./tests:/data/tests:ro" ports: - "3000:3000" extra_hosts: diff --git a/jest.config.js b/jest.config.js index 14ef211..7e1d746 100644 --- a/jest.config.js +++ b/jest.config.js @@ -3,6 +3,18 @@ const moduleNameMapper = require("tsconfig-paths-jest")(tsconfig); module.exports = { moduleNameMapper, - transformIgnorePatterns: ["/node_modules/(?!(env-paths)/)"], - reporters: ["default", ["jest-ctrf-json-reporter", {}]], + testPathIgnorePatterns: [ + "/node_modules/", + "/dist/" + ], + modulePathIgnorePatterns: [ + "/dist/" + ], + transformIgnorePatterns: [ + "/node_modules/(?!(env-paths)/)" + ], + reporters: [ + "default", + ["jest-ctrf-json-reporter", {}] + ], }; From 7b863e156c5348204d5760db35c124167c16f75b Mon Sep 17 00:00:00 2001 From: ZorTik Date: Thu, 11 Jun 2026 15:15:37 +0200 Subject: [PATCH 10/53] feat: Makefile --- Makefile | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..a53bd75 --- /dev/null +++ b/Makefile @@ -0,0 +1,27 @@ +.PHONY: build test up down restart logs shell ps + +all: build + +build: + docker compose build + +test: + docker compose run --rm nsm npm test + +up: + docker compose up -d + +down: + docker compose down + +restart: + docker compose restart + +logs: + docker compose logs -f + +shell: + docker compose exec nsm sh + +ps: + docker compose ps From 0ca37803150b700efc39093165f5c11da6233af2 Mon Sep 17 00:00:00 2001 From: ZorTik Date: Thu, 11 Jun 2026 16:21:49 +0200 Subject: [PATCH 11/53] feat: dind for dev environment --- docker-compose.yml | 24 +++++++++++++++++++++--- package.json | 5 +++-- src/app.ts | 1 + src/engine/docker/client.ts | 3 +++ src/logger.ts | 6 ++++++ 5 files changed, 34 insertions(+), 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 3a43b62..11131ef 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,6 @@ services: nsm: build: . volumes: - - "/var/run/docker.sock:/var/run/docker.sock" - "./dev/templates/test:/data/resources/templates/test:ro" - "./tests:/data/tests:ro" ports: @@ -10,12 +9,15 @@ services: extra_hosts: - "docker.host.internal:host-gateway" environment: - - "CONFIG_DOCKER_HOST=///var/run/docker.sock" + - "CONFIG_DOCKER_HOST=http://docker:2375" - "CONFIG_RESOURCES_PATH=/data/resources" - "DATABASE_URL=mysql://root:test@db:3306/nsm" + - "DOCKER_HOST=tcp://docker:2375" depends_on: db: condition: service_healthy + docker: + condition: service_healthy healthcheck: test: ["CMD-SHELL", "curl -f http://localhost:3000/ || exit 1"] interval: 5s @@ -31,7 +33,22 @@ services: - ./logs - ./node_packages - ./package-lock.json - + docker: + image: docker:29.1.3-dind + privileged: true + volumes: + - docker_data:/var/lib/docker + environment: + # in dev environment, we don't need TLS for Dind. + DOCKER_TLS_CERTDIR: "" + ports: + - "2377:2375" + healthcheck: + test: ["CMD", "docker", "info"] + interval: 10s + timeout: 5s + retries: 5 + start_period: 30s db: image: mariadb:10.4 environment: @@ -50,3 +67,4 @@ services: volumes: nsm_db: + docker_data: \ No newline at end of file diff --git a/package.json b/package.json index 14a24ad..eb9aa51 100644 --- a/package.json +++ b/package.json @@ -5,8 +5,9 @@ "main": "index.js", "scripts": { "build": "node installTempDeps.js && tsc && tscp", - "start": "cross-env TS_NODE_BASEURL=./dist node -r tsconfig-paths/register dist/index.js", - "test": "jest" + "migrate": "prisma migrate deploy", + "start": "npm run migrate && cross-env TS_NODE_BASEURL=./dist node -r tsconfig-paths/register dist/index.js", + "test": "npm run migrate && jest" }, "keywords": [], "author": "ZorTik", diff --git a/src/app.ts b/src/app.ts index cbc4b90..d7408dc 100644 --- a/src/app.ts +++ b/src/app.ts @@ -99,6 +99,7 @@ export const init = async ( ): Promise => { // Prepare logging const logger = initGlobalLogger(); + logging.setCurrentGlobalLogger(logger); prepareFolders(); diff --git a/src/engine/docker/client.ts b/src/engine/docker/client.ts index 62cf004..96e2ce0 100644 --- a/src/engine/docker/client.ts +++ b/src/engine/docker/client.ts @@ -1,5 +1,6 @@ import DockerClient from "dockerode"; import { AppConfig } from "@nsm/config"; +import {currentGlobalLogger} from "@nsm/logger"; export function initDockerClient(appConfig: AppConfig) { let host = appConfig.getDockerHost(); @@ -28,6 +29,8 @@ export function initDockerClient(appConfig: AppConfig) { ); } + currentGlobalLogger.info(`Initializing Docker client on ${protocol}://${host}:${port}`); + client = new DockerClient({ protocol, host, port }); } else { throw new Error( diff --git a/src/logger.ts b/src/logger.ts index 522ce15..bb83588 100644 --- a/src/logger.ts +++ b/src/logger.ts @@ -5,6 +5,12 @@ import { getResourcesTargetPath } from "@nsm/filestructure"; const { combine, timestamp, label, errors, printf } = winston.format; +export let currentGlobalLogger: winston.Logger; + +export function setCurrentGlobalLogger(logger: winston.Logger) { + currentGlobalLogger = logger; +} + export function createLatestLogFile() { if ( fs.existsSync(path.join(getResourcesTargetPath(), "logs", "latest.log")) From 483bee6205e245539e86e236012eadf5396df4e5 Mon Sep 17 00:00:00 2001 From: ZorTik Date: Thu, 11 Jun 2026 16:23:30 +0200 Subject: [PATCH 12/53] feat: delete db migrate from Dockerfile cmd --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 7cfc4b2..d3c8838 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,4 +23,4 @@ COPY index.ts ./ RUN npm run build -CMD npx prisma migrate deploy && npm run start \ No newline at end of file +CMD npm start \ No newline at end of file From 2156b18c3ef9cf6113b3ec6ec7ac327c41b0d6f9 Mon Sep 17 00:00:00 2001 From: ZorTik Date: Thu, 11 Jun 2026 20:05:24 +0200 Subject: [PATCH 13/53] fix: tests --- Makefile | 9 +- package.json | 1 + src/app.ts | 4 +- src/config.ts | 30 +++- src/engine/asyncp.ts | 41 ++--- src/engine/image.ts | 13 +- src/engine/manager.ts | 71 +++++++-- src/engine/middle.ts | 6 - src/engine/monitoring/templateDirWatcher.ts | 8 +- src/engine/monitoring/util.ts | 8 - src/filestructure.ts | 25 ++-- src/logger.ts | 10 +- src/resources.ts | 11 +- src/router/v1/service/resumeRoute.ts | 2 +- src/router/v1/service/stopRoute.ts | 9 +- tests/api/api.test.ts | 10 +- tests/engine/image.test.ts | 158 ++++++++++---------- tests/testUtils.ts | 9 ++ 18 files changed, 241 insertions(+), 184 deletions(-) diff --git a/Makefile b/Makefile index a53bd75..1234982 100644 --- a/Makefile +++ b/Makefile @@ -1,12 +1,19 @@ .PHONY: build test up down restart logs shell ps +ATTACH ?= 0 + all: build build: docker compose build test: - docker compose run --rm nsm npm test +ifeq ($(ATTACH),1) # if ATTACH=1, run tests with debugger attached + docker compose run --rm -p 9229:9229 nsm \ + sh -c 'npm run migrate && node --inspect-brk=0.0.0.0:9229 ./node_modules/.bin/jest --runInBand $(ARGS)' +else + docker compose run --rm nsm npm run test $(ARGS) +endif up: docker compose up -d diff --git a/package.json b/package.json index eb9aa51..4164d96 100644 --- a/package.json +++ b/package.json @@ -62,6 +62,7 @@ "folder-hash": "^4.1.1", "ignore": "^5.3.1", "jest": "^29.7.0", + "jest-mock-extended": "^4.0.1", "npm": "^7.24.2", "tar": "^6.2.0", "tsconfig-paths-jest": "^0.0.1", diff --git a/src/app.ts b/src/app.ts index d7408dc..acf7da0 100644 --- a/src/app.ts +++ b/src/app.ts @@ -2,7 +2,7 @@ import dotenv from "dotenv"; import { loadAppConfig } from "@nsm/config"; import { init as initFileStructure, - getResourcesTargetPath, + getResourcesPath, prepareFolders, } from "@nsm/filestructure"; @@ -159,7 +159,7 @@ export const init = async ( }; const prepareTestResources = () => { - if (fs.existsSync(path.join(getResourcesTargetPath(), "templates", "test"))) { + if (fs.existsSync(path.join(getResourcesPath(), "templates", "test"))) { return; } diff --git a/src/config.ts b/src/config.ts index e4ec708..af82545 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1,8 +1,10 @@ import { loadYamlFile } from "@nsm/util/yaml"; import path from "path"; -import { currentPaths } from "@nsm/filestructure"; import { saveResource } from "@nsm/resources"; import z from "zod"; +import envPaths, {Paths} from "env-paths"; + +export const currentPaths: Paths = envPaths("nsm"); export interface AppConfig { getNodeId(): string; @@ -13,7 +15,13 @@ export interface AppConfig { getDockerHost(): string; - getResourcesPath(): string | undefined; + getResourcesPath(): string; + + getTemplatesPath(): string; + + getTemplateBuildDir(template: string): string; + + getTempPath(): string; } /** @@ -57,8 +65,22 @@ export class YamlAppConfig implements AppConfig { return this.data["docker_host"]; } - getResourcesPath(): string | undefined { - return this.data["resources_path"]; + getResourcesPath(): string { + const resourcesPath = this.data["resources_path"]; + + return resourcesPath ? path.resolve(resourcesPath) : path.join(currentPaths.data); + } + + getTemplatesPath(): string { + return path.join(this.getResourcesPath(), "templates"); + } + + getTemplateBuildDir(template: string): string { + return path.join(this.getTemplatesPath(), template); + } + + getTempPath(): string { + return currentPaths.temp; } private validate = () => { diff --git a/src/engine/asyncp.ts b/src/engine/asyncp.ts index 181afc7..857174b 100644 --- a/src/engine/asyncp.ts +++ b/src/engine/asyncp.ts @@ -20,19 +20,34 @@ export function lockBusyAction(id: string, tp: string) { status_types[id] = tp; // type of action return (err?: any) => { - delete statuses[id]; - delete status_types[id]; - - (obs.get(id) ?? []).forEach((o) => o(id, tp, err)); - obs.delete(id); - - if (pendingCount() == 0) { - obsAll.forEach((o) => o()); - obsAll.splice(0, obsAll.length); + if (getActionType(id) !== tp) { + throw new Error( + `Unlocking action type ${tp} does not match the current action type ${getActionType(id)} for service ${id}`, + ); } + + unlockBusyAction(id, err); }; } +export function unlockBusyAction(id: string, err?: any) { + const tp = getActionType(id); + if (!tp) { + throw new Error("No busy action in process"); + } + + delete statuses[id]; + delete status_types[id]; + + (obs.get(id) ?? []).forEach((o) => o(id, tp, err)); + obs.delete(id); + + if (pendingCount() == 0) { + obsAll.forEach((o) => o()); + obsAll.splice(0, obsAll.length); + } +} + export function whenUnlocked(id: string, cb: UnlockObserver) { if (isServicePending(id)) { obs.set(id, obs.get(id) ?? []); @@ -50,14 +65,6 @@ export function whenUnlockedAll(cb: () => void) { } } -export function lckStatusTp(id: string, tp: string) { - status_types[id] = tp; -} - -export function ulckStatusTp(id: string) { - delete status_types[id]; -} - export function isServicePending(id: string): boolean { return statuses[id] || false; } diff --git a/src/engine/image.ts b/src/engine/image.ts index feb3984..4d1a712 100644 --- a/src/engine/image.ts +++ b/src/engine/image.ts @@ -1,31 +1,34 @@ import { Database, ImageModel } from "@nsm/database"; import winston from "winston"; -import { MessageListener, ServiceEngineI } from "@nsm/engine/engine"; -import { templateBuildDir } from "@nsm/engine/monitoring/util"; +import {MessageListener, ServiceEngine} from "@nsm/engine/engine"; import { TemplateManager } from "@nsm/engine/template"; import { TemplateDirWatcher } from "@nsm/engine/monitoring/templateDirWatcher"; +import {AppConfig} from "@nsm/config"; type BuildOptionsMap = { [key: string]: string; }; -let engine: ServiceEngineI; +let engine: ServiceEngine; let templateManager: TemplateManager; let templateDirWatcher: TemplateDirWatcher; +let appConfig: AppConfig; let db: Database; let logger: winston.Logger; export const init = ( - engine_: ServiceEngineI, + engine_: ServiceEngine, templateManager_: TemplateManager, templateDirWatcher_: TemplateDirWatcher, db_: Database, + appConfig_: AppConfig, logger_: winston.Logger, ) => { engine = engine_; templateManager = templateManager_; templateDirWatcher = templateDirWatcher_; db = db_; + appConfig = appConfig_; logger = logger_; }; @@ -172,7 +175,7 @@ const buildImage = async ( const hash = templateDirWatcher.getTemplateHash(templateId); imageId = await engine.build( imageId, - templateBuildDir(templateId), + appConfig.getTemplateBuildDir(templateId), options, messageListener, ); diff --git a/src/engine/manager.ts b/src/engine/manager.ts index 1480ba1..4c8a38e 100644 --- a/src/engine/manager.ts +++ b/src/engine/manager.ts @@ -19,10 +19,8 @@ import { randomPort as retrieveRandomPort } from "@nsm/util/port"; import { Database, PermaModel } from "../database"; import { isServicePending, - lckStatusTp, lockBusyAction, - reqNotPending, - ulckStatusTp, + reqNotPending, unlockBusyAction, UnlockObserver, whenUnlocked, whenUnlockedAll, @@ -125,6 +123,10 @@ type ServiceManagerEvents = { stop: ServiceEvent; }; +/** + * The event handler for service manager events. + * If the handler returns true or nothing, it will be unsubscribed after this call. + */ type EventHandler = ( event: ServiceManagerEvents[T], ) => boolean | void; @@ -194,15 +196,9 @@ export type ServiceManager = ServiceManagerEventBus & { * Stop a service. * * @param id The service ID + * @param force Whether to force stop (kill) the service. */ - stopService(id: string): Promise; - - /** - * Stop a service forcibly (kill). - * - * @param id The service ID - */ - stopServiceForcibly(id: string): Promise; + stopService(id: string, force?: boolean): Promise; /** * Send pre-configured stop signal to the service. @@ -287,10 +283,17 @@ export type ServiceManager = ServiceManagerEventBus & { */ stopRunning(): Promise; + /** + * Kill all running services on this instance. + */ + killRunning(): Promise; + isRunning(id: string): boolean; waitForBusyAction(id: string): Promise; + waitForStopped(id: string): Promise; + // DON'T call those until you really know what you are doing. expandEngine(exp?: T): Promise; @@ -383,6 +386,7 @@ export async function init( templateManager, templateDirWatcher, db_, + appConfig_, currentContext.logger, ); initSessionEngine(db_); @@ -639,6 +643,8 @@ export async function stopService(id: string, force?: boolean) { try { if (force) { await engine.kill(internalSession.containerId, metaStorageForService(id)); + + } else { // lock only on soft stop, to allow hard-killing if any issues happen during stopping const unlock = lockBusyAction(id, "stop"); @@ -652,7 +658,6 @@ export async function stopService(id: string, force?: boolean) { if (isServicePending(id)) { unlock(error); } - ulckStatusTp(internalSession.containerId); return true; }); @@ -674,10 +679,6 @@ export async function stopService(id: string, force?: boolean) { } } -export async function stopServiceForcibly(id: string) { - return stopService(id, true); -} - export async function sendStopSignal(id: string) { const perma = await reqExists(id); const { internalSession } = reqRunning(id); @@ -820,12 +821,42 @@ export async function stopRunning() { await Promise.all(tasks); } +export async function killRunning() { + await Promise.all( + started.map( + async ({ id }) => stopService(id, true).catch((e) => currentContext.logger.error(e)) + ) + ) +} + export async function waitForBusyAction(id: string) { return new Promise((resolve, reject) => { whenUnlocked(id, (_, __, err) => (err ? reject(err) : resolve(null))); }); } +export async function waitForStopped(id: string) { + if (!isRunning(id)) { + // service not running, so we continue immediately + return; + } + + return new Promise((resolve, reject) => { + on("stop", ({ id, error }) => { + if (id !== id) { + // This call is not for me + return false; + } + + if (error) { + reject(error); + } else { + resolve(); + } + }); + }); +} + export function isRunning(id: string) { return getRunningService(id) != undefined; } @@ -918,6 +949,14 @@ function buildRunListener(session: ActiveServiceSession): RunListener { onClose: async () => { clearRunningServiceIfExists(serviceId); startedStates.delete(serviceId); + // clear any busy action that may potentially still be locked + try { + unlockBusyAction(serviceId); + } catch (e) { + if (e.message && e.message.includes("No busy action")) { + // ignore, since it just means there is no busy action to unlock, so nothing to do + } + } // Call stop event on the manager for the stopService() to potentially // unlock a busy action diff --git a/src/engine/middle.ts b/src/engine/middle.ts index f2cfd0e..217a50d 100644 --- a/src/engine/middle.ts +++ b/src/engine/middle.ts @@ -149,12 +149,6 @@ export const middleLayer = (manager: ServiceManager): ServiceManager => { argServiceIdExtractor(0), ), - stopServiceForcibly: decorateFunc( - manager.stopServiceForcibly, - "forceStop", - argServiceIdExtractor(0), - ), - sendStopSignal: decorateFunc( manager.sendStopSignal, "sendStopSignal", diff --git a/src/engine/monitoring/templateDirWatcher.ts b/src/engine/monitoring/templateDirWatcher.ts index 6e82d43..0069a5a 100644 --- a/src/engine/monitoring/templateDirWatcher.ts +++ b/src/engine/monitoring/templateDirWatcher.ts @@ -1,11 +1,11 @@ -import { templateBuildDir, debounce } from "@nsm/engine/monitoring/util"; +import { debounce } from "@nsm/engine/monitoring/util"; import { hashElement } from "folder-hash"; import { getFilteredPaths } from "@nsm/engine/ignore"; import { getAllTemplates } from "@nsm/engine/template"; import winston from "winston"; import chokidar, { FSWatcher } from "chokidar"; import path from "path"; -import { getTemplatesPath } from "@nsm/filestructure"; +import {getTemplateBuildDir, getTemplatesPath} from "@nsm/filestructure"; export type TemplateDirWatcher = { /** @@ -92,7 +92,7 @@ const watchTemplateDir = async (template: string) => { await recalculateTemplateHash(template); - const dir = templateBuildDir(template); + const dir = getTemplateBuildDir(template); const excluded = getFilteredPaths(dir); const recalc = debounce(() => recalculateTemplateHash(template), 2000); @@ -119,7 +119,7 @@ const watchTemplateDir = async (template: string) => { * @param template The name of the template to recalculate the hash for. */ const recalculateTemplateHash = async (template: string) => { - const dir = templateBuildDir(template); + const dir = getTemplateBuildDir(template); const excluded = getFilteredPaths(dir); if (hashingInProgress.has(template)) { diff --git a/src/engine/monitoring/util.ts b/src/engine/monitoring/util.ts index f171755..ab8094c 100644 --- a/src/engine/monitoring/util.ts +++ b/src/engine/monitoring/util.ts @@ -1,11 +1,3 @@ -import path from "path"; -import { getTemplatesPath } from "@nsm/filestructure"; - -// Returns the build directory for the template -export function templateBuildDir(template: string) { - return path.join(getTemplatesPath(), template); -} - /** * Returns a debounced version of the given function. * The debounced function will only be called after it has not been called for the specified number of milliseconds. diff --git a/src/filestructure.ts b/src/filestructure.ts index 2ae1213..d4f0275 100644 --- a/src/filestructure.ts +++ b/src/filestructure.ts @@ -1,35 +1,28 @@ import path from "path"; -import envPaths, { Paths } from "env-paths"; import { AppConfig } from "@nsm/config"; import fs from "fs"; -export const currentPaths: Paths = envPaths("nsm"); - let appConfig: AppConfig; export const init = (appConfig_: AppConfig) => { appConfig = appConfig_; }; -// The local resources dir (not the source of truth) -export const resourcesPath = path.join(process.cwd(), "resources"); - // The target (platform-agnostic) resources dir (the source of truth) -export const getResourcesTargetPath = () => { - const result = appConfig.getResourcesPath(); - if (result) { - return path.resolve(result); - } else { - return path.join(currentPaths.data); - } +export const getResourcesPath = () => { + return appConfig.getResourcesPath(); }; export const getTemplatesPath = () => { - return path.join(getResourcesTargetPath(), "templates"); + return appConfig.getTemplatesPath(); }; +export const getTemplateBuildDir = (template: string) => { + return appConfig.getTemplateBuildDir(template); +} + export const getTempPath = () => { - return currentPaths.temp; + return appConfig.getTempPath(); }; export const mkdirTemp = (...p: string[]) => { @@ -48,7 +41,7 @@ export const mkdirTemp = (...p: string[]) => { }; export const prepareFolders = () => { - const resourcesTargetPath = getResourcesTargetPath(); + const resourcesTargetPath = getResourcesPath(); if (!fs.existsSync(resourcesTargetPath)) { fs.mkdirSync(resourcesTargetPath, { recursive: true }); } diff --git a/src/logger.ts b/src/logger.ts index bb83588..c0a0c18 100644 --- a/src/logger.ts +++ b/src/logger.ts @@ -1,7 +1,7 @@ import winston from "winston"; import fs from "fs"; import path from "path"; -import { getResourcesTargetPath } from "@nsm/filestructure"; +import { getResourcesPath } from "@nsm/filestructure"; const { combine, timestamp, label, errors, printf } = winston.format; @@ -13,7 +13,7 @@ export function setCurrentGlobalLogger(logger: winston.Logger) { export function createLatestLogFile() { if ( - fs.existsSync(path.join(getResourcesTargetPath(), "logs", "latest.log")) + fs.existsSync(path.join(getResourcesPath(), "logs", "latest.log")) ) { const date = new Date(Date.now()).toJSON().slice(2, 10) + @@ -23,8 +23,8 @@ export function createLatestLogFile() { new Date(Date.now()).getMinutes(); fs.renameSync( - path.join(getResourcesTargetPath(), "logs", "latest.log"), - path.join(getResourcesTargetPath(), "logs", date + ".log"), + path.join(getResourcesPath(), "logs", "latest.log"), + path.join(getResourcesPath(), "logs", date + ".log"), ); } } @@ -46,7 +46,7 @@ export function createLogger(options?: { label?: string }) { transports: [ new winston.transports.Console(), new winston.transports.File({ - dirname: path.join(getResourcesTargetPath(), "logs"), + dirname: path.join(getResourcesPath(), "logs"), filename: "latest.log", }), ], diff --git a/src/resources.ts b/src/resources.ts index e0d7672..c186f55 100644 --- a/src/resources.ts +++ b/src/resources.ts @@ -1,6 +1,9 @@ import path from "path"; import fs from "fs"; -import { getResourcesTargetPath, resourcesPath } from "@nsm/filestructure"; +import { getResourcesPath} from "@nsm/filestructure"; + +// The local resources dir (not the source of truth) +const resourcesPath = path.join(process.cwd(), "resources"); /** * Reads resource from target dir. @@ -8,7 +11,7 @@ import { getResourcesTargetPath, resourcesPath } from "@nsm/filestructure"; * @param name The name of the resource in the target dir. */ export const readResource = (name: string) => { - const p = path.join(getResourcesTargetPath(), name); + const p = path.join(getResourcesPath(), name); return fs.readFileSync(p, "utf8"); }; @@ -19,7 +22,7 @@ export const readResource = (name: string) => { * @param name The name of the dir in the target dir. */ export const mkdirResource = (name: string) => { - const p = path.join(getResourcesTargetPath(), name); + const p = path.join(getResourcesPath(), name); fs.mkdirSync(p, { recursive: true }); }; @@ -36,7 +39,7 @@ export const saveResource = ( name: string, targetName: string, skipIfExists: boolean = false, - targetDirPath: string = getResourcesTargetPath(), + targetDirPath: string = getResourcesPath(), ) => { const targetPath = path.join(targetDirPath, targetName); // Create parent dirs if missing diff --git a/src/router/v1/service/resumeRoute.ts b/src/router/v1/service/resumeRoute.ts index 91e0c0f..c215907 100644 --- a/src/router/v1/service/resumeRoute.ts +++ b/src/router/v1/service/resumeRoute.ts @@ -32,7 +32,7 @@ export default async function ({ if (manager.isRunning(id)) { res .status(409) - .json({ status: 400, message: "Service is already running." }); + .json({ status: 409, message: "Service is already running." }); return; } diff --git a/src/router/v1/service/stopRoute.ts b/src/router/v1/service/stopRoute.ts index 34295a0..f2de0fd 100644 --- a/src/router/v1/service/stopRoute.ts +++ b/src/router/v1/service/stopRoute.ts @@ -8,7 +8,6 @@ import { consumeEnginePowerAction } from "@nsm/helpers"; export default async function ({ manager, - logger, }: AppContext): Promise { return { url: "/service/:id/stop", @@ -38,13 +37,7 @@ export default async function ({ return; } - consumeEnginePowerAction(async () => { - if (req.query.force === "true") { - await manager.stopServiceForcibly(id); - } else { - await manager.stopService(id); - } - }); + consumeEnginePowerAction(async () => manager.stopService(id, req.query.force === "true")); res.status(200).json({ status: 200, diff --git a/tests/api/api.test.ts b/tests/api/api.test.ts index 301bb1f..aa52d53 100644 --- a/tests/api/api.test.ts +++ b/tests/api/api.test.ts @@ -32,9 +32,9 @@ async function miniService(ctx: AppBootContext) { } } -async function stopMini(ctx: AppBootContext, id: string) { - await ctx.manager.stopService(id); - await ctx.manager.waitForBusyAction(id); // Await stop +async function killMini(ctx: AppBootContext, id: string) { + await ctx.manager.stopService(id, true); + await ctx.manager.waitForStopped(id); } describe("Test v1 API models", () => { @@ -160,7 +160,7 @@ describe("Test v1 API models", () => { test("Test /v1/service/{serviceId}/resume", async () => { const id = await miniService(ctx); log(id); - await stopMini(ctx, id); + await killMini(ctx, id); const res = await request(server).post("/v1/service/" + id + "/resume"); expect(res.status).toBe(200); expectProps(res.body, ["status", 200, "message", undefined]); @@ -218,7 +218,7 @@ describe("Test v1 API models", () => { return; } - return ctx.manager.stopRunning(); + return ctx.manager.killRunning(); }, 60000); // TODO: /v1/service//options diff --git a/tests/engine/image.test.ts b/tests/engine/image.test.ts index b4eb40b..8ec43a1 100644 --- a/tests/engine/image.test.ts +++ b/tests/engine/image.test.ts @@ -1,45 +1,13 @@ -import { afterAll, beforeAll, expect, it } from "@jest/globals"; -import { ServiceEngineI } from "@nsm/engine"; -import createEngine from "@nsm/engine/engine"; -import { init as initImageEngine } from "@nsm/engine/image"; -import getDb from "@nsm/database"; -import { Database } from "@nsm/database"; -import { StartedMariaDbContainer } from "@testcontainers/mariadb"; -import { initDbContainerForTest } from "../testUtils"; -import { PrismaClient } from "@prisma/client"; -import { processImage } from "@nsm/engine/image"; -import { createLogger } from "@nsm/logger"; -import { Template, TemplateManager } from "@nsm/engine/template"; -import { TemplateDirWatcher } from "@nsm/engine/monitoring/templateDirWatcher"; -import * as templateManager from "@nsm/engine/template"; -import * as templateDirWatcher from "@nsm/engine/monitoring/templateDirWatcher"; -import { YamlAppConfig } from "@nsm/config"; - -let container: StartedMariaDbContainer; - -let engine: ServiceEngineI; -let db: Database; - -beforeAll(async () => { - const [container_, dbUrl_] = await initDbContainerForTest(); - - container = container_; - engine = createEngine( - // Just to prevent assertion errors - new YamlAppConfig(), - ); - db = getDb( - new PrismaClient({ - datasourceUrl: dbUrl_, - }), - ); -}, 20000); - -afterAll(async () => { - if (container) { - await container.stop(); - } -}); +import {expect, it} from "@jest/globals"; +import {ServiceEngine} from "@nsm/engine"; +import {init as initImageEngine} from "@nsm/engine/image"; +import {processImage } from "@nsm/engine/image"; +import {prepareEnvForTemplate, Template, TemplateManager} from "@nsm/engine/template"; +import {TemplateDirWatcher} from "@nsm/engine/monitoring/templateDirWatcher"; +import {DeepMockProxy, mock, mockDeep} from "jest-mock-extended"; +import {Database, ImageModel} from "@nsm/database"; +import {createTestLogger} from "../testUtils"; +import {AppConfig} from "@nsm/config"; it("reuses image with same options", async () => { const template: Template = { @@ -54,51 +22,40 @@ it("reuses image with same options", async () => { }, }; - let buildCount = 0; + const engineMock = mock(); + engineMock + .build + .mockImplementation(async (imageId) => + imageId ?? "generated-image-id-" + (Math.random() * 1000000).toFixed(0)); - const customEngine: ServiceEngineI = { - ...engine, - build( - imageId: string | undefined, - _: string | undefined, - __: { - [p: string]: string; - }, - ): Promise { - buildCount++; + const templateManagerMock = mock(); + templateManagerMock + .prepareEnvForTemplate + .mockImplementation((template, env) => prepareEnvForTemplate(template, env)); + templateManagerMock + .getTemplate + .mockImplementation((id) => id === "test-template" ? template : null) - return Promise.resolve( - imageId ?? "generated-image-id-" + (Math.random() * 1000000).toFixed(0), - ); - }, - }; - const customTemplateManager: TemplateManager = { - ...templateManager, - getTemplate(id: string): Template | null { - if (id == "test-template") { - return template; - } + const templateDirWatcherMock = mock(); + templateDirWatcherMock.getTemplateHash.mockImplementation((template) => { + if (template == "test-template") { + return "test-hash"; + } - return null; - }, - }; - const customTemplateDirWatcher: TemplateDirWatcher = { - ...templateDirWatcher, - getTemplateHash(template: string): string { - if (template == "test-template") { - return "test-hash"; - } + throw new Error(`Unknown template ${template}`); + }); - throw new Error(`Unknown template ${template}`); - }, - }; + const dbMock = createMockDatabase(); + const appConfigMock = mock(); + appConfigMock.getTemplateBuildDir.mockImplementation(() => "/tmp/test-build-dir"); initImageEngine( - customEngine, - customTemplateManager, - customTemplateDirWatcher, - db, - createLogger(), + engineMock, + templateManagerMock, + templateDirWatcherMock, + dbMock, + appConfigMock, + createTestLogger(), ); const buildOptions = { @@ -112,5 +69,42 @@ it("reuses image with same options", async () => { expect(imageId2).not.toBeNull(); expect(imageId2).toEqual(imageId); - expect(buildCount).toBe(1); + expect(engineMock.build).toBeCalledTimes(1); }); + +const createMockDatabase = (): DeepMockProxy => { + const images: ImageModel[] = []; + + const db = mockDeep(); + db.imageRepository.saveImage.mockImplementation(async (image) => { + const existingIndex = images.findIndex((img) => img.id === image.id); + if (existingIndex !== -1) { + images[existingIndex] = image; // Overwrite existing image + } else { + images.push(image); // Add new image + } + return true; + }); + db.imageRepository.getImage.mockImplementation(async (id) => { + return images.find((image) => image.id === id); + }); + db.imageRepository.listImagesByOptions.mockImplementation(async (templateId, options) => { + return images.filter((image) => { + if (image.templateId !== templateId) { + return false; + } + + for (const key in options) { + if (image.buildOptions[key] !== options[key]) { + return false; + } + } + + return true; + }); + }); + db.permaRepository.listPermaUsingImage.mockImplementation(async () => {{ + return []; // In this mock, no services ae using any image + }}); + return db; +} diff --git a/tests/testUtils.ts b/tests/testUtils.ts index b40cc9a..8adc361 100644 --- a/tests/testUtils.ts +++ b/tests/testUtils.ts @@ -3,6 +3,7 @@ import { StartedMariaDbContainer, } from "@testcontainers/mariadb"; import { execSync } from "child_process"; +import winston from "winston"; export const initDbContainerForTest = async (): Promise< [StartedMariaDbContainer, string] @@ -19,3 +20,11 @@ export const initDbContainerForTest = async (): Promise< }); return [container, dbUrl]; }; + +export const createTestLogger = (): winston.Logger => { + return winston.createLogger({ + level: "debug", + format: winston.format.simple(), + transports: [new winston.transports.Console()], + }); +} \ No newline at end of file From a1d74c7e4f231a976b04030b634fc0cca5519e0c Mon Sep 17 00:00:00 2001 From: ZorTik Date: Thu, 11 Jun 2026 20:11:19 +0200 Subject: [PATCH 14/53] todo --- src/router/v1/service/stopCmdRoute.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/router/v1/service/stopCmdRoute.ts b/src/router/v1/service/stopCmdRoute.ts index 21860b7..f3469d0 100644 --- a/src/router/v1/service/stopCmdRoute.ts +++ b/src/router/v1/service/stopCmdRoute.ts @@ -6,7 +6,7 @@ import { checkServicePending } from "@nsm/router/util/preconditions"; export default async function ({ manager, -}: AppContext): Promise { +}: AppContext): Promise { // TODO: remove this return { url: "/service/:id/stopcmd", routes: { From b3fbf53c962b468d6db5d54bdb48bb152da14b97 Mon Sep 17 00:00:00 2001 From: ZorTik Date: Thu, 11 Jun 2026 22:37:56 +0200 Subject: [PATCH 15/53] fix: reboot --- src/engine/manager.ts | 3 +++ src/router/v1/service/rebootRoute.ts | 18 ++++++------------ tests/api/api.test.ts | 2 +- 3 files changed, 10 insertions(+), 13 deletions(-) diff --git a/src/engine/manager.ts b/src/engine/manager.ts index 4c8a38e..99f3762 100644 --- a/src/engine/manager.ts +++ b/src/engine/manager.ts @@ -194,6 +194,7 @@ export type ServiceManager = ServiceManagerEventBus & { /** * Stop a service. + * This hereby sends a stop signal and does not wait for it to be stopped. For waiting, use {@link waitForStopped}. * * @param id The service ID * @param force Whether to force stop (kill) the service. @@ -649,6 +650,8 @@ export async function stopService(id: string, force?: boolean) { // lock only on soft stop, to allow hard-killing if any issues happen during stopping const unlock = lockBusyAction(id, "stop"); // wait for stop + // this is really not necessary because any busy action is unlocked on stop, but + // just in case on("stop", ({ id: stoppedId, error }) => { if (stoppedId !== id) { // This call is not for me diff --git a/src/router/v1/service/rebootRoute.ts b/src/router/v1/service/rebootRoute.ts index af6961f..0d55c3c 100644 --- a/src/router/v1/service/rebootRoute.ts +++ b/src/router/v1/service/rebootRoute.ts @@ -8,13 +8,13 @@ import { consumeEnginePowerAction } from "@nsm/helpers"; export default async function ({ manager, - logger, }: AppContext): Promise { return { url: "/service/:id/reboot", routes: { post: async (req, res) => { const id = req.params.id; + const isForce = req.query.force === "true"; if (!id) { res .status(400) @@ -32,17 +32,11 @@ export default async function ({ } consumeEnginePowerAction(() => - manager.stopService(id).then(() => { - // Service stopped successfully, now wait for it to be unlocked before resuming. - - manager.whenUnlocked(id, (_, __, err) => { - if (err) { - logger.error(err); - } else { - manager.resumeService(id); - } - }); - }), + manager.stopService(id, isForce) + // continue after service is stopped + .then(() => manager.waitForStopped(id)) + // resume + .then(() => manager.resumeService(id)), ); res.status(200).json({ diff --git a/tests/api/api.test.ts b/tests/api/api.test.ts index aa52d53..32c7cba 100644 --- a/tests/api/api.test.ts +++ b/tests/api/api.test.ts @@ -187,7 +187,7 @@ describe("Test v1 API models", () => { test("Test /v1/service/{serviceId}/reboot", async () => { const id = await miniService(ctx); log(id); - const res = await request(server).post("/v1/service/" + id + "/reboot"); + const res = await request(server).post("/v1/service/" + id + "/reboot?force=true"); expect(res.status).toBe(200); expectProps(res.body, ["status", 200, "message", undefined]); // Wait for it to be started From 741ee73a886046fa67d15f3391e7401ba348052f Mon Sep 17 00:00:00 2001 From: ZorTik Date: Thu, 11 Jun 2026 22:53:34 +0200 Subject: [PATCH 16/53] feat: openapi force param in reboot route --- openapi.yml | 5 +++++ src/engine/manager.ts | 18 +++++++++++++++--- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/openapi.yml b/openapi.yml index 0799f3c..7cb90ff 100644 --- a/openapi.yml +++ b/openapi.yml @@ -468,6 +468,11 @@ paths: required: true schema: type: "string" + - name: "force" + in: "query" + required: false + schema: + type: boolean responses: "200": description: "Successfully rebooted service." diff --git a/src/engine/manager.ts b/src/engine/manager.ts index 99f3762..8927286 100644 --- a/src/engine/manager.ts +++ b/src/engine/manager.ts @@ -118,9 +118,14 @@ type ServiceEvent = { error?: Error; }; +type ServiceStateChangeEvent = ServiceEvent & { + state: State; +} + type ServiceManagerEvents = { resume: ServiceEvent; stop: ServiceEvent; + statechange: ServiceStateChangeEvent; }; /** @@ -947,7 +952,7 @@ function buildRunListener(session: ActiveServiceSession): RunListener { // The internal run listener of this manager const internalRunListener: RunListener = { onStateChange: (state) => { - startedStates.set(serviceId, state.ready ? "RUNNING" : "BUILDING"); + setServiceState(serviceId, state.ready ? "RUNNING" : "BUILDING"); }, onClose: async () => { clearRunningServiceIfExists(serviceId); @@ -961,8 +966,6 @@ function buildRunListener(session: ActiveServiceSession): RunListener { } } - // Call stop event on the manager for the stopService() to potentially - // unlock a busy action callManagerEvent("stop", { id: serviceId }); currentContext.logger.debug("Service " + serviceId + " stopped"); @@ -976,6 +979,15 @@ function buildRunListener(session: ActiveServiceSession): RunListener { ]); } +function setServiceState(id: string, state: State) { + startedStates.set(id, state); + + callManagerEvent("statechange", { + id, + state, + }); +} + /** * Returns the local service state managed by this manager. * From 011d240d683a667016d11e851e0085893db786d1 Mon Sep 17 00:00:00 2001 From: ZorTik Date: Fri, 12 Jun 2026 01:17:08 +0200 Subject: [PATCH 17/53] feat: refactor http handling & parseModel --- src/router/index.ts | 26 ++++++++++++------- src/router/middlewares/parseModel.ts | 39 ++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 10 deletions(-) create mode 100644 src/router/middlewares/parseModel.ts diff --git a/src/router/index.ts b/src/router/index.ts index 5419d4b..04b87fb 100644 --- a/src/router/index.ts +++ b/src/router/index.ts @@ -5,7 +5,7 @@ import { measureEventLoop } from "@nsm/profiler"; export type RouterHandler = { url: string; - routes: { [method: string]: RequestHandler }; + routes: { [method: string]: RequestHandler|RequestHandler[] }; }; type RouterInit = (context: AppContext) => Promise; @@ -15,7 +15,7 @@ async function api(ver: string, context: AppContext, routes: RouterInit[]) { const router = Router(); router.use(json()); if (context.debug) { - router.use((req, res, next) => { + router.use((req, _, next) => { if (req.body) { context.logger.debug(`Body: ${JSON.stringify(req.body)}`); } else { @@ -36,16 +36,22 @@ async function api(ver: string, context: AppContext, routes: RouterInit[]) { let reg = false; for (const method of ["get", "post", "put", "delete"]) { - if (handler.routes[method]) { - // Register handler to express - router[method]( - handler.url, - (req, res, next) => { + const userDefinedRoutes = handler.routes[method]; + if (userDefinedRoutes) { + const handlers: RequestHandler[] = [ + (req, _, next) => { context.logger.debug(`${method.toUpperCase()} ${req.url}`); next(); - }, - handler.routes[method], - ); + } + ]; + if (Array.isArray(userDefinedRoutes)) { + handlers.push(...userDefinedRoutes); + } else { + handlers.push(userDefinedRoutes); + } + + // Register handler to express + router[method](handler.url, ...handlers); reg = true; } } diff --git a/src/router/middlewares/parseModel.ts b/src/router/middlewares/parseModel.ts new file mode 100644 index 0000000..b14f93b --- /dev/null +++ b/src/router/middlewares/parseModel.ts @@ -0,0 +1,39 @@ +import express from "express"; +import z from "zod"; + +export interface ParseModelOptions { + model: { + body: z.ZodObject; + query: z.ZodObject; + params: z.ZodObject; + } +} + +/** + * Middleware to parse and validate request parts using Zod schemas. + * + * @param options The options. + */ +export const parseModel = ( + options: ParseModelOptions +): express.RequestHandler => { + return (req, res, next) => { + for (const key in options.model) { + const model = options.model[key as keyof ParseModelOptions["model"]]; + + const result = model.safeParse(req[key as keyof express.Request]); + if (result.success) { + continue; + } + + res.status(400).json({ + status: 400, + message: `Invalid ${key} format.`, + errors: result.error.errors, + }); + return; + } + + next(); + } +} \ No newline at end of file From 334bdb1cea398c3e68ecf17c7647efe3d4338e54 Mon Sep 17 00:00:00 2001 From: ZorTik Date: Fri, 12 Jun 2026 15:04:09 +0200 Subject: [PATCH 18/53] feat: synchronout actions processing in routes --- src/engine/error.ts | 58 +++++ src/engine/manager.ts | 235 +++++++++++-------- src/engine/middle.ts | 60 +++-- src/router/index.ts | 31 +-- src/router/middlewares/catchKnownErrors.ts | 18 ++ src/router/middlewares/debugRequestLogger.ts | 27 +++ src/router/middlewares/eventLoopProfiler.ts | 12 + src/router/v1/service/createRoute.ts | 41 ++-- src/router/v1/service/deleteRoute.ts | 14 +- src/router/v1/service/logsRoute.ts | 76 +++--- src/router/v1/service/rebootRoute.ts | 23 +- src/router/v1/service/resumeRoute.ts | 22 +- src/router/v1/service/sessionsRoute.ts | 36 ++- src/router/v1/service/stopRoute.ts | 22 +- src/router/v1/session/sessionLogsRoute.ts | 28 +-- src/util/promises.ts | 6 + 16 files changed, 389 insertions(+), 320 deletions(-) create mode 100644 src/engine/error.ts create mode 100644 src/router/middlewares/catchKnownErrors.ts create mode 100644 src/router/middlewares/debugRequestLogger.ts create mode 100644 src/router/middlewares/eventLoopProfiler.ts diff --git a/src/engine/error.ts b/src/engine/error.ts new file mode 100644 index 0000000..7ab28b3 --- /dev/null +++ b/src/engine/error.ts @@ -0,0 +1,58 @@ +export class InternalError extends Error { + constructor(message: string) { + super(message); + } +} + +export class KnownError extends Error { + constructor( + public readonly code: number, + message: string + ) { + super(message); + } +} + +export class InvalidMetaError extends KnownError { + constructor(message: string) { + super(400, message); + } +} + +export class ServiceNotFoundError extends KnownError { + constructor( + public readonly serviceId: string + ) { + super(404, `Service with ID ${serviceId} not found.`); + } +} + +export class ServiceNotRunningError extends KnownError { + constructor( + public readonly serviceId: string + ) { + super(409, `Service with ID ${serviceId} is not running.`); + } +} + +export class ServiceAlreadyRunningError extends KnownError { + constructor( + public readonly serviceId: string + ) { + super(409, `Service with ID ${serviceId} is already running.`); + } +} + +export class ServiceWasNeverActiveError extends KnownError { + constructor() { + super(400, "Service was never active."); + } +} + +export class TemplateNotFoundError extends KnownError { + constructor( + public readonly templateId: string + ) { + super(404, `Template with ID ${templateId} not found.`); + } +} \ No newline at end of file diff --git a/src/engine/manager.ts b/src/engine/manager.ts index 8927286..e479561 100644 --- a/src/engine/manager.ts +++ b/src/engine/manager.ts @@ -13,6 +13,7 @@ import { getAllTemplates, } from "./template"; import * as templateManager from "./template"; +import * as sessionManager from "./session"; import * as templateDirWatcher from "./monitoring/templateDirWatcher"; import crypto from "crypto"; import { randomPort as retrieveRandomPort } from "@nsm/util/port"; @@ -27,7 +28,7 @@ import { } from "./asyncp"; import winston from "winston"; import { isDebug } from "../helpers"; -import { resolveSequentially } from "@nsm/util/promises"; +import {AsyncTask, resolveSequentially} from "@nsm/util/promises"; import { watchTemplateDirChanges } from "@nsm/engine/monitoring/templateDirWatcher"; import { processImage, @@ -42,6 +43,13 @@ import { init as initSessionEngine, } from "@nsm/engine/session"; import { AppConfig } from "@nsm/config"; +import { + InternalError, + InvalidMetaError, + ServiceAlreadyRunningError, + ServiceNotFoundError, + ServiceNotRunningError, ServiceWasNeverActiveError, TemplateNotFoundError +} from "@nsm/engine/error"; export type Options = { /** @@ -186,6 +194,7 @@ export type ServiceManager = ServiceManagerEventBus & { * @param template The template ID (folder name) to use * @param options The options to use. Options will be stored for later use. * @returns The service ID + * @throws InvalidMetaError if the template meta is invalid */ createService(template: string, options: Options): Promise; // Service ID @@ -193,9 +202,8 @@ export type ServiceManager = ServiceManagerEventBus & { * Resume a service. * * @param id The service ID - * @returns Whether the service was resumed */ - resumeService(id: string): Promise; + resumeService(id: string): Promise>; /** * Stop a service. @@ -204,13 +212,14 @@ export type ServiceManager = ServiceManagerEventBus & { * @param id The service ID * @param force Whether to force stop (kill) the service. */ - stopService(id: string, force?: boolean): Promise; + stopService(id: string, force?: boolean): Promise>; /** * Send pre-configured stop signal to the service. * * @param id The service ID * @returns Whether the signal has been sent + * @throws InvalidMetaError if the service does not have the required meta for stop signal (e.g. stop command) */ sendStopSignal(id: string): Promise; @@ -257,6 +266,14 @@ export type ServiceManager = ServiceManagerEventBus & { */ getLastPowerError(id: string): Error | undefined; + /** + * Get the last session ID of a service. + * + * @param id The service ID + * @throws ServiceWasNeverActiveError if the service was never active and thus does not have a last session + */ + getLastSession(id: string): Promise; + /** * Get list of running services on this node. */ @@ -331,20 +348,6 @@ export type ServiceInfo = PermaModel & { export type State = "RUNNING" | "BUILDING" | "STOPPED"; -// 1 = unknown, 2 = conflict, 3 = not found -export type StatusCode = 1 | 2 | 3; - -export class _InternalError extends Error { - readonly code: StatusCode; - readonly msg: string; - - constructor(msg: string, code?: StatusCode) { - super(msg); - this.code = code ?? 1; - this.msg = msg; - } -} - export let engine: ServiceEngineI = undefined; export let nodeId: string; @@ -500,7 +503,7 @@ export async function createService(template: string, options: Options) { ...(serviceSettings.meta ?? {}), }; if (!meta || !meta.stopCmd) { - throw new _InternalError("Invalid template meta for " + template); + throw new InvalidMetaError("Invalid template meta for " + template); } const serviceId = crypto.randomUUID(); // Create new unique service id @@ -525,7 +528,7 @@ export async function createService(template: string, options: Options) { let err: any; // Save permanent info if (!(await db.permaRepository.savePerma(perma))) { - err = new _InternalError("Failed to save perma info to database"); + err = new InternalError("Failed to save perma info to database"); } if (err) { @@ -543,7 +546,7 @@ export async function createService(template: string, options: Options) { export async function resumeService(id: string) { reqNotRunning(id); - let { template, options, env, network, port } = await getPermaModel(id); + let { template, options, env, network, port } = await reqExists(id); const { defaults, env: settingsEnv } = reqTemplate(template).settings; // Filter env to only those that are defined in settings.yml, because those are the only ones that @@ -575,7 +578,7 @@ export async function resumeService(id: string) { }; const perma = await db.permaRepository.getPerma(id); - let image = perma.imageId; + //let image = perma.imageId; // Propagate other options to env, so they can be used in image processing and building propagateOptionsToEnv(runOptions, runOptions.env); @@ -586,60 +589,65 @@ export async function resumeService(id: string) { // image rebuild const { SERVICE_ID, SERVICE_PORT, SERVICE_PORTS, ...buildEnv } = runOptions.env; - const processedImage = await processImage(image, template, buildEnv); // TODO: tato funkce má poslední parametr messageListener, vymyslet jak sem propagovat message listener z session - // If the image was changed by processing (e.g. it was built or rebuilt), update the image id in database - if (processedImage != image) { - image = processedImage; - - // Update image in database if it was changed by processing - perma.imageId = image; - await db.permaRepository.savePerma(perma); - } - let session: ActiveServiceSession | undefined; - let containerId: string | undefined; - try { - // Run the container with the built image and save the container id for later use. - if (image) { - session = await beginServiceSession(id); - containerId = await engine.run( - image, - id, - runOptions, - meta, - buildRunListener(session), - ); - } - } catch (e) { - currentContext.logger.error("Failed to run container for service " + id); - currentContext.logger.error(e); - } + return new AsyncTask( + // TODO: tato funkce má poslední parametr messageListener, vymyslet jak sem propagovat message listener z session + processImage(perma.imageId, template, buildEnv) + .then(async (image) => { + // If the image was changed by processing (e.g. it was built or rebuilt), update the image id in database + if (image != perma.imageId) { - let success: boolean = false; - if (containerId) { - const runningService: RunningService = { - id, - session, - internalSession: { - containerId, - }, - }; - started.push(runningService); - success = true; - } + // Update image in database if it was changed by processing + perma.imageId = image; + await db.permaRepository.savePerma(perma); + } - if (success == true) { - currentContext.logger.debug("Service " + id + " resumed"); - callManagerEvent("resume", { id }); - } else { - errors[id] = new Error("Failed to resume service"); - clearRunningServiceIfExists(id); - callManagerEvent("resume", { id, error: errors[id] }); - } + return image; + }) + .then(async (image) => { + let session: ActiveServiceSession | undefined; + let containerId: string | undefined; + try { + // Run the container with the built image and save the container id for later use. + if (image) { + session = await beginServiceSession(id); + containerId = await engine.run( + image, + id, + runOptions, + meta, + buildRunListener(session), + ); + } + } catch (e) { + currentContext.logger.error("Failed to run container for service " + id); + currentContext.logger.error(e); + } - unlock(); + let success: boolean = false; + if (containerId) { + const runningService: RunningService = { + id, + session, + internalSession: { + containerId, + }, + }; + started.push(runningService); + success = true; + } - return true; + if (success == true) { + currentContext.logger.debug("Service " + id + " resumed"); + callManagerEvent("resume", { id }); + } else { + errors[id] = new Error("Failed to resume service"); + clearRunningServiceIfExists(id); + callManagerEvent("resume", { id, error: errors[id] }); + } + }) + .finally(() => unlock()) + ); } export async function stopService(id: string, force?: boolean) { @@ -647,27 +655,31 @@ export async function stopService(id: string, force?: boolean) { const { internalSession } = reqRunning(id); try { + let awaitingPromise: Promise; if (force) { await engine.kill(internalSession.containerId, metaStorageForService(id)); - - + // resolves immediately on kill + awaitingPromise = Promise.resolve(); } else { // lock only on soft stop, to allow hard-killing if any issues happen during stopping const unlock = lockBusyAction(id, "stop"); - // wait for stop - // this is really not necessary because any busy action is unlocked on stop, but - // just in case - on("stop", ({ id: stoppedId, error }) => { - if (stoppedId !== id) { - // This call is not for me - return false; - } - - if (isServicePending(id)) { - unlock(error); - } - return true; - }); + awaitingPromise = new Promise((resolve) => { + // wait for stop + // this is really not necessary because any busy action is unlocked on stop, but + // just in case and for the promise + on("stop", ({ id: stoppedId, error }) => { + if (stoppedId !== id) { + // This call is not for me + return false; + } + + if (isServicePending(id)) { + unlock(error); + } + resolve(); + return true; + }); + }) // TODO: stop strategy const service = await getService(id); @@ -680,6 +692,8 @@ export async function stopService(id: string, force?: boolean) { await engine.stop(internalSession.containerId); } } + + return new AsyncTask(awaitingPromise); } catch (e) { currentContext.logger.error(e); @@ -693,7 +707,7 @@ export async function sendStopSignal(id: string) { const stopCmd = perma.meta?.stopCmd; if (!stopCmd) { - throw new _InternalError("Service does not have stop command set."); + throw new InvalidMetaError("Service does not have stop command set."); } await engine.cmd(internalSession.containerId, stopCmd); @@ -705,7 +719,7 @@ export async function deleteService(id: string) { await stopService(id, true); } catch (e) { // Skip not running error - if (!(e.code && e.code == 2)) { + if (!(e instanceof ServiceNotRunningError)) { throw e; } } @@ -801,6 +815,28 @@ export function getLastPowerError(id: string) { return errors[id]; } +export async function getLastSession(id: string) { + await reqExists(id); + + const runningService = getRunningService(id); + if (runningService) { + // Service currently running, we can use logs from the current session + return runningService.session; + } else { + // Service not running, so we need to retrieve last session ID + const lastSession = await sessionManager.listSessions({ + filter: { serviceId: id }, + sort: { by: "startedAt", direction: "desc" }, + page: { index: 0, size: 1 }, + }); + if (lastSession && lastSession.length > 0) { + return lastSession[0]; + } + } + + throw new ServiceWasNeverActiveError(); +} + export async function listServices(options: ListServicesOptions) { const meta = options.filter?.meta; return db.permaRepository @@ -1000,29 +1036,20 @@ function getServiceState(id: string) { // --------------------------------------------------------------------------------------- -async function getPermaModel(id: string) { +async function reqExists(id: string) { const perma_ = await db.permaRepository.getPerma(id); if (!perma_) { - // Service does not exist - throw new _InternalError("Not found.", 3); + // service does not exist + throw new ServiceNotFoundError(id); } return perma_; } -async function reqExists(id: string) { - const perma = await db.permaRepository.getPerma(id); - if (!perma) { - throw new _InternalError("Service not found.", 3); - } - - return perma; -} - function reqRunning(id: string) { const session = getRunningService(id); if (!session) { - throw new _InternalError("This service is not running.", 2); + throw new ServiceNotRunningError(id); } return session; @@ -1030,14 +1057,14 @@ function reqRunning(id: string) { function reqNotRunning(id: string) { if (isRunning(id)) { - throw new _InternalError("Already running.", 2); + throw new ServiceAlreadyRunningError(id); } } function reqTemplate(id: string) { const template = getTemplate(id); if (!template) { - throw new _InternalError("" + "Template not found.", 3); + throw new TemplateNotFoundError(id); } return template; diff --git a/src/engine/middle.ts b/src/engine/middle.ts index 217a50d..0eb968f 100644 --- a/src/engine/middle.ts +++ b/src/engine/middle.ts @@ -1,5 +1,7 @@ -import { _InternalError, ServiceManager } from "@nsm/engine/manager"; -import { currentContext } from "@nsm/app"; +import {ServiceManager} from "@nsm/engine/manager"; +import {currentContext} from "@nsm/app"; +import {KnownError} from "@nsm/engine/error"; +import {AsyncTask} from "@nsm/util/promises"; export type ServiceActionType = | "create" @@ -65,28 +67,48 @@ const decorateFunc = ) => Promise>( ) => { return async (...args: Parameters) => { try { - return await fn(...args); + // @ts-ignore + const result = await fn(...args); + if (result instanceof AsyncTask) { + // if the result is a scheduled task, attach error handler to catch any errors during the execution of the task + result.promise.catch((e) => handleExecutionError(serviceIdExtractor, args, actionType, e)); + } + + return result; } catch (e) { - const action: ServiceActionError = { - serviceId: serviceIdExtractor?.(args), - type: actionType, - message: e instanceof Error ? e.message : String(e), - }; - await publishError(action); - - // don't log stack trace of known errors - const errorMeta: any[] = - e instanceof _InternalError && e.code != 1 ? [] : [e]; - currentContext.logger.error( - `${action.serviceId ? `Service ${action.serviceId} f` : "F"}ailed action ${action.type}: ${action.message}`, - ...errorMeta, - ); - - throw e; + await handleExecutionError(serviceIdExtractor, args, actionType, e); } }; }; +/** + * Handles errors that occur during the execution of a service action. + * + * @see {@link decorateFunc} + */ +const handleExecutionError = async ) => Promise>( + serviceIdExtractor: (args: Parameters) => string, + args: Parameters, + actionType: ServiceActionType, + e: Error +) => { + const action: ServiceActionError = { + serviceId: serviceIdExtractor?.(args), + type: actionType, + message: e instanceof Error ? e.message : String(e), + }; + await publishError(action); + + // don't log stack trace of known errors + const errorMeta: any[] = e instanceof KnownError ? [] : [e]; + currentContext.logger.error( + `${action.serviceId ? `Service ${action.serviceId} f` : "F"}ailed action ${action.type}: ${action.message}`, + ...errorMeta, + ); + + throw e; +} + /** * Creates a service ID extractor function that extracts the service ID from the * specified argument index of the function arguments. diff --git a/src/router/index.ts b/src/router/index.ts index 04b87fb..45c045f 100644 --- a/src/router/index.ts +++ b/src/router/index.ts @@ -1,7 +1,9 @@ import { AppContext } from "../app"; import { json, RequestHandler, Router } from "express"; import v1Routes from "./v1"; -import { measureEventLoop } from "@nsm/profiler"; +import {eventLoopProfiler} from "@nsm/router/middlewares/eventLoopProfiler"; +import {debugRequestLogger} from "@nsm/router/middlewares/debugRequestLogger"; +import {catchKnownErrors} from "@nsm/router/middlewares/catchKnownErrors"; export type RouterHandler = { url: string; @@ -15,35 +17,21 @@ async function api(ver: string, context: AppContext, routes: RouterInit[]) { const router = Router(); router.use(json()); if (context.debug) { - router.use((req, _, next) => { - if (req.body) { - context.logger.debug(`Body: ${JSON.stringify(req.body)}`); - } else { - context.logger.debug("No body"); - } - next(); - }); + router.use(debugRequestLogger({context})); // Measure event loop process time if in debug mode - router.use((_, __, next) => { - measureEventLoop(); - next(); - }); + router.use(eventLoopProfiler()); } + for (let init of routes) { // Create handler with changed router to the sub-router that will be // used specifically for this API version const handler = await init({ ...context, router }); - let reg = false; + let reg = false; for (const method of ["get", "post", "put", "delete"]) { const userDefinedRoutes = handler.routes[method]; if (userDefinedRoutes) { - const handlers: RequestHandler[] = [ - (req, _, next) => { - context.logger.debug(`${method.toUpperCase()} ${req.url}`); - next(); - } - ]; + const handlers: RequestHandler[] = []; if (Array.isArray(userDefinedRoutes)) { handlers.push(...userDefinedRoutes); } else { @@ -55,10 +43,13 @@ async function api(ver: string, context: AppContext, routes: RouterInit[]) { reg = true; } } + if (reg) { context.logger.debug(`Registered route ${handler.url}`); } } + router.use(catchKnownErrors()); + context.router.use(`/${ver}`, router); } diff --git a/src/router/middlewares/catchKnownErrors.ts b/src/router/middlewares/catchKnownErrors.ts new file mode 100644 index 0000000..ea6c8f3 --- /dev/null +++ b/src/router/middlewares/catchKnownErrors.ts @@ -0,0 +1,18 @@ +import express from "express"; +import {KnownError} from "@nsm/engine/error"; + +/** + * Middleware to catch known errors and respond properly. + */ +export const catchKnownErrors = (): express.ErrorRequestHandler => { + return (err, _, res) => { + let status = 500; + let message = "Internal Server Error"; + if (err instanceof KnownError) { + status = err.code; + message = err.message; + } + + res.status(status).json({ status, message }).end(); + } +} \ No newline at end of file diff --git a/src/router/middlewares/debugRequestLogger.ts b/src/router/middlewares/debugRequestLogger.ts new file mode 100644 index 0000000..f3a12a0 --- /dev/null +++ b/src/router/middlewares/debugRequestLogger.ts @@ -0,0 +1,27 @@ +import express from "express"; +import {AppContext} from "@nsm/app"; + +export interface Options { + context: AppContext; +} + +/** + * Middleware for logging incoming requests in debug mode. + * + * @param options The options. + */ +export const debugRequestLogger = ( + options: Options +): express.RequestHandler => { + return (req, _, next) => { + const context = options.context; + + context.logger.debug(`${req.method.toUpperCase()} ${req.url}`); + if (req.body) { + context.logger.debug(`Body: ${JSON.stringify(req.body)}`); + } else { + context.logger.debug("No body"); + } + next(); + } +} \ No newline at end of file diff --git a/src/router/middlewares/eventLoopProfiler.ts b/src/router/middlewares/eventLoopProfiler.ts new file mode 100644 index 0000000..8652623 --- /dev/null +++ b/src/router/middlewares/eventLoopProfiler.ts @@ -0,0 +1,12 @@ +import express from "express"; +import {measureEventLoop} from "@nsm/profiler"; + +/** + * Middleware to measure the event loop delay for each request. + */ +export const eventLoopProfiler = (): express.RequestHandler => { + return (_, __, next) => { + measureEventLoop(); + next(); + } +} \ No newline at end of file diff --git a/src/router/v1/service/createRoute.ts b/src/router/v1/service/createRoute.ts index 8549f27..5a8bbd0 100644 --- a/src/router/v1/service/createRoute.ts +++ b/src/router/v1/service/createRoute.ts @@ -3,7 +3,7 @@ import { AppContext } from "@nsm/app"; import { Options } from "@nsm/engine"; import { clock } from "@nsm/util/clock"; import { prepareEnvForTemplate } from "@nsm/engine/template"; -import { consumeEnginePowerAction } from "@nsm/helpers"; +import {TemplateNotFoundError} from "@nsm/engine/error"; export default async function ({ manager, @@ -22,12 +22,9 @@ export default async function ({ } const template = manager.getTemplate(req.body.template); if (!template) { - res - .status(400) - .json({ status: 400, message: "Invalid template ID." }) - .end(); - return; + throw new TemplateNotFoundError(req.body.template); } + let env = req.body.env ?? {}; try { env = prepareEnvForTemplate(template, env); @@ -39,27 +36,21 @@ export default async function ({ // Build options const options: Options = req.body; options.env = env; - // Create the service - try { - const serviceId = await manager.createService(template.id, options); - // Resume right afterward - consumeEnginePowerAction(() => manager.resumeService(serviceId)); + const serviceId = await manager.createService(template.id, options); - res - .status(200) - .json({ - status: 200, - message: - "Service create action successfully registered to be completed in a moment.", - serviceId, - statusPath: "/v1/service/" + serviceId + "/powerstatus", - time: clk.durFromCreation(), - }) - .end(); - } catch (e) { - res.status(500).json({ status: 500, message: e.message }).end(); - } + await manager.resumeService(serviceId); + + res + .status(200) + .json({ + status: 200, + message: "Service created successfully.", + serviceId, + statusPath: "/v1/service/" + serviceId + "/powerstatus", + time: clk.durFromCreation(), + }) + .end(); }, }, }; diff --git a/src/router/v1/service/deleteRoute.ts b/src/router/v1/service/deleteRoute.ts index 63b2f3c..bfb7f07 100644 --- a/src/router/v1/service/deleteRoute.ts +++ b/src/router/v1/service/deleteRoute.ts @@ -1,7 +1,5 @@ import { AppContext } from "@nsm/app"; import { RouterHandler } from "../../index"; -import { handleErr } from "@nsm/util/routes"; -import { checkServiceExists } from "@nsm/router/util/preconditions"; export default async function ({ manager, @@ -20,16 +18,10 @@ export default async function ({ }); return; } - if (!(await checkServiceExists(id, manager, res))) { - return; - } - try { - await manager.deleteService(id); - res.status(200).json({ status: 200, message: "Service deleted." }); - } catch (e) { - handleErr(e, res); - } + await manager.deleteService(id); + + res.status(200).json({ status: 200, message: "Service deleted." }); }, }, }; diff --git a/src/router/v1/service/logsRoute.ts b/src/router/v1/service/logsRoute.ts index 55aa338..c957e85 100644 --- a/src/router/v1/service/logsRoute.ts +++ b/src/router/v1/service/logsRoute.ts @@ -1,7 +1,7 @@ import { AppContext } from "@nsm/app"; import { RouterHandler } from "@nsm/router"; -import { ListRecordsArgs } from "@nsm/database"; -import { checkServiceExists } from "@nsm/router/util/preconditions"; +import {ServiceWasNeverActiveError} from "@nsm/engine/error"; +import {ServiceLogRecordModel} from "@nsm/database"; export default async function (ctx: AppContext): Promise { return { @@ -18,60 +18,40 @@ export default async function (ctx: AppContext): Promise { }); return; } - if (!(await checkServiceExists(id, ctx.manager, res))) { - return; - } - - let sessionId: string; - - const runningService = ctx.manager.getRunningService(id); - if (runningService) { - // Service currently running, we can use logs from the current session - sessionId = runningService.session.id; - } else { - // Service not running, so we need to retrieve last session ID - const lastSession = await ctx.sessionManager.listSessions({ - filter: { serviceId: id }, - sort: { by: "startedAt", direction: "desc" }, - page: { index: 0, size: 1 }, - }); - if (lastSession && lastSession.length > 0) { - sessionId = lastSession[0].id; - } - } - - if (!sessionId) { - res - .status(400) - .json({ status: 400, message: "Service was never active." }); - return; - } - - const pageIndex = req.query.pageIndex ? Number(req.query.pageIndex) : 0; - const pageSize = req.query.pageSize ? Number(req.query.pageSize) : 10; // Use pagination only if it was requested by params const page = req.query.pageIndex || req.query.pageSize ? { - index: pageIndex, - size: pageSize, - } + index: req.query.pageIndex ? Number(req.query.pageIndex) : 0, + size: req.query.pageSize ? Number(req.query.pageSize) : 10, + } : undefined; - const args: ListRecordsArgs = { - filter: { - sessionId, - }, - sort: { - by: "timestamp", - direction: "asc", - }, - page, - }; - const logs = await ctx.sessionManager.listSessionLogs(args); + let logs: ServiceLogRecordModel[]; + try { + const session = await ctx.manager.getLastSession(id); + logs = await ctx.sessionManager.listSessionLogs({ + filter: { + sessionId: session.id, + }, + sort: { + by: "timestamp", + direction: "asc", + }, + page, + }); + } catch (e) { + if (e instanceof ServiceWasNeverActiveError) { + logs = []; + } else { + throw e; + } + } - res.status(200).json({ logs }); + res.status(200).json({ + logs + }); }, }, }; diff --git a/src/router/v1/service/rebootRoute.ts b/src/router/v1/service/rebootRoute.ts index 0d55c3c..eac1cf5 100644 --- a/src/router/v1/service/rebootRoute.ts +++ b/src/router/v1/service/rebootRoute.ts @@ -1,10 +1,5 @@ import { AppContext } from "@nsm/app"; import { RouterHandler } from "../../index"; -import { - checkServiceExists, - checkServicePending, -} from "@nsm/router/util/preconditions"; -import { consumeEnginePowerAction } from "@nsm/helpers"; export default async function ({ manager, @@ -24,25 +19,13 @@ export default async function ({ }); return; } - if (!(await checkServiceExists(id, manager, res))) { - return; - } - if (!checkServicePending(id, res)) { - return; - } - consumeEnginePowerAction(() => - manager.stopService(id, isForce) - // continue after service is stopped - .then(() => manager.waitForStopped(id)) - // resume - .then(() => manager.resumeService(id)), - ); + const task = await manager.stopService(id, isForce); + task.promise.then(() => manager.resumeService(id)); res.status(200).json({ status: 200, - message: - "Service reboot action successfully registered to be completed in a moment.", + message: "Service reboot action scheduled.", }); }, }, diff --git a/src/router/v1/service/resumeRoute.ts b/src/router/v1/service/resumeRoute.ts index c215907..7140bbc 100644 --- a/src/router/v1/service/resumeRoute.ts +++ b/src/router/v1/service/resumeRoute.ts @@ -1,10 +1,5 @@ import { AppContext } from "@nsm/app"; import { RouterHandler } from "../../index"; -import { - checkServiceExists, - checkServicePending, -} from "@nsm/router/util/preconditions"; -import { consumeEnginePowerAction } from "@nsm/helpers"; export default async function ({ manager, @@ -23,25 +18,12 @@ export default async function ({ }); return; } - if (!(await checkServiceExists(id, manager, res))) { - return; - } - if (!checkServicePending(id, res)) { - return; - } - if (manager.isRunning(id)) { - res - .status(409) - .json({ status: 409, message: "Service is already running." }); - return; - } - consumeEnginePowerAction(() => manager.resumeService(id)); + await manager.resumeService(id); res.status(200).json({ status: 200, - message: - "Service resume action successfully registered to be completed in a moment.", + message: "Service resumed.", statusPath: "/v1/service/" + id + "/powerstatus", }); }, diff --git a/src/router/v1/service/sessionsRoute.ts b/src/router/v1/service/sessionsRoute.ts index 8d8a854..d19ea63 100644 --- a/src/router/v1/service/sessionsRoute.ts +++ b/src/router/v1/service/sessionsRoute.ts @@ -1,7 +1,6 @@ import { AppContext } from "@nsm/app"; import { RouterHandler } from "@nsm/router"; import { checkServiceExists } from "@nsm/router/util/preconditions"; -import { ListSessionsArgs } from "@nsm/database"; export default async function (ctx: AppContext): Promise { return { @@ -17,24 +16,23 @@ export default async function (ctx: AppContext): Promise { return; } - const args: ListSessionsArgs = { - filter: { - serviceId: id, - }, - sort: { - by: "startedAt", - direction: "desc", - }, - page: { - index: pageIndex, - size: pageSize, - }, - }; - const sessionIds = await ctx.sessionManager - .listSessions(args) - .then((sessions) => sessions.map((session) => session.id)); - - res.status(200).json({ sessions: sessionIds }); + res.status(200).json({ + sessions: await ctx.sessionManager + .listSessions({ + filter: { + serviceId: id, + }, + sort: { + by: "startedAt", + direction: "desc", + }, + page: { + index: pageIndex, + size: pageSize, + }, + }) + .then((sessions) => sessions.map((session) => session.id)) + }); }, }, }; diff --git a/src/router/v1/service/stopRoute.ts b/src/router/v1/service/stopRoute.ts index f2de0fd..9857f0c 100644 --- a/src/router/v1/service/stopRoute.ts +++ b/src/router/v1/service/stopRoute.ts @@ -1,10 +1,5 @@ import { AppContext } from "@nsm/app"; import { RouterHandler } from "../../index"; -import { - checkServiceExists, - checkServicePending, -} from "@nsm/router/util/preconditions"; -import { consumeEnginePowerAction } from "@nsm/helpers"; export default async function ({ manager, @@ -24,25 +19,12 @@ export default async function ({ }); return; } - if (!(await checkServiceExists(id, manager, res))) { - return; - } - if (!isForce && !checkServicePending(id, res)) { - return; - } - if (!manager.isRunning(id)) { - res - .status(409) - .json({ status: 400, message: "Service is not running." }); - return; - } - consumeEnginePowerAction(async () => manager.stopService(id, req.query.force === "true")); + await manager.stopService(id, isForce); res.status(200).json({ status: 200, - message: - "Service stop action successfully registered to be completed in a moment.", + message: "Service stop called.", statusPath: "/v1/service/" + id + "/powerstatus", }); }, diff --git a/src/router/v1/session/sessionLogsRoute.ts b/src/router/v1/session/sessionLogsRoute.ts index 423294e..624c248 100644 --- a/src/router/v1/session/sessionLogsRoute.ts +++ b/src/router/v1/session/sessionLogsRoute.ts @@ -1,6 +1,5 @@ import { AppContext } from "@nsm/app"; import { RouterHandler } from "@nsm/router"; -import { ListRecordsArgs } from "@nsm/database"; export default async function (ctx: AppContext): Promise { return { @@ -21,19 +20,20 @@ export default async function (ctx: AppContext): Promise { } : undefined; - const args: ListRecordsArgs = { - filter: { - sessionId: id, - }, - sort: { - by: "timestamp", - direction: "asc", - }, - page, - }; - const logs = await ctx.sessionManager.listSessionLogs(args); - - res.status(200).json({ logs }); + res + .status(200) + .json({ + logs: await ctx.sessionManager.listSessionLogs({ + filter: { + sessionId: id, + }, + sort: { + by: "timestamp", + direction: "asc", + }, + page, + }) + }); }, }, }; diff --git a/src/util/promises.ts b/src/util/promises.ts index 4b75219..e892b19 100644 --- a/src/util/promises.ts +++ b/src/util/promises.ts @@ -1,3 +1,9 @@ +export class AsyncTask { + constructor( + public readonly promise: Promise, + ) {} +} + export async function resolveSequentially(...funcs: any[]) { for (const func of funcs) { if (typeof func == "function") { From e377fd71160ff2ba7b00e2ede46fc69ca802923f Mon Sep 17 00:00:00 2001 From: ZorTik Date: Fri, 12 Jun 2026 15:27:07 +0200 Subject: [PATCH 19/53] fix: express version --- addons/example_addon/libraries.txt | 2 +- package.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/addons/example_addon/libraries.txt b/addons/example_addon/libraries.txt index 828d07c..722f54a 100644 --- a/addons/example_addon/libraries.txt +++ b/addons/example_addon/libraries.txt @@ -1 +1 @@ -express=4.18.2 \ No newline at end of file +express=5.2.1 \ No newline at end of file diff --git a/package.json b/package.json index 4164d96..7e4e146 100644 --- a/package.json +++ b/package.json @@ -57,7 +57,7 @@ "dockerode": "^4.0.2", "dotenv": "^16.4.5", "env-paths": "^3.0.0", - "express": "^4.18.2", + "express": "^5.2.1", "express-fileupload": "^1.5.0", "folder-hash": "^4.1.1", "ignore": "^5.3.1", From fde8779253ca4ebbe40f7388bd018e238640fbf6 Mon Sep 17 00:00:00 2001 From: ZorTik Date: Fri, 12 Jun 2026 17:29:26 +0200 Subject: [PATCH 20/53] fix: global error handling --- src/engine/error.ts | 6 +++--- src/router/middlewares/catchKnownErrors.ts | 6 +++++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/engine/error.ts b/src/engine/error.ts index 7ab28b3..d285709 100644 --- a/src/engine/error.ts +++ b/src/engine/error.ts @@ -23,7 +23,7 @@ export class ServiceNotFoundError extends KnownError { constructor( public readonly serviceId: string ) { - super(404, `Service with ID ${serviceId} not found.`); + super(404, `Service not found.`); } } @@ -31,7 +31,7 @@ export class ServiceNotRunningError extends KnownError { constructor( public readonly serviceId: string ) { - super(409, `Service with ID ${serviceId} is not running.`); + super(409, `Service is not running.`); } } @@ -39,7 +39,7 @@ export class ServiceAlreadyRunningError extends KnownError { constructor( public readonly serviceId: string ) { - super(409, `Service with ID ${serviceId} is already running.`); + super(409, `Service is already running.`); } } diff --git a/src/router/middlewares/catchKnownErrors.ts b/src/router/middlewares/catchKnownErrors.ts index ea6c8f3..bc0eb0f 100644 --- a/src/router/middlewares/catchKnownErrors.ts +++ b/src/router/middlewares/catchKnownErrors.ts @@ -5,7 +5,11 @@ import {KnownError} from "@nsm/engine/error"; * Middleware to catch known errors and respond properly. */ export const catchKnownErrors = (): express.ErrorRequestHandler => { - return (err, _, res) => { + return (err, _, res, next) => { + if (res.headersSent) { + return next(err); + } + let status = 500; let message = "Internal Server Error"; if (err instanceof KnownError) { From cee507658e453be0e7cea0e49be5aa2041750a92 Mon Sep 17 00:00:00 2001 From: ZorTik Date: Fri, 12 Jun 2026 17:44:11 +0200 Subject: [PATCH 21/53] fix: reboot endpoint --- src/router/v1/service/rebootRoute.ts | 29 ++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/src/router/v1/service/rebootRoute.ts b/src/router/v1/service/rebootRoute.ts index eac1cf5..78628e9 100644 --- a/src/router/v1/service/rebootRoute.ts +++ b/src/router/v1/service/rebootRoute.ts @@ -1,5 +1,6 @@ import { AppContext } from "@nsm/app"; import { RouterHandler } from "../../index"; +import {KnownError, ServiceNotRunningError} from "@nsm/engine/error"; export default async function ({ manager, @@ -20,8 +21,32 @@ export default async function ({ return; } - const task = await manager.stopService(id, isForce); - task.promise.then(() => manager.resumeService(id)); + let promise: Promise; + try { + const task = await manager.stopService(id, isForce); + promise = task.promise; + } catch (e) { + if (e instanceof ServiceNotRunningError) { + // not running, just start it + promise = Promise.resolve(); + } else { + throw e; + } + } + promise.then(async () => { + try { + const task = await manager.resumeService(id); + + await task.promise; + } catch (e) { + // just log + if (e instanceof KnownError) { + console.error("Error while resuming service after reboot ", e.message); + } else { + console.error("Error while resuming service after reboot", e); + } + } + }); res.status(200).json({ status: 200, From 88ecf5298c9ef865bd229b63ed15df5947ec2cc4 Mon Sep 17 00:00:00 2001 From: ZorTik Date: Sat, 13 Jun 2026 01:35:51 +0200 Subject: [PATCH 22/53] feat: fixes --- src/engine/asyncp.ts | 4 +- src/engine/error.ts | 9 +++++ src/engine/manager.ts | 90 ++++++++++++++++++++++++------------------- 3 files changed, 62 insertions(+), 41 deletions(-) diff --git a/src/engine/asyncp.ts b/src/engine/asyncp.ts index 857174b..e745c7e 100644 --- a/src/engine/asyncp.ts +++ b/src/engine/asyncp.ts @@ -1,3 +1,5 @@ +import {ServicePendingActionError} from "@nsm/engine/error"; + export type UnlockObserver = (id: string, status?: string, err?: any) => void; const statuses = {}; @@ -75,7 +77,7 @@ export function getActionType(id: string): string | undefined { export function reqNotPending(id: string) { if (stopping == false && isServicePending(id)) { - throw new Error("Service is pending another action."); + throw new ServicePendingActionError(id, getActionType(id)); } } diff --git a/src/engine/error.ts b/src/engine/error.ts index d285709..36634c7 100644 --- a/src/engine/error.ts +++ b/src/engine/error.ts @@ -49,6 +49,15 @@ export class ServiceWasNeverActiveError extends KnownError { } } +export class ServicePendingActionError extends KnownError { + constructor( + public readonly serviceId: string, + public readonly pendingAction: string + ) { + super(409, `Service has a pending action '${pendingAction}'.`); + } +} + export class TemplateNotFoundError extends KnownError { constructor( public readonly templateId: string diff --git a/src/engine/manager.ts b/src/engine/manager.ts index e479561..4143573 100644 --- a/src/engine/manager.ts +++ b/src/engine/manager.ts @@ -19,6 +19,7 @@ import crypto from "crypto"; import { randomPort as retrieveRandomPort } from "@nsm/util/port"; import { Database, PermaModel } from "../database"; import { + getActionType, isServicePending, lockBusyAction, reqNotPending, unlockBusyAction, @@ -48,7 +49,7 @@ import { InvalidMetaError, ServiceAlreadyRunningError, ServiceNotFoundError, - ServiceNotRunningError, ServiceWasNeverActiveError, TemplateNotFoundError + ServiceNotRunningError, ServicePendingActionError, ServiceWasNeverActiveError, TemplateNotFoundError } from "@nsm/engine/error"; export type Options = { @@ -322,8 +323,6 @@ export type ServiceManager = ServiceManagerEventBus & { initEngineForcibly(): Promise; // -} & { - whenUnlocked: typeof whenUnlocked; }; type RunningService = { @@ -654,36 +653,52 @@ export async function stopService(id: string, force?: boolean) { await reqExists(id); const { internalSession } = reqRunning(id); - try { - let awaitingPromise: Promise; - if (force) { - await engine.kill(internalSession.containerId, metaStorageForService(id)); - // resolves immediately on kill - awaitingPromise = Promise.resolve(); - } else { - // lock only on soft stop, to allow hard-killing if any issues happen during stopping - const unlock = lockBusyAction(id, "stop"); - awaitingPromise = new Promise((resolve) => { - // wait for stop - // this is really not necessary because any busy action is unlocked on stop, but - // just in case and for the promise - on("stop", ({ id: stoppedId, error }) => { - if (stoppedId !== id) { - // This call is not for me - return false; - } - if (isServicePending(id)) { - unlock(error); - } - resolve(); - return true; - }); - }) + const callEngine = async (task: () => Promise) => { + try { + await task(); + } catch (e) { + currentContext.logger.error(e); + callManagerEvent("stop", { id, error: e }); + } + } + + let awaitingPromise: Promise; + if (force) { + const pendingAction = getActionType(id); + if (pendingAction && getActionType(id) !== "stop") { + // the service is locked and not stopping, the force stop can't be allowed + throw new ServicePendingActionError(id, pendingAction); + } + + await callEngine(async () => engine.kill(internalSession.containerId, metaStorageForService(id))); + // resolves immediately on kill + awaitingPromise = Promise.resolve(); + } else { + // lock only on soft stop, to allow hard-killing if any issues happen during stopping + const unlock = lockBusyAction(id, "stop"); + awaitingPromise = new Promise((resolve) => { + // wait for stop + // this is really not necessary because any busy action is unlocked on stop, but + // just in case and for the promise + on("stop", ({ id: stoppedId, error }) => { + if (stoppedId !== id) { + // This call is not for me + return false; + } - // TODO: stop strategy - const service = await getService(id); - const stopCmd = service.meta?.stopCmd; + if (isServicePending(id)) { + unlock(error); + } + resolve(); + return true; + }); + }); + + // TODO: stop strategy + const service = await getService(id); + const stopCmd = service.meta?.stopCmd; + await callEngine(async () => { if (stopCmd) { // send stop cmd if set await engine.cmd(internalSession.containerId, stopCmd); @@ -691,14 +706,11 @@ export async function stopService(id: string, force?: boolean) { // send stop signal await engine.stop(internalSession.containerId); } - } - - return new AsyncTask(awaitingPromise); - } catch (e) { - currentContext.logger.error(e); - - callManagerEvent("stop", { id, error: e }); + }); } + awaitingPromise = awaitingPromise.then(() => waitForStopped(id)); + + return new AsyncTask(awaitingPromise); } export async function sendStopSignal(id: string) { @@ -950,8 +962,6 @@ export function on( evtHandlers.get(evt).push(h); } -export { whenUnlocked }; - function clearRunningServiceIfExists(id: string) { const service = getRunningService(id); From 91faed00c4f610ffc787d6dc25fb0f7587fc57cf Mon Sep 17 00:00:00 2001 From: ZorTik Date: Sat, 13 Jun 2026 01:40:22 +0200 Subject: [PATCH 23/53] feat: STOPPING state --- openapi.yml | 3 ++- src/engine/manager.ts | 7 ++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/openapi.yml b/openapi.yml index 7cb90ff..2bc5b0b 100644 --- a/openapi.yml +++ b/openapi.yml @@ -142,10 +142,11 @@ components: description: "The template ID used to create the service" state: type: string - description: "The current state of the service. One of: 'RUNNING', 'BUILDING', 'STOPPED'." + description: "The current state of the service. One of: 'RUNNING', 'BUILDING', 'STOPPING', 'STOPPED'." enum: - "RUNNING" - "BUILDING" + - "STOPPING" - "STOPPED" port: type: "integer" diff --git a/src/engine/manager.ts b/src/engine/manager.ts index 4143573..e5b33ff 100644 --- a/src/engine/manager.ts +++ b/src/engine/manager.ts @@ -345,7 +345,7 @@ export type ServiceInfo = PermaModel & { internalSession?: InternalSession; }; -export type State = "RUNNING" | "BUILDING" | "STOPPED"; +export type State = "BUILDING" | "RUNNING" | "STOPPING" | "STOPPED"; export let engine: ServiceEngineI = undefined; export let nodeId: string; @@ -1041,6 +1041,11 @@ function setServiceState(id: string, state: State) { * @returns The state of the service */ function getServiceState(id: string) { + if (getActionType(id) === "stop") { + // service has stop locked, so is stopping + return "STOPPING"; + } + return startedStates.get(id) ?? "STOPPED"; } From aaf97cad17467986a42d0d9c493428c43909a1e2 Mon Sep 17 00:00:00 2001 From: ZorTik Date: Sat, 13 Jun 2026 02:58:21 +0200 Subject: [PATCH 24/53] refactor: ServiceManager --- src/engine/manager.ts | 91 +++++++++++++++++++++---------------------- 1 file changed, 45 insertions(+), 46 deletions(-) diff --git a/src/engine/manager.ts b/src/engine/manager.ts index e5b33ff..54f9ef3 100644 --- a/src/engine/manager.ts +++ b/src/engine/manager.ts @@ -43,7 +43,6 @@ import { ServiceSession, init as initSessionEngine, } from "@nsm/engine/session"; -import { AppConfig } from "@nsm/config"; import { InternalError, InvalidMetaError, @@ -373,11 +372,11 @@ const evtHandlers: Map[]> = new Map(); }; }); -export async function init( - db_: Database, - appConfig_: AppConfig, - logger: winston.Logger, -) { +export const init: ServiceManager["init"] = async ( + db_, + appConfig_, + logger, +) => { const nodeId_ = appConfig_.getNodeId(); logger.info(`Initializing service manager for node ${nodeId_}...`); @@ -406,7 +405,7 @@ export async function init( logger.info(`Using engine: ${engine.name}`); } -async function deleteGarbage(logger: winston.Logger) { +const deleteGarbage = async (logger: winston.Logger) => { // TODO: delete containers that are not running and remained from last session } @@ -416,7 +415,7 @@ async function deleteGarbage(logger: winston.Logger) { * * @param logger The logger to use */ -async function reattachStaleContainers(logger: winston.Logger) { +const reattachStaleContainers = async (logger: winston.Logger) => { const running = await engine .listRunning(Filters.node(nodeId)) .then((containerIds) => @@ -466,9 +465,9 @@ async function reattachStaleContainers(logger: winston.Logger) { await new Promise((resolve) => whenUnlockedAll(() => resolve(null))); } -export async function expandEngine( +export const expandEngine: ServiceManager["expandEngine"] = async ( exp?: T, -): Promise { +): Promise => { if (exp) { if (!engine && (!currentContext || !currentContext.appConfig)) { throw new Error("Engine is not yet loaded and can't be loaded forcibly!"); @@ -492,7 +491,7 @@ export async function expandEngine( return engine as any; } -export async function createService(template: string, options: Options) { +export const createService: ServiceManager["createService"] = async (template, options) => { const { ram, cpu, disk, ports, env, network } = options; const serviceSettings = reqTemplate(template).settings; @@ -543,7 +542,7 @@ export async function createService(template: string, options: Options) { } } -export async function resumeService(id: string) { +export const resumeService: ServiceManager["resumeService"] = async (id) => { reqNotRunning(id); let { template, options, env, network, port } = await reqExists(id); @@ -649,7 +648,7 @@ export async function resumeService(id: string) { ); } -export async function stopService(id: string, force?: boolean) { +export const stopService: ServiceManager["stopService"] = async (id, force) => { await reqExists(id); const { internalSession } = reqRunning(id); @@ -713,7 +712,7 @@ export async function stopService(id: string, force?: boolean) { return new AsyncTask(awaitingPromise); } -export async function sendStopSignal(id: string) { +export const sendStopSignal: ServiceManager["sendStopSignal"] = async (id) => { const perma = await reqExists(id); const { internalSession } = reqRunning(id); @@ -726,7 +725,7 @@ export async function sendStopSignal(id: string) { return true; } -export async function deleteService(id: string) { +export const deleteService: ServiceManager["deleteService"] = async (id) => { try { await stopService(id, true); } catch (e) { @@ -770,7 +769,7 @@ export async function deleteService(id: string) { whenUnlocked(id, unlockHandler); } -export async function updateOptions(id: string, options: Options) { +export const updateOptions: ServiceManager["updateOptions"] = async (id, options) => { reqNotPending(id); const perma = await db.permaRepository.getPerma(id); const data: PermaModel = { @@ -788,14 +787,14 @@ export async function updateOptions(id: string, options: Options) { return db.permaRepository.savePerma(data); } -export function getTemplate(id: string) { +export const getTemplate: ServiceManager["getTemplate"] = (id) => { return loadTemplate(id); } -export async function getService( - from: string, - options?: { includeSession?: boolean; otherNodes?: boolean }, -): ReturnType { +export const getService: ServiceManager["getService"] = async ( + from, + options, +): ReturnType => { const data = typeof from === "string" ? await db.permaRepository.getPerma(from) : from; if (data && (data.nodeId == nodeId || options?.otherNodes === true)) { @@ -823,11 +822,11 @@ export async function getService( } } -export function getLastPowerError(id: string) { +export const getLastPowerError: ServiceManager["getLastPowerError"] = (id) => { return errors[id]; } -export async function getLastSession(id: string) { +export const getLastSession: ServiceManager["getLastSession"] = async (id) => { await reqExists(id); const runningService = getRunningService(id); @@ -849,18 +848,18 @@ export async function getLastSession(id: string) { throw new ServiceWasNeverActiveError(); } -export async function listServices(options: ListServicesOptions) { +export const listServices: ServiceManager["listServices"] = async (options) => { const meta = options.filter?.meta; return db.permaRepository .listPerma(nodeId, options.page, options.pageSize, meta) .then((list) => list.map((d) => d.serviceId)); } -export async function listTemplates(): Promise { +export const listTemplates: ServiceManager["listTemplates"] = async () => { return getAllTemplates().map((template) => template.id); } -export async function stopRunning() { +export const stopRunning: ServiceManager["stopRunning"] = async () => { const tasks = started.map( ({ id }) => new Promise((resolve) => { @@ -877,7 +876,7 @@ export async function stopRunning() { await Promise.all(tasks); } -export async function killRunning() { +export const killRunning: ServiceManager["killRunning"] = async () => { await Promise.all( started.map( async ({ id }) => stopService(id, true).catch((e) => currentContext.logger.error(e)) @@ -885,13 +884,13 @@ export async function killRunning() { ) } -export async function waitForBusyAction(id: string) { +export const waitForBusyAction: ServiceManager["waitForBusyAction"] = async (id: string) => { return new Promise((resolve, reject) => { whenUnlocked(id, (_, __, err) => (err ? reject(err) : resolve(null))); }); } -export async function waitForStopped(id: string) { +export const waitForStopped: ServiceManager["waitForStopped"] = async (id: string) => { if (!isRunning(id)) { // service not running, so we continue immediately return; @@ -913,15 +912,15 @@ export async function waitForStopped(id: string) { }); } -export function isRunning(id: string) { +export const isRunning: ServiceManager["isRunning"] = (id: string) => { return getRunningService(id) != undefined; } -export function getRunningService(id: string) { +export const getRunningService: ServiceManager["getRunningService"] = (id: string) => { return started.find((service) => service.id === id); } -function metaStorageForService(id: string): MetaStorage { +const metaStorageForService = (id: string): MetaStorage => { // service id return { set: async (key, value) => { @@ -935,7 +934,7 @@ function metaStorageForService(id: string): MetaStorage { }; } -export async function initEngineForcibly() { +export const initEngineForcibly = async () => { if (engine) { throw new Error("Engine is already loaded."); } @@ -948,21 +947,21 @@ export async function initEngineForcibly() { engine.cast = () => engine as any; } -export function getRunningServices() { +export const getRunningServices: ServiceManager["getRunningServices"] = () => { return [...started]; } -export function on( +export const on: ServiceManager["on"] = ( evt: T, h: EventHandler, -) { +) => { if (!evtHandlers.has(evt)) { evtHandlers.set(evt, []); } evtHandlers.get(evt).push(h); } -function clearRunningServiceIfExists(id: string) { +const clearRunningServiceIfExists = (id: string) => { const service = getRunningService(id); if (service) { @@ -970,10 +969,10 @@ function clearRunningServiceIfExists(id: string) { } } -function callManagerEvent( +const callManagerEvent = ( e: T, event: ServiceManagerEvents[T], -) { +) => { if (!evtHandlers.has(e)) { return; } @@ -992,7 +991,7 @@ function callManagerEvent( * * @param session The session for whom to create the session. */ -function buildRunListener(session: ActiveServiceSession): RunListener { +const buildRunListener = (session: ActiveServiceSession): RunListener => { const { serviceId } = session; // The internal run listener of this manager @@ -1025,7 +1024,7 @@ function buildRunListener(session: ActiveServiceSession): RunListener { ]); } -function setServiceState(id: string, state: State) { +const setServiceState = (id: string, state: State) => { startedStates.set(id, state); callManagerEvent("statechange", { @@ -1040,7 +1039,7 @@ function setServiceState(id: string, state: State) { * @param id The id of the service. * @returns The state of the service */ -function getServiceState(id: string) { +const getServiceState = (id: string) => { if (getActionType(id) === "stop") { // service has stop locked, so is stopping return "STOPPING"; @@ -1051,7 +1050,7 @@ function getServiceState(id: string) { // --------------------------------------------------------------------------------------- -async function reqExists(id: string) { +const reqExists = async (id: string) => { const perma_ = await db.permaRepository.getPerma(id); if (!perma_) { // service does not exist @@ -1061,7 +1060,7 @@ async function reqExists(id: string) { return perma_; } -function reqRunning(id: string) { +const reqRunning = (id: string) => { const session = getRunningService(id); if (!session) { throw new ServiceNotRunningError(id); @@ -1070,13 +1069,13 @@ function reqRunning(id: string) { return session; } -function reqNotRunning(id: string) { +const reqNotRunning = (id: string) => { if (isRunning(id)) { throw new ServiceAlreadyRunningError(id); } } -function reqTemplate(id: string) { +const reqTemplate = (id: string) => { const template = getTemplate(id); if (!template) { throw new TemplateNotFoundError(id); From 7bc4ebf193ec16d91ac02b78bced27e82c97007a Mon Sep 17 00:00:00 2001 From: ZorTik Date: Sat, 13 Jun 2026 14:28:43 +0200 Subject: [PATCH 25/53] refactor: ServiceManager --- src/engine/manager.ts | 155 +++++++++++++++++++++++------------------- src/engine/session.ts | 25 +++++++ 2 files changed, 111 insertions(+), 69 deletions(-) diff --git a/src/engine/manager.ts b/src/engine/manager.ts index 54f9ef3..d57b42e 100644 --- a/src/engine/manager.ts +++ b/src/engine/manager.ts @@ -130,10 +130,15 @@ type ServiceStateChangeEvent = ServiceEvent & { state: State; } +type ServiceEngineErrorEvent = ServiceEvent & { + error: Error; +} + type ServiceManagerEvents = { resume: ServiceEvent; stop: ServiceEvent; statechange: ServiceStateChangeEvent; + engine_err: ServiceEngineErrorEvent; }; /** @@ -350,8 +355,9 @@ export let engine: ServiceEngineI = undefined; export let nodeId: string; let db: Database; +let logger: winston.Logger; -// Save errors somewhere else? +// TODO: Save errors somewhere else? // Could it be a memory leak if there are tons of them?? const errors = {}; // Service IDs that are currently running @@ -365,7 +371,7 @@ const evtHandlers: Map[]> = new Map(); // Emit services change within those methods if (isDebug()) { - currentContext.logger.debug("Service registry changed"); + logger.debug("Service registry changed"); } return result; @@ -375,13 +381,13 @@ const evtHandlers: Map[]> = new Map(); export const init: ServiceManager["init"] = async ( db_, appConfig_, - logger, + logger_, ) => { - const nodeId_ = appConfig_.getNodeId(); + db = db_; + logger = logger_; + const nodeId_ = appConfig_.getNodeId(); logger.info(`Initializing service manager for node ${nodeId_}...`); - - db = db_; if (!engine) { // Init only if it has not already been force-initialized await initEngineForcibly(); @@ -394,11 +400,13 @@ export const init: ServiceManager["init"] = async ( templateDirWatcher, db_, appConfig_, - currentContext.logger, + logger, ); initSessionEngine(db_); - watchTemplateDirChanges(currentContext.logger); + watchTemplateDirChanges(logger); + gatherEngineErrors(); + registerLoggingEventHandlers(); await deleteGarbage(logger); await reattachStaleContainers(logger); @@ -465,6 +473,32 @@ const reattachStaleContainers = async (logger: winston.Logger) => { await new Promise((resolve) => whenUnlockedAll(() => resolve(null))); } +const gatherEngineErrors = () => { + on("engine_err", (event) => { + errors[event.id] = event.error; + }); +} + +/** + * Registers event handlers for logging in debug mode. + */ +const registerLoggingEventHandlers = () => { + const notifyIfSuccess = ( + messageProvider: (serviceId: string) => string + ): EventHandler => { + return ({ id, error }) => { + if (error) { + return; + } + + logger.debug(messageProvider(id)); + } + } + + on("resume", notifyIfSuccess((id) => `Service ${id} resumed`)); + on("stop", notifyIfSuccess((id) => `Service ${id} stopped`)); +} + export const expandEngine: ServiceManager["expandEngine"] = async ( exp?: T, ): Promise => { @@ -523,23 +557,12 @@ export const createService: ServiceManager["createService"] = async (template, o env: env ?? {}, network, }; - let err: any; // Save permanent info if (!(await db.permaRepository.savePerma(perma))) { - err = new InternalError("Failed to save perma info to database"); + throw new InternalError("Failed to save perma info to database"); } - if (err) { - // Save to be later retrieved - errors[serviceId] = err; - currentContext.logger.error(err.message); - } - - if (err) { - throw err; - } else { - return serviceId; - } + return serviceId; } export const resumeService: ServiceManager["resumeService"] = async (id) => { @@ -588,42 +611,33 @@ export const resumeService: ServiceManager["resumeService"] = async (id) => { const { SERVICE_ID, SERVICE_PORT, SERVICE_PORTS, ...buildEnv } = runOptions.env; - return new AsyncTask( - // TODO: tato funkce má poslední parametr messageListener, vymyslet jak sem propagovat message listener z session - processImage(perma.imageId, template, buildEnv) - .then(async (image) => { - // If the image was changed by processing (e.g. it was built or rebuilt), update the image id in database - if (image != perma.imageId) { + const updateImageIfChanged = async (image: string) => { + // If the image was changed by processing (e.g. it was built or rebuilt), update the image id in database + if (image != perma.imageId) { - // Update image in database if it was changed by processing - perma.imageId = image; - await db.permaRepository.savePerma(perma); - } + // Update image in database if it was changed by processing + perma.imageId = image; + await db.permaRepository.savePerma(perma); + } - return image; - }) + return image; + } + + return new AsyncTask( + // TODO: logovat někam message z image processingu pomocí posledního parametru + processImage(perma.imageId, template, buildEnv) + .then(updateImageIfChanged) .then(async (image) => { - let session: ActiveServiceSession | undefined; - let containerId: string | undefined; + const session = await beginServiceSession(id); + // Run the container with the built image and save the container id for later use. try { - // Run the container with the built image and save the container id for later use. - if (image) { - session = await beginServiceSession(id); - containerId = await engine.run( - image, - id, - runOptions, - meta, - buildRunListener(session), - ); - } - } catch (e) { - currentContext.logger.error("Failed to run container for service " + id); - currentContext.logger.error(e); - } - - let success: boolean = false; - if (containerId) { + const containerId = await engine.run( + image, + id, + runOptions, + meta, + buildRunListener(session), + ); const runningService: RunningService = { id, session, @@ -632,16 +646,11 @@ export const resumeService: ServiceManager["resumeService"] = async (id) => { }, }; started.push(runningService); - success = true; - } - if (success == true) { - currentContext.logger.debug("Service " + id + " resumed"); callManagerEvent("resume", { id }); - } else { - errors[id] = new Error("Failed to resume service"); - clearRunningServiceIfExists(id); - callManagerEvent("resume", { id, error: errors[id] }); + } catch (e) { + callManagerEvent("resume", { id, error: e }); + callServiceEngineError(id, e); } }) .finally(() => unlock()) @@ -657,7 +666,7 @@ export const stopService: ServiceManager["stopService"] = async (id, force) => { try { await task(); } catch (e) { - currentContext.logger.error(e); + logger.error(e); callManagerEvent("stop", { id, error: e }); } } @@ -713,10 +722,10 @@ export const stopService: ServiceManager["stopService"] = async (id, force) => { } export const sendStopSignal: ServiceManager["sendStopSignal"] = async (id) => { - const perma = await reqExists(id); + const { meta } = await reqExists(id); const { internalSession } = reqRunning(id); - const stopCmd = perma.meta?.stopCmd; + const stopCmd = meta?.stopCmd; if (!stopCmd) { throw new InvalidMetaError("Service does not have stop command set."); } @@ -762,7 +771,7 @@ export const deleteService: ServiceManager["deleteService"] = async (id) => { ), ) .then(() => { - currentContext.logger.debug(`Service ${id} deleted`); + logger.debug(`Service ${id} deleted`); }); }; @@ -865,7 +874,7 @@ export const stopRunning: ServiceManager["stopRunning"] = async () => { new Promise((resolve) => { whenUnlocked(id, () => { stopService(id) - .catch((e) => currentContext.logger.error(e)) + .catch((e) => logger.error(e)) .then(() => { whenUnlocked(id, () => resolve(null)); }); @@ -879,7 +888,7 @@ export const stopRunning: ServiceManager["stopRunning"] = async () => { export const killRunning: ServiceManager["killRunning"] = async () => { await Promise.all( started.map( - async ({ id }) => stopService(id, true).catch((e) => currentContext.logger.error(e)) + async ({ id }) => stopService(id, true).catch((e) => logger.error(e)) ) ) } @@ -985,6 +994,16 @@ const callManagerEvent = ( evtHandlers.set(e, newArray); } +/** + * Notifies about an error that happened during internal engine calling. + * + * @param id The service ID for which the error happened + * @param error The error that happened + */ +const callServiceEngineError = (id: string, error: Error) => { + callManagerEvent("engine_err", { id, error }); +} + /** * Collects all relevant run listeners and builds a composite one * to be used directly when running/attaching service container. @@ -1012,8 +1031,6 @@ const buildRunListener = (session: ActiveServiceSession): RunListener => { } callManagerEvent("stop", { id: serviceId }); - - currentContext.logger.debug("Service " + serviceId + " stopped"); }, }; // Combine collected listeners diff --git a/src/engine/session.ts b/src/engine/session.ts index 44b8a55..6fa5fbb 100644 --- a/src/engine/session.ts +++ b/src/engine/session.ts @@ -9,13 +9,38 @@ import { } from "@nsm/database"; export interface SessionManager { + /** + * Initializes the session manager with the given database instance. + * + * @param db The database instance to use for storing session and log data. + * This method must be called before using any other methods of the session manager. + */ init(db: Database): void; + /** + * Begins a new service session for the given service ID. + * + * @param serviceId The ID of the service for which to begin a session. + * @return An object representing the active service session, including a run listener for handling session events. + */ beginServiceSession(serviceId: string): Promise; + /** + * Lists service sessions. + * + * @param args The arguments for listing sessions + * @return A list of service sessions matching the given criteria, or undefined if no sessions were found. + */ listSessions( args: ListSessionsArgs, ): Promise; + + /** + * Lists log records for a service session. + * + * @param args The arguments for listing log records + * @return A list of log records matching the given criteria, or undefined if no records were found. + */ listSessionLogs( args: ListRecordsArgs, ): Promise; From a7e29d8e92664a791caa637ad771bfee2fba69fc Mon Sep 17 00:00:00 2001 From: ZorTik Date: Sun, 14 Jun 2026 14:45:37 +0200 Subject: [PATCH 26/53] feat: remove stop cmd route --- src/engine/manager.ts | 22 ------------ src/router/v1/index.ts | 2 -- src/router/v1/service/stopCmdRoute.ts | 51 --------------------------- 3 files changed, 75 deletions(-) delete mode 100644 src/router/v1/service/stopCmdRoute.ts diff --git a/src/engine/manager.ts b/src/engine/manager.ts index d57b42e..461074f 100644 --- a/src/engine/manager.ts +++ b/src/engine/manager.ts @@ -219,15 +219,6 @@ export type ServiceManager = ServiceManagerEventBus & { */ stopService(id: string, force?: boolean): Promise>; - /** - * Send pre-configured stop signal to the service. - * - * @param id The service ID - * @returns Whether the signal has been sent - * @throws InvalidMetaError if the service does not have the required meta for stop signal (e.g. stop command) - */ - sendStopSignal(id: string): Promise; - /** * Delete a service. * @@ -721,19 +712,6 @@ export const stopService: ServiceManager["stopService"] = async (id, force) => { return new AsyncTask(awaitingPromise); } -export const sendStopSignal: ServiceManager["sendStopSignal"] = async (id) => { - const { meta } = await reqExists(id); - const { internalSession } = reqRunning(id); - - const stopCmd = meta?.stopCmd; - if (!stopCmd) { - throw new InvalidMetaError("Service does not have stop command set."); - } - - await engine.cmd(internalSession.containerId, stopCmd); - return true; -} - export const deleteService: ServiceManager["deleteService"] = async (id) => { try { await stopService(id, true); diff --git a/src/router/v1/index.ts b/src/router/v1/index.ts index 2e657bd..a5352fd 100644 --- a/src/router/v1/index.ts +++ b/src/router/v1/index.ts @@ -7,7 +7,6 @@ import stopRoute from "./service/stopRoute"; import createRoute from "./service/createRoute"; import rebootRoute from "./service/rebootRoute"; import powerStatusRoute from "./service/powerStatusRoute"; -import stopCmdRoute from "@nsm/router/v1/service/stopCmdRoute"; import optionsRoute from "@nsm/router/v1/service/optionsRoute"; import sessionsRoute from "@nsm/router/v1/service/sessionsRoute"; import sessionLogsRoute from "@nsm/router/v1/session/sessionLogsRoute"; @@ -21,7 +20,6 @@ export default [ deleteRoute, resumeRoute, rebootRoute, - stopCmdRoute, stopRoute, powerStatusRoute, optionsRoute, diff --git a/src/router/v1/service/stopCmdRoute.ts b/src/router/v1/service/stopCmdRoute.ts deleted file mode 100644 index f3469d0..0000000 --- a/src/router/v1/service/stopCmdRoute.ts +++ /dev/null @@ -1,51 +0,0 @@ -import { AppContext } from "@nsm/app"; -import { RouterHandler } from "@nsm/router"; -import { isServicePending } from "@nsm/engine/asyncp"; -import { handleErr } from "@nsm/util/routes"; -import { checkServicePending } from "@nsm/router/util/preconditions"; - -export default async function ({ - manager, -}: AppContext): Promise { // TODO: remove this - return { - url: "/service/:id/stopcmd", - routes: { - post: async (req, res) => { - const id = req.params.id; - if (!id) { - res - .status(400) - .json({ - status: 400, - message: "Required 'id' field not present in the body.", - }); - return; - } - if (!checkServicePending(id, res)) { - return; - } - if (!(await manager.getService(id))) { - res.status(404).json({ status: 404, message: "Service not found." }); - return; - } - try { - const result = await manager.sendStopSignal(id); - if (result) { - res - .status(200) - .json({ status: 200, message: "Service stop signal sent." }); - } else { - res - .status(404) - .json({ - status: 404, - message: "Service not found or unknown error occured.", - }); - } - } catch (e) { - handleErr(e, res); - } - }, - }, - }; -} From 4c999a92b627c69d2f99d5c98907f746fe965700 Mon Sep 17 00:00:00 2001 From: ZorTik Date: Sun, 14 Jun 2026 21:15:29 +0200 Subject: [PATCH 27/53] refactor: 1.2 --- Dockerfile | 4 - addons/example_addon/index.ts | 14 - addons/example_addon/libraries.txt | 1 - installTempDeps.js | 20 - package.json | 4 +- src/addon.ts | 123 --- src/app.ts | 91 +-- src/cleanup.ts | 19 +- src/engine/engine.ts | 15 +- src/engine/facade.ts | 85 ++ src/engine/image.ts | 4 +- src/engine/manager.ts | 930 ++-------------------- src/engine/middle.ts | 30 +- src/engine/runner.ts | 641 +++++++++++++++ src/engine/session.ts | 26 +- src/router/v1/service/createRoute.ts | 6 +- src/router/v1/service/deleteRoute.ts | 4 +- src/router/v1/service/listRoute.ts | 3 +- src/router/v1/service/logsRoute.ts | 2 +- src/router/v1/service/lookupRoute.ts | 9 +- src/router/v1/service/powerStatusRoute.ts | 8 +- src/router/v1/service/rebootRoute.ts | 6 +- src/router/v1/service/resumeRoute.ts | 3 +- src/router/v1/service/stopRoute.ts | 4 +- src/router/v1/status/index.ts | 27 +- tests/api/api.test.ts | 20 +- tests/engine/middle.test.ts | 45 +- 27 files changed, 955 insertions(+), 1189 deletions(-) delete mode 100644 addons/example_addon/index.ts delete mode 100644 addons/example_addon/libraries.txt delete mode 100644 installTempDeps.js delete mode 100644 src/addon.ts create mode 100644 src/engine/facade.ts create mode 100644 src/engine/runner.ts diff --git a/Dockerfile b/Dockerfile index d3c8838..8349399 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,10 +2,6 @@ FROM node:22 WORKDIR /data -# Copy addons before install to install dependencies for addons as well -COPY addons ./addons - -COPY installTempDeps.js ./ COPY package*.json ./ RUN npm install diff --git a/addons/example_addon/index.ts b/addons/example_addon/index.ts deleted file mode 100644 index 3c2be3a..0000000 --- a/addons/example_addon/index.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { Addon } from "@nsm/addon"; -import winston from "winston"; - -async function initAfterLogger(ctx: { logger: winston.Logger }) { - ctx.logger.info("Hello from example addon!"); -} - -export default { - name: "example_addon", - disabled: true, - steps: { - BEFORE_CONFIG: initAfterLogger, - }, -} as Addon; diff --git a/addons/example_addon/libraries.txt b/addons/example_addon/libraries.txt deleted file mode 100644 index 722f54a..0000000 --- a/addons/example_addon/libraries.txt +++ /dev/null @@ -1 +0,0 @@ -express=5.2.1 \ No newline at end of file diff --git a/installTempDeps.js b/installTempDeps.js deleted file mode 100644 index a6afbb1..0000000 --- a/installTempDeps.js +++ /dev/null @@ -1,20 +0,0 @@ -const fs = require("fs"); -const npm = require("npm"); - -console.log("Preinstalling dependencies for build..."); - -npm.load().then(() => { - for (let addon of fs.readdirSync(process.cwd() + "/addons")) { - const libFPath = process.cwd() + "/addons/" + addon + "/libraries.txt"; - if (!fs.existsSync(libFPath)) { - continue; - } - const libs = fs - .readFileSync(libFPath, "utf8") - .split("\n") - .map((lib) => lib.split("=")[0] + "@" + lib.split("=")[1]); - npm.commands.install(libs, (err) => { - console.log(err); - }); - } -}); diff --git a/package.json b/package.json index 7e4e146..dafebe5 100644 --- a/package.json +++ b/package.json @@ -4,9 +4,9 @@ "description": "A new service control engine, built on docker.", "main": "index.js", "scripts": { - "build": "node installTempDeps.js && tsc && tscp", + "build": "tsc && tscp", "migrate": "prisma migrate deploy", - "start": "npm run migrate && cross-env TS_NODE_BASEURL=./dist node -r tsconfig-paths/register dist/index.js", + "start": "npm run migrate && cross-env TS_NODE_BASEURL=./dist node -r tsconfig-paths/register --enable-source-maps dist/index.js", "test": "npm run migrate && jest" }, "keywords": [], diff --git a/src/addon.ts b/src/addon.ts deleted file mode 100644 index 81f8435..0000000 --- a/src/addon.ts +++ /dev/null @@ -1,123 +0,0 @@ -import winston from "winston"; -import { AppContext } from "./app"; -import * as fs from "fs"; -import npm from "npm"; -import * as http from "http"; -import { isDebug } from "./helpers"; -import { createLogger } from "./logger"; - -type FunctionTypes = { - BEFORE_CONFIG: (ctx: { logger: winston.Logger }) => Promise; - BEFORE_DB: (ctx: { logger: winston.Logger; appConfig: any }) => Promise; - BEFORE_ENGINE: (ctx: AppContext) => Promise; - BEFORE_SECURITY: (ctx: AppContext) => Promise; - BEFORE_ROUTES: (ctx: AppContext) => Promise; - BEFORE_SERVER: (ctx: AppContext) => Promise; - BOOT: (ctx: AppContext, srv: http.Server) => Promise; - EXIT: (ctx: AppContext) => Promise; -}; - -export type Moment = keyof FunctionTypes; -export type AddonSteps = { - [key in Moment]: FunctionTypes[key]; -}; -export type Addon = { - name: string; - briefName?: string; - author?: string; - version?: string; - disabled?: boolean; - steps: AddonSteps; -}; - -async function initNpm() { - await npm.load(); - npm.config.set("save", false); - npm.config.set("save-dev", false); -} - -// Installs dependencies written in libraries.txt -async function installLibs( - logger: winston.Logger, - libs: { [key: string]: string }, -) { - const libsArray = Object.keys(libs).map((key) => key + "@" + libs[key]); - logger.info(`Installing ${libsArray.join(", ")}`); - await new Promise((resolve, reject) => { - npm.commands.install(libsArray, (err) => { - if (err) { - reject(err); - } else { - resolve(true); - } - }); - }); -} - -// Load addons -export default async function (logger: winston.Logger) { - // Load NPM client - await initNpm(); - - const addons: Addon[] = []; - // Loop addon dirs - // Directories array - for (const dir of fs - .readdirSync(__dirname + "/../addons") - .map((dir) => __dirname + "/../addons/" + dir) - .filter((file) => fs.existsSync(file + "/index.js"))) { - if (dir.endsWith("example_addon")) { - // Skip default example addon - continue; - } - logger.info(`Loading addon from ${dir}`); - if (fs.existsSync(dir + "/libraries.txt")) { - await installLibs( - logger, - // Libraries mapped - fs - .readFileSync(dir + "/libraries.txt", "utf8") - .split("\n") - .filter((lib) => lib.includes("=")) - .map((lib) => lib.split("=")) - .reduce( - (acc, [name, version]) => { - acc[name] = version.replace("\r", ""); - return acc; - }, - {} as { [key: string]: string }, - ), - ); - } - - const addon = require(dir + "/index.js").default as Addon; - if (!addon.disabled) { - addons.push(addon); - - const { name, author, version } = addon; - - logger.info( - `Loaded addon ${name}${author ? ` by ${author}` : ``}${version ? ` (v${version})` : ``}`, - ); - } - } - return (step: T, ctx: any, ...args: any[]) => { - if (isDebug()) { - logger.info(`Running step ${step}`); - } - addons - .filter((addon) => addon.steps[step]) - .forEach((addon) => { - const f = addon.steps[step]; - // Make temporary duplicate - const ctxAddon = { ...ctx }; - if (ctxAddon.logger) { - // Make custom logger for each addon - ctxAddon.logger = createLogger({ - label: addon.briefName ?? addon.name, - }); - } - f.apply(f, [ctxAddon, ...args]); - }); - }; -} diff --git a/src/app.ts b/src/app.ts index acf7da0..d05601c 100644 --- a/src/app.ts +++ b/src/app.ts @@ -16,12 +16,17 @@ initFileStructure(appConfig); import { Router } from "express"; import { Database } from "@nsm/database"; import { ServiceManager } from "@nsm/engine"; -import loadAddons from "./addon"; import loadAppRoutes from "@nsm/router"; import createDbManager from "@nsm/database"; import loadSecurity from "@nsm/security"; +import createEngine from "@nsm/engine/engine"; +import { init as initImageEngine } from "@nsm/engine/image"; +import * as facade from "@nsm/engine/facade"; import * as manager from "@nsm/engine/manager"; +import * as runner from "@nsm/engine/runner"; import * as sessionManager from "@nsm/engine/session"; +import * as templateManager from "@nsm/engine/template"; +import * as templateDirWatcher from "@nsm/engine/monitoring/templateDirWatcher"; import * as logging from "./logger"; import winston from "winston"; import { Application } from "express-ws"; @@ -31,14 +36,18 @@ import { SessionManager } from "@nsm/engine/session"; import { mkdirResource, saveResource } from "@nsm/resources"; import path from "path"; import { AppConfig } from "@nsm/config"; - -export type AppBootContext = AppContext & { steps: any }; +import { ServiceRunner } from "@nsm/engine/runner"; +import {TemplateManager} from "@nsm/engine/template"; +import {Facade} from "@nsm/engine/facade"; // Passed context to the routes export type AppContext = { router: Router; + facade: Facade, manager: ServiceManager; sessionManager: SessionManager; + templateManager: TemplateManager; + runner: ServiceRunner; database: Database; appConfig: AppConfig; logger: winston.Logger; @@ -57,36 +66,6 @@ function initGlobalLogger() { return logging.createLogger(); } -// Decorate all manager functions except those excluded to disallow using them -// before manager.engine is initialized. This is necessary as the manager is being -// used (mainly for expandEngine()) even before manager.init() is called. -function managerForUnsafeUse() { - const excludeKeys: (keyof ServiceManager)[] = [ - "expandEngine", - "initEngineForcibly", - "engine", - ]; - // - const managerRef = { ...manager }; - const handler: ProxyHandler = { - get(target, prop, receiver) { - // If it's key of base manager, not expanded object and is not excluded, deny access - if ( - (Object.keys(managerRef) as any[]).includes(prop) && - !(excludeKeys as any[]).includes(prop) - ) { - throw new Error( - "ServiceManager is not initialized yet! " + - "You can only access those members now: " + - excludeKeys.join(", "), - ); - } - return Reflect.get(target, prop, receiver); - }, - }; - return new Proxy(manager, handler); -} - /** * App orchestration code. * @@ -96,7 +75,7 @@ function managerForUnsafeUse() { export const init = async ( router: Application, options?: AppBootOptions, -): Promise => { +): Promise => { // Prepare logging const logger = initGlobalLogger(); logging.setCurrentGlobalLogger(logger); @@ -109,53 +88,45 @@ export const init = async ( prepareTestResources(); // Copy resources for test } - // Load addon steps - const steps = await loadAddons(logger); - - steps("BEFORE_CONFIG", { logger }); - - // Database connection layer - steps("BEFORE_DB", { logger, appConfig }); const database = createDbManager(); // Temporarily lock manager until it's initialized - const ctx = (currentContext = { + const ctx: AppContext = (currentContext = { router, - manager: managerForUnsafeUse(), + facade, + manager, + runner, sessionManager, + templateManager, database, appConfig, logger, debug: process.env.DEBUG === "true", }); - // Service (virtualization) layer - steps("BEFORE_ENGINE", ctx); - await manager.init(database, appConfig, logger); + const engine = createEngine(appConfig); + logger.info(`Using engine: ${engine.name}`); - // Bring back original manager - ctx.manager = currentContext.manager = middleLayer(manager); + initImageEngine(engine, templateManager, templateDirWatcher, database, appConfig, logger); + sessionManager.init(database); - // Load security - steps("BEFORE_SECURITY", ctx); - await loadSecurity(ctx); + await ctx.manager.init(appConfig, database, engine, logger); + templateDirWatcher.watchTemplateDirChanges(logger); - // Load HTTP routes - steps("BEFORE_ROUTES", ctx); - await loadAppRoutes(ctx); + await runner.init(engine, appConfig, templateManager, manager, database, logger); + ctx.runner = currentContext.runner = middleLayer(runner); - // Start the server - steps("BEFORE_SERVER", ctx); + await loadSecurity(ctx); + await loadAppRoutes(ctx); - let srv = undefined; if (options?.test == undefined || options.test == false) { logger.info(`Starting server`); - srv = router.listen(appConfig.getPort(), () => { + + router.listen(appConfig.getPort(), () => { logger.info(`Server started on port ${appConfig.getPort()}`); }); } - steps("BOOT", ctx, srv); - return { ...ctx, steps }; + return ctx; }; const prepareTestResources = () => { diff --git a/src/cleanup.ts b/src/cleanup.ts index 7f8e095..c50fca6 100644 --- a/src/cleanup.ts +++ b/src/cleanup.ts @@ -1,12 +1,11 @@ -import { AppBootContext } from "@nsm/app"; +import {AppContext} from "@nsm/app"; import { setStatus } from "@nsm/server"; -import { resolveSequentially } from "@nsm/util/promises"; import { setStopping } from "@nsm/engine/asyncp"; let active = false; -const cleanup = (ctx: AppBootContext, exit?: boolean) => { - const { manager, logger, steps } = ctx; +const cleanup = (ctx: AppContext, exit?: boolean) => { + const { runner, logger } = ctx; if (active == true) { return; @@ -19,22 +18,14 @@ const cleanup = (ctx: AppBootContext, exit?: boolean) => { setStopping(); } - resolveSequentially( - ...(exit == true - ? [ - // Those steps that should only be called on exit - () => steps("EXIT", ctx), - ] - : []), - () => manager.stopRunning(), - ).then(() => { + runner.stopRunning().then(() => { if (exit == true) { process.exit(0); } }); }; -export const postInit = (ctx: AppBootContext) => { +export const postInit = (ctx: AppContext) => { // Cleanup on start cleanup(ctx); diff --git a/src/engine/engine.ts b/src/engine/engine.ts index 610c22d..915757d 100644 --- a/src/engine/engine.ts +++ b/src/engine/engine.ts @@ -1,8 +1,7 @@ import DockerClient from "dockerode"; import buildDockerEngine from "./docker"; -import { getSingleton } from "../depend"; -import { MetaStorage } from "./manager"; -import { AppConfig } from "@nsm/config"; +import {getSingleton} from "../depend"; +import {AppConfig} from "@nsm/config"; /** * The options for running a service. @@ -93,6 +92,16 @@ export type RunListener = MessageListener & { onClose?: () => Promise | void; }; +/** + * Per-service storage. + * Data set here are being persisted to the relational database and being kept + * as long term data. Every key set here is per-service. + */ +export type MetaStorage = { + set: (key: string, value: any) => Promise; + get: (key: string, def?: T) => Promise; +}; + export type DockerServiceEngine = ServiceEngineI & { dockerClient: DockerClient; /** diff --git a/src/engine/facade.ts b/src/engine/facade.ts new file mode 100644 index 0000000..e623252 --- /dev/null +++ b/src/engine/facade.ts @@ -0,0 +1,85 @@ +import {Service} from "@nsm/engine/manager"; +import {ServiceSession} from "@nsm/engine/session"; +import {InternalSession} from "@nsm/engine/runner"; +import {PermaModel} from "@nsm/database"; + +import * as manager from "@nsm/engine/manager"; +import * as runner from "@nsm/engine/runner"; +import {getActionType} from "@nsm/engine/asyncp"; + +export type ServiceInfo = Service & { + state: State; + session?: ServiceSession; + internalSession?: InternalSession; +} + +export type State = "BUILDING" | "RUNNING" | "STOPPING" | "STOPPED"; + +export interface Facade { + /** + * Deletes a service by its ID. If the service is currently running, it will be stopped before deletion. + * + * @param id The ID of the service to delete. + */ + deleteService(id: string): Promise; + + /** + * Retrieves information about a service, including its current state and session information if requested. + * + * @param from The identifier for the service, which can be either a string ID or a PermaModel instance. + * @param options Optional parameters for retrieving service information. + * @returns A promise that resolves to the service information, or undefined if the service is not found. + */ + getServiceInfo( + from: string | PermaModel, + options?: { includeSession?: boolean }, + ): Promise; + + /** + * Retrieves the current state of a service by its ID. + * + * @param id The ID of the service to check the state of. + * @returns A promise that resolves to the current state of the service. + */ + getServiceState(id: string): Promise; +} + +export const deleteService: Facade["deleteService"] = async (id) => { + if (runner.isRunning(id)) { + // if running, stop the service first before deleting + await runner.stopService(id, true); + } + + await manager.deleteService(id); +} + +export const getServiceInfo: Facade["getServiceInfo"] = async (from, options) => { + const service = await manager.getService(from); + if (!service) { + return undefined; + } + + const runningService = options?.includeSession + ? runner.getRunningService(service.serviceId) + : null; + + return { + ...service, + state: await getServiceState(service.serviceId), + session: runningService ? runningService.session : undefined, + internalSession: runningService ? runningService.internalSession : undefined, + } +} + +export const getServiceState: Facade["getServiceState"] = async (id) => { + if (getActionType(id) === "stop") { + return "STOPPING"; + } + + const stage = runner.getServiceStage(id); + if (stage) { + return stage.state.ready ? "RUNNING" : "BUILDING"; + } else { + return "STOPPED"; + } +} \ No newline at end of file diff --git a/src/engine/image.ts b/src/engine/image.ts index 4d1a712..a60bedc 100644 --- a/src/engine/image.ts +++ b/src/engine/image.ts @@ -1,9 +1,9 @@ import { Database, ImageModel } from "@nsm/database"; import winston from "winston"; -import {MessageListener, ServiceEngine} from "@nsm/engine/engine"; +import { MessageListener, ServiceEngine } from "@nsm/engine/engine"; import { TemplateManager } from "@nsm/engine/template"; import { TemplateDirWatcher } from "@nsm/engine/monitoring/templateDirWatcher"; -import {AppConfig} from "@nsm/config"; +import { AppConfig } from "@nsm/config"; type BuildOptionsMap = { [key: string]: string; diff --git a/src/engine/manager.ts b/src/engine/manager.ts index 461074f..402a260 100644 --- a/src/engine/manager.ts +++ b/src/engine/manager.ts @@ -1,55 +1,25 @@ -import { currentContext } from "../app"; -import createEngine, { - RunOptions, - RunListener, - ServiceEngineI, - StandardLabel, - Filters, - combineRunListeners, -} from "./engine"; import { - Template, - getTemplate as loadTemplate, - getAllTemplates, -} from "./template"; + ServiceEngine, +} from "./engine"; import * as templateManager from "./template"; -import * as sessionManager from "./session"; -import * as templateDirWatcher from "./monitoring/templateDirWatcher"; import crypto from "crypto"; import { randomPort as retrieveRandomPort } from "@nsm/util/port"; import { Database, PermaModel } from "../database"; import { - getActionType, - isServicePending, - lockBusyAction, - reqNotPending, unlockBusyAction, - UnlockObserver, - whenUnlocked, - whenUnlockedAll, + reqNotPending, } from "./asyncp"; import winston from "winston"; -import { isDebug } from "../helpers"; -import {AsyncTask, resolveSequentially} from "@nsm/util/promises"; -import { watchTemplateDirChanges } from "@nsm/engine/monitoring/templateDirWatcher"; +import {resolveSequentially} from "@nsm/util/promises"; import { - processImage, - init as initImageEngine, deleteImageIfUnused, } from "@nsm/engine/image"; -import { propagateOptionsToEnv } from "@nsm/engine/docker/util/env"; -import { - ActiveServiceSession, - beginServiceSession, - ServiceSession, - init as initSessionEngine, -} from "@nsm/engine/session"; import { InternalError, InvalidMetaError, - ServiceAlreadyRunningError, ServiceNotFoundError, - ServiceNotRunningError, ServicePendingActionError, ServiceWasNeverActiveError, TemplateNotFoundError + TemplateNotFoundError } from "@nsm/engine/error"; +import {AppConfig} from "@nsm/config"; export type Options = { /** @@ -107,52 +77,11 @@ export type Options = { }; }; -/** - * Per-service storage. - * Data set here are being persisted to the relational database and being kept - * as long term data. Every key set here is per-service. - */ -export type MetaStorage = { - set: (key: string, value: any) => Promise; - get: (key: string, def?: T) => Promise; -}; - -export type EngineExpansion = { - [k in keyof ServiceEngineI | string]: any; -}; - -type ServiceEvent = { - id: string; - error?: Error; -}; - -type ServiceStateChangeEvent = ServiceEvent & { - state: State; -} - -type ServiceEngineErrorEvent = ServiceEvent & { - error: Error; +export type UpdateServiceOptions = { + imageId?: string; + options?: Options; } -type ServiceManagerEvents = { - resume: ServiceEvent; - stop: ServiceEvent; - statechange: ServiceStateChangeEvent; - engine_err: ServiceEngineErrorEvent; -}; - -/** - * The event handler for service manager events. - * If the handler returns true or nothing, it will be unsubscribed after this call. - */ -type EventHandler = ( - event: ServiceManagerEvents[T], -) => boolean | void; - -type ServiceManagerEventBus = { - on(evt: T, h: EventHandler): void; -}; - export type ListServicesOptions = { /** * The page number (index). @@ -174,24 +103,21 @@ export type ListServicesOptions = { }; }; -export type ServiceManager = ServiceManagerEventBus & { - /** - * This NSM instance ID - */ - nodeId: string; - /** - * Internal engine implementation - */ - engine: ServiceEngineI; - +export interface ServiceManager { /** * Initialize the service manager. * - * @param db The database * @param appConfig The app config + * @param db The database + * @param engine The service engine to use * @param logger The global logger */ - init(db: Database, appConfig: any, logger: winston.Logger): Promise; + init( + appConfig: AppConfig, + db: Database, + engine: ServiceEngine, + logger: winston.Logger + ): Promise; /** * Create a new service. @@ -203,22 +129,6 @@ export type ServiceManager = ServiceManagerEventBus & { */ createService(template: string, options: Options): Promise; // Service ID - /** - * Resume a service. - * - * @param id The service ID - */ - resumeService(id: string): Promise>; - - /** - * Stop a service. - * This hereby sends a stop signal and does not wait for it to be stopped. For waiting, use {@link waitForStopped}. - * - * @param id The service ID - * @param force Whether to force stop (kill) the service. - */ - stopService(id: string, force?: boolean): Promise>; - /** * Delete a service. * @@ -235,52 +145,19 @@ export type ServiceManager = ServiceManagerEventBus & { updateOptions(id: string, options: Options): Promise; /** - * Get the template by ID. + * Update the service. * - * @param id The template ID - * @returns The template wrapper + * @param id The service ID + * @param options The update options */ - getTemplate(id: string): Template | undefined; + updateService(id: string, options: UpdateServiceOptions): Promise; /** * Get the service by ID. * * @param from The service ID, or model - * @param options The get options - * includeSession: Whether to include the session to result - * otherNodes: If true, we will include services on other NSM nodes to search - */ - getService( - from: string | PermaModel, - options?: { includeSession?: boolean; otherNodes?: boolean }, - ): Promise; - - /** - * Get the last power error of a service. - * - * @param id The service ID - */ - getLastPowerError(id: string): Error | undefined; - - /** - * Get the last session ID of a service. - * - * @param id The service ID - * @throws ServiceWasNeverActiveError if the service was never active and thus does not have a last session - */ - getLastSession(id: string): Promise; - - /** - * Get list of running services on this node. - */ - getRunningServices(): RunningService[]; - - /** - * Get the running service by ID. - * - * @param id The service ID */ - getRunningService(id: string): RunningService | undefined; + getService(from: string | PermaModel): Promise; /** * List all available services. @@ -289,236 +166,39 @@ export type ServiceManager = ServiceManagerEventBus & { * @returns The list of service IDs */ listServices(options: ListServicesOptions): Promise; +} - /** - * List all available templates. - * - * @returns The list of template IDs - */ - listTemplates(): Promise; - - /** - * Stop all running services on this instance. - */ - stopRunning(): Promise; - - /** - * Kill all running services on this instance. - */ - killRunning(): Promise; - - isRunning(id: string): boolean; - - waitForBusyAction(id: string): Promise; - - waitForStopped(id: string): Promise; - - // DON'T call those until you really know what you are doing. - expandEngine(exp?: T): Promise; - - initEngineForcibly(): Promise; - // -}; - -type RunningService = { - id: string; - session: ServiceSession; - internalSession: InternalSession; -}; - -export type InternalSession = { - containerId: string; - // TODO: add more useful information? -}; - -export type ServiceInfo = PermaModel & { +export type Service = PermaModel & { optionsRam: number; // From options.ram optionsCpu: number; // From options.cpu optionsDisk: number; // From options.disk - state: State; - session?: ServiceSession; - internalSession?: InternalSession; }; -export type State = "BUILDING" | "RUNNING" | "STOPPING" | "STOPPED"; - -export let engine: ServiceEngineI = undefined; -export let nodeId: string; - +let nodeId: string; let db: Database; +let engine: ServiceEngine; let logger: winston.Logger; -// TODO: Save errors somewhere else? -// Could it be a memory leak if there are tons of them?? -const errors = {}; -// Service IDs that are currently running -const started: RunningService[] = []; -const startedStates: Map = new Map(); -const evtHandlers: Map[]> = new Map(); - -["push", "splice"].forEach((funcName) => { - started[funcName] = (...args: any[]) => { - const result = Array.prototype[funcName].apply(started, args); - - // Emit services change within those methods - if (isDebug()) { - logger.debug("Service registry changed"); - } - - return result; - }; -}); - export const init: ServiceManager["init"] = async ( - db_, appConfig_, + db_, + engine_, logger_, ) => { + nodeId = appConfig_.getNodeId(); db = db_; + engine = engine_; logger = logger_; - - const nodeId_ = appConfig_.getNodeId(); - logger.info(`Initializing service manager for node ${nodeId_}...`); - if (!engine) { - // Init only if it has not already been force-initialized - await initEngineForcibly(); - } - nodeId = nodeId_ as string; - - initImageEngine( - engine, - templateManager, - templateDirWatcher, - db_, - appConfig_, - logger, - ); - initSessionEngine(db_); - watchTemplateDirChanges(logger); - - gatherEngineErrors(); - registerLoggingEventHandlers(); - await deleteGarbage(logger); - await reattachStaleContainers(logger); - - logger.info(`Using engine: ${engine.name}`); -} - -const deleteGarbage = async (logger: winston.Logger) => { - // TODO: delete containers that are not running and remained from last session -} - -/** - * Reattach to containers that are still running from the previous session. - * This may happen if NSM was force-stopped and not properly cleared up resources. - * - * @param logger The logger to use - */ -const reattachStaleContainers = async (logger: winston.Logger) => { - const running = await engine - .listRunning(Filters.node(nodeId)) - .then((containerIds) => - containerIds - // Filter out those that we have already started in this session, just in case - // this was started more than once a session - .filter( - (id) => - !started.find( - (runningService) => - runningService.internalSession.containerId === id, - ), - ), - ); - - for (let containerId of running) { - const labels = await engine.getLabels(containerId); - if (!labels[StandardLabel.ServiceId]) { - // The container was in the running list, but does not have the required labels - // Should not happen, but just in case - logger.warn( - `Found a running container with id ${containerId} that does not have a service id label, stopping.`, - ); - - await engine.stop(containerId); - } - - const serviceId = labels[StandardLabel.ServiceId]; - - // We must begin a new session since the previous was interrupted - const session = await beginServiceSession(serviceId); - // Reattach and watch the container - await engine.reattach(containerId, buildRunListener(session)); - - // Save session in-memory - const info: RunningService = { - id: serviceId, - session, - internalSession: { - containerId, - }, - }; - started.push(info); - logger.info(`Reattached container ${containerId} for service ${serviceId}`); - } - - await new Promise((resolve) => whenUnlockedAll(() => resolve(null))); -} - -const gatherEngineErrors = () => { - on("engine_err", (event) => { - errors[event.id] = event.error; - }); -} - -/** - * Registers event handlers for logging in debug mode. - */ -const registerLoggingEventHandlers = () => { - const notifyIfSuccess = ( - messageProvider: (serviceId: string) => string - ): EventHandler => { - return ({ id, error }) => { - if (error) { - return; - } - - logger.debug(messageProvider(id)); - } - } - - on("resume", notifyIfSuccess((id) => `Service ${id} resumed`)); - on("stop", notifyIfSuccess((id) => `Service ${id} stopped`)); -} - -export const expandEngine: ServiceManager["expandEngine"] = async ( - exp?: T, -): Promise => { - if (exp) { - if (!engine && (!currentContext || !currentContext.appConfig)) { - throw new Error("Engine is not yet loaded and can't be loaded forcibly!"); - } else if (!engine) { - // Engine is not initialized yet, but we want to expand it, so - // we need to force load it. - await initEngineForcibly(); - } - // An expansion is provided, so there are changes to be applied. - Object.keys(exp).forEach((expKey) => { - if (!Number.isNaN(Number(expKey))) { - throw new Error( - "Invalid expansion format, please replace functions within with lambda functions. " + - "Invalid: { funcName(param) {}, funcName2(param) {} }" + - "Valid: { funcName: (param) => {}, funcName2: (param) => {} }", - ); - } - engine[expKey] = exp[expKey]; - }); - } - return engine as any; } export const createService: ServiceManager["createService"] = async (template, options) => { const { ram, cpu, disk, ports, env, network } = options; - const serviceSettings = reqTemplate(template).settings; + + const foundTemplate = templateManager.getTemplate(template); + if (!foundTemplate) { + throw new TemplateNotFoundError(template); + } + const serviceSettings = foundTemplate.settings; // Join meta supplied by user and template meta const meta = { @@ -543,217 +223,62 @@ export const createService: ServiceManager["createService"] = async (template, o template, nodeId, port, - options: { ram, cpu, disk, ports }, + options: { + ram, + cpu, + disk, + ports + }, meta, env: env ?? {}, network, }; // Save permanent info - if (!(await db.permaRepository.savePerma(perma))) { + const saved = await db.permaRepository.savePerma(perma); + if (!saved) { throw new InternalError("Failed to save perma info to database"); } return serviceId; } -export const resumeService: ServiceManager["resumeService"] = async (id) => { - reqNotRunning(id); - let { template, options, env, network, port } = await reqExists(id); - - const { defaults, env: settingsEnv } = reqTemplate(template).settings; - // Filter env to only those that are defined in settings.yml, because those are the only ones that - // we can guarantee to be used and will not make problems when handling images. - env = { - ...Object.entries(env) - .filter(([key]) => settingsEnv && key in settingsEnv) - .reduce((obj, [key, value]) => ({ ...obj, [key]: value }), {}), - }; - - const meta = metaStorageForService(id); - const unlock = lockBusyAction(id, "resume"); - - const runOptions: RunOptions = { - ram: options.ram ?? (defaults.ram as number), - cpu: options.cpu ?? (defaults.cpu as number), - disk: options.disk ?? (defaults.disk as number), - env: env ?? (defaults.env as { [key: string]: string }), - port, - ports: options.ports ?? [], - network, - labels: { - [StandardLabel.Nsm]: "true", - [StandardLabel.ServiceId]: id, - [StandardLabel.NodeId]: nodeId, - [StandardLabel.VolumeId]: id, - [StandardLabel.TemplateId]: template, +export const deleteService: ServiceManager["deleteService"] = async (id) => { + const image = await db.permaRepository + .getPerma(id) + .then((perma) => + perma.imageId + ? db.imageRepository.getImage(perma.imageId) + : undefined, + ); + await resolveSequentially( + async () => engine.deleteVolume(id), + async () => db.permaRepository.deletePerma(id), + async () => { + if (image) { + // If the image becomes unused after service deletion, delete it + await deleteImageIfUnused(image); + } }, - }; - - const perma = await db.permaRepository.getPerma(id); - //let image = perma.imageId; - - // Propagate other options to env, so they can be used in image processing and building - propagateOptionsToEnv(runOptions, runOptions.env); - // Include service ID in env - runOptions.env.SERVICE_ID = id; - - // Omit the always-changing args from build env, since they would always trigger an - // image rebuild - const { SERVICE_ID, SERVICE_PORT, SERVICE_PORTS, ...buildEnv } = - runOptions.env; - - const updateImageIfChanged = async (image: string) => { - // If the image was changed by processing (e.g. it was built or rebuilt), update the image id in database - if (image != perma.imageId) { - - // Update image in database if it was changed by processing - perma.imageId = image; - await db.permaRepository.savePerma(perma); - } - - return image; - } - - return new AsyncTask( - // TODO: logovat někam message z image processingu pomocí posledního parametru - processImage(perma.imageId, template, buildEnv) - .then(updateImageIfChanged) - .then(async (image) => { - const session = await beginServiceSession(id); - // Run the container with the built image and save the container id for later use. - try { - const containerId = await engine.run( - image, - id, - runOptions, - meta, - buildRunListener(session), - ); - const runningService: RunningService = { - id, - session, - internalSession: { - containerId, - }, - }; - started.push(runningService); - - callManagerEvent("resume", { id }); - } catch (e) { - callManagerEvent("resume", { id, error: e }); - callServiceEngineError(id, e); - } - }) - .finally(() => unlock()) ); -} - -export const stopService: ServiceManager["stopService"] = async (id, force) => { - await reqExists(id); - const { internalSession } = reqRunning(id); - - const callEngine = async (task: () => Promise) => { - try { - await task(); - } catch (e) { - logger.error(e); - callManagerEvent("stop", { id, error: e }); - } - } + logger.debug(`Service ${id} deleted`); +} - let awaitingPromise: Promise; - if (force) { - const pendingAction = getActionType(id); - if (pendingAction && getActionType(id) !== "stop") { - // the service is locked and not stopping, the force stop can't be allowed - throw new ServicePendingActionError(id, pendingAction); +export const updateService: ServiceManager["updateService"] = async (id, options) => { + let success = true; + if (options.imageId) { + const perma = await db.permaRepository.getPerma(id); + if (!perma) { + throw new ServiceNotFoundError(id); } + perma.imageId = options.imageId; - await callEngine(async () => engine.kill(internalSession.containerId, metaStorageForService(id))); - // resolves immediately on kill - awaitingPromise = Promise.resolve(); - } else { - // lock only on soft stop, to allow hard-killing if any issues happen during stopping - const unlock = lockBusyAction(id, "stop"); - awaitingPromise = new Promise((resolve) => { - // wait for stop - // this is really not necessary because any busy action is unlocked on stop, but - // just in case and for the promise - on("stop", ({ id: stoppedId, error }) => { - if (stoppedId !== id) { - // This call is not for me - return false; - } - - if (isServicePending(id)) { - unlock(error); - } - resolve(); - return true; - }); - }); - - // TODO: stop strategy - const service = await getService(id); - const stopCmd = service.meta?.stopCmd; - await callEngine(async () => { - if (stopCmd) { - // send stop cmd if set - await engine.cmd(internalSession.containerId, stopCmd); - } else { - // send stop signal - await engine.stop(internalSession.containerId); - } - }); + success = await db.permaRepository.savePerma(perma); } - awaitingPromise = awaitingPromise.then(() => waitForStopped(id)); - - return new AsyncTask(awaitingPromise); -} - -export const deleteService: ServiceManager["deleteService"] = async (id) => { - try { - await stopService(id, true); - } catch (e) { - // Skip not running error - if (!(e instanceof ServiceNotRunningError)) { - throw e; - } + if (options.options && !await updateOptions(id, options.options)) { + success = false; } - - const unlockHandler: UnlockObserver = (_, __, ___) => { - const resolveDeleteImageFunc = async () => { - const image = await db.permaRepository - .getPerma(id) - .then((perma) => - perma.imageId - ? db.imageRepository.getImage(perma.imageId) - : undefined, - ); - - return async () => { - if (image) { - // If the image becomes unused after service deletion, delete it - await deleteImageIfUnused(image); - } - }; - }; - - resolveDeleteImageFunc() - .then((deleteImageFunc) => - resolveSequentially( - async () => engine.deleteVolume(id), - async () => db.permaRepository.deletePerma(id), - deleteImageFunc, - ), - ) - .then(() => { - logger.debug(`Service ${id} deleted`); - }); - }; - - whenUnlocked(id, unlockHandler); + return success; } export const updateOptions: ServiceManager["updateOptions"] = async (id, options) => { @@ -774,307 +299,24 @@ export const updateOptions: ServiceManager["updateOptions"] = async (id, options return db.permaRepository.savePerma(data); } -export const getTemplate: ServiceManager["getTemplate"] = (id) => { - return loadTemplate(id); -} - -export const getService: ServiceManager["getService"] = async ( - from, - options, -): ReturnType => { - const data = - typeof from === "string" ? await db.permaRepository.getPerma(from) : from; - if (data && (data.nodeId == nodeId || options?.otherNodes === true)) { - let session = undefined; - let internalSession = undefined; - if (options?.includeSession === true) { - const runningService = getRunningService(data.serviceId); - if (runningService) { - session = runningService.session; - internalSession = runningService.internalSession; - } - } - - return { - ...data, - optionsRam: data.env.SERVICE_RAM ? Number(data.env.SERVICE_RAM) : 0, - optionsCpu: data.env.SERVICE_CPU ? Number(data.env.SERVICE_CPU) : 0, - optionsDisk: data.env.SERVICE_DISK ? Number(data.env.SERVICE_DISK) : 0, - state: getServiceState(data.serviceId), - session, - internalSession, - }; - } else { +export const getService: ServiceManager["getService"] = async (from) => { + const data = typeof from === "string" ? await db.permaRepository.getPerma(from) : from; + if (!data) { return undefined; } -} - -export const getLastPowerError: ServiceManager["getLastPowerError"] = (id) => { - return errors[id]; -} - -export const getLastSession: ServiceManager["getLastSession"] = async (id) => { - await reqExists(id); - - const runningService = getRunningService(id); - if (runningService) { - // Service currently running, we can use logs from the current session - return runningService.session; - } else { - // Service not running, so we need to retrieve last session ID - const lastSession = await sessionManager.listSessions({ - filter: { serviceId: id }, - sort: { by: "startedAt", direction: "desc" }, - page: { index: 0, size: 1 }, - }); - if (lastSession && lastSession.length > 0) { - return lastSession[0]; - } - } - - throw new ServiceWasNeverActiveError(); -} - -export const listServices: ServiceManager["listServices"] = async (options) => { - const meta = options.filter?.meta; - return db.permaRepository - .listPerma(nodeId, options.page, options.pageSize, meta) - .then((list) => list.map((d) => d.serviceId)); -} - -export const listTemplates: ServiceManager["listTemplates"] = async () => { - return getAllTemplates().map((template) => template.id); -} - -export const stopRunning: ServiceManager["stopRunning"] = async () => { - const tasks = started.map( - ({ id }) => - new Promise((resolve) => { - whenUnlocked(id, () => { - stopService(id) - .catch((e) => logger.error(e)) - .then(() => { - whenUnlocked(id, () => resolve(null)); - }); - }); - }), - ); - - await Promise.all(tasks); -} - -export const killRunning: ServiceManager["killRunning"] = async () => { - await Promise.all( - started.map( - async ({ id }) => stopService(id, true).catch((e) => logger.error(e)) - ) - ) -} - -export const waitForBusyAction: ServiceManager["waitForBusyAction"] = async (id: string) => { - return new Promise((resolve, reject) => { - whenUnlocked(id, (_, __, err) => (err ? reject(err) : resolve(null))); - }); -} - -export const waitForStopped: ServiceManager["waitForStopped"] = async (id: string) => { - if (!isRunning(id)) { - // service not running, so we continue immediately - return; - } - return new Promise((resolve, reject) => { - on("stop", ({ id, error }) => { - if (id !== id) { - // This call is not for me - return false; - } - - if (error) { - reject(error); - } else { - resolve(); - } - }); - }); -} - -export const isRunning: ServiceManager["isRunning"] = (id: string) => { - return getRunningService(id) != undefined; -} - -export const getRunningService: ServiceManager["getRunningService"] = (id: string) => { - return started.find((service) => service.id === id); -} - -const metaStorageForService = (id: string): MetaStorage => { - // service id return { - set: async (key, value) => { - return db.serviceMetaRepository.setServiceMeta(id, key, value); - }, - get: async (key, def) => { - const meta = await db.serviceMetaRepository.getServiceMeta(id, key); - - return meta ?? def; - }, + ...data, + optionsRam: data.env.SERVICE_RAM ? Number(data.env.SERVICE_RAM) : 0, + optionsCpu: data.env.SERVICE_CPU ? Number(data.env.SERVICE_CPU) : 0, + optionsDisk: data.env.SERVICE_DISK ? Number(data.env.SERVICE_DISK) : 0, }; } -export const initEngineForcibly = async () => { - if (engine) { - throw new Error("Engine is already loaded."); - } - if (!currentContext || !currentContext.appConfig) { - throw new Error("Engine can't be loaded forcibly!"); - } - engine = createEngine(currentContext.appConfig); - // I set it here to keep the exact reference if the engine - // is changed in the future. - engine.cast = () => engine as any; -} - -export const getRunningServices: ServiceManager["getRunningServices"] = () => { - return [...started]; -} - -export const on: ServiceManager["on"] = ( - evt: T, - h: EventHandler, -) => { - if (!evtHandlers.has(evt)) { - evtHandlers.set(evt, []); - } - evtHandlers.get(evt).push(h); -} - -const clearRunningServiceIfExists = (id: string) => { - const service = getRunningService(id); - - if (service) { - started.splice(started.indexOf(service, 1)); - } -} - -const callManagerEvent = ( - e: T, - event: ServiceManagerEvents[T], -) => { - if (!evtHandlers.has(e)) { - return; - } - const newArray = evtHandlers.get(e).filter((handler) => { - // Filter out those who returned true, which means they want to be unsubscribed after this call. - const result = handler(event); - - return typeof result != "boolean" || !result; - }); - evtHandlers.set(e, newArray); -} - -/** - * Notifies about an error that happened during internal engine calling. - * - * @param id The service ID for which the error happened - * @param error The error that happened - */ -const callServiceEngineError = (id: string, error: Error) => { - callManagerEvent("engine_err", { id, error }); -} - -/** - * Collects all relevant run listeners and builds a composite one - * to be used directly when running/attaching service container. - * - * @param session The session for whom to create the session. - */ -const buildRunListener = (session: ActiveServiceSession): RunListener => { - const { serviceId } = session; - - // The internal run listener of this manager - const internalRunListener: RunListener = { - onStateChange: (state) => { - setServiceState(serviceId, state.ready ? "RUNNING" : "BUILDING"); - }, - onClose: async () => { - clearRunningServiceIfExists(serviceId); - startedStates.delete(serviceId); - // clear any busy action that may potentially still be locked - try { - unlockBusyAction(serviceId); - } catch (e) { - if (e.message && e.message.includes("No busy action")) { - // ignore, since it just means there is no busy action to unlock, so nothing to do - } - } - - callManagerEvent("stop", { id: serviceId }); - }, - }; - // Combine collected listeners - return combineRunListeners([ - internalRunListener, - // Add listener from the session - session.runListener, - ]); -} - -const setServiceState = (id: string, state: State) => { - startedStates.set(id, state); - - callManagerEvent("statechange", { - id, - state, - }); -} - -/** - * Returns the local service state managed by this manager. - * - * @param id The id of the service. - * @returns The state of the service - */ -const getServiceState = (id: string) => { - if (getActionType(id) === "stop") { - // service has stop locked, so is stopping - return "STOPPING"; - } - - return startedStates.get(id) ?? "STOPPED"; -} - -// --------------------------------------------------------------------------------------- - -const reqExists = async (id: string) => { - const perma_ = await db.permaRepository.getPerma(id); - if (!perma_) { - // service does not exist - throw new ServiceNotFoundError(id); - } - - return perma_; -} - -const reqRunning = (id: string) => { - const session = getRunningService(id); - if (!session) { - throw new ServiceNotRunningError(id); - } - - return session; -} - -const reqNotRunning = (id: string) => { - if (isRunning(id)) { - throw new ServiceAlreadyRunningError(id); - } -} +export const listServices: ServiceManager["listServices"] = async (options) => { + const meta = options.filter?.meta; -const reqTemplate = (id: string) => { - const template = getTemplate(id); - if (!template) { - throw new TemplateNotFoundError(id); - } + const list = await db.permaRepository.listPerma(nodeId, options.page, options.pageSize, meta); - return template; -} + return list.map((d) => d.serviceId); +} \ No newline at end of file diff --git a/src/engine/middle.ts b/src/engine/middle.ts index 0eb968f..935c5cb 100644 --- a/src/engine/middle.ts +++ b/src/engine/middle.ts @@ -1,7 +1,7 @@ -import {ServiceManager} from "@nsm/engine/manager"; import {currentContext} from "@nsm/app"; import {KnownError} from "@nsm/engine/error"; import {AsyncTask} from "@nsm/util/promises"; +import {ServiceRunner} from "@nsm/engine/runner"; export type ServiceActionType = | "create" @@ -146,41 +146,27 @@ const argServiceIdExtractor = ( }; /** - * Wraps a {@link ServiceManager} instance with additional capabilities. + * Wraps a {@link ServiceRunner} instance with additional capabilities. * Asynchronous service lifecycle methods are decorated to allow * additional behavior. * - * @param manager The original ServiceManager instance to wrap. - * @returns A new ServiceManager instance with decorated methods. + * @param runner The original ServiceRunner instance to wrap. + * @returns A new ServiceRunner instance with decorated methods. */ -export const middleLayer = (manager: ServiceManager): ServiceManager => { +export const middleLayer = (runner: ServiceRunner): ServiceRunner => { return { - ...manager, - - createService: decorateFunc(manager.createService, "create"), + ...runner, resumeService: decorateFunc( - manager.resumeService, + runner.resumeService, "resume", argServiceIdExtractor(0), ), stopService: decorateFunc( - manager.stopService, + runner.stopService, "stop", argServiceIdExtractor(0), ), - - sendStopSignal: decorateFunc( - manager.sendStopSignal, - "sendStopSignal", - argServiceIdExtractor(0), - ), - - deleteService: decorateFunc( - manager.deleteService, - "delete", - argServiceIdExtractor(0), - ), }; }; diff --git a/src/engine/runner.ts b/src/engine/runner.ts new file mode 100644 index 0000000..bab558d --- /dev/null +++ b/src/engine/runner.ts @@ -0,0 +1,641 @@ +import {AsyncTask} from "@nsm/util/promises"; +import {ActiveServiceSession, beginServiceSession, ServiceSession} from "@nsm/engine/session"; +import { + getActionType, + isServicePending, + lockBusyAction, + unlockBusyAction, + whenUnlocked, + whenUnlockedAll +} from "@nsm/engine/asyncp"; +import { + combineRunListeners, + Filters, + MetaStorage, + RunListener, + RunOptions, ServiceEngine, + ServiceState, + StandardLabel +} from "@nsm/engine/engine"; +import {propagateOptionsToEnv} from "@nsm/engine/docker/util/env"; +import {processImage} from "@nsm/engine/image"; +import { + InternalError, + ServiceAlreadyRunningError, + ServiceNotFoundError, ServiceNotRunningError, ServicePendingActionError, + TemplateNotFoundError +} from "@nsm/engine/error"; +import {ServiceManager} from "@nsm/engine/manager"; +import {TemplateManager} from "@nsm/engine/template"; +import {Database} from "@nsm/database"; +import {isDebug} from "@nsm/helpers"; +import winston from "winston"; +import {AppConfig} from "@nsm/config"; + +type ServiceEvent = { + id: string; + error?: Error; +}; + +type ServiceStateChangeEvent = ServiceEvent & { + state: ServiceState; +} + +type ServiceEngineErrorEvent = ServiceEvent & { + error: Error; +} + +type ServiceRunnerEvents = { + resume: ServiceEvent; + stop: ServiceEvent; + statechange: ServiceStateChangeEvent; + engine_err: ServiceEngineErrorEvent; +}; + +/** + * The event handler for service runner events. + * If the handler returns true or nothing, it will be unsubscribed after this call. + */ +type EventHandler = ( + event: ServiceRunnerEvents[T], +) => boolean | void; + +interface ServiceRunnerEventBus { + on(evt: T, h: EventHandler): void; +} + +export interface ServiceRunner extends ServiceRunnerEventBus { + engine: ServiceEngine; + + /** + * Resume a service. + * + * @param id The service ID + */ + resumeService(id: string): Promise>; + + /** + * Stop a service. + * This hereby sends a stop signal and does not wait for it to be stopped. For waiting, use {@link waitForStopped}. + * + * @param id The service ID + * @param force Whether to force stop (kill) the service. + */ + stopService(id: string, force?: boolean): Promise>; + + /** + * Get list of running services on this node. + */ + getRunningServices(): RunningService[]; + + /** + * Get the running service by ID. + * + * @param id The service ID + */ + getRunningService(id: string): RunningService | undefined; + + /** + * Get the current stage of a service, that is, currently being handled by the runner. + * + * @param id The service ID + */ + getServiceStage(id: string): HandledServiceStage | undefined; + + /** + * Get the last power error of a service. + * + * @param id The service ID + */ + getLastPowerError(id: string): Error | undefined; + + /** + * Stop all running services on this instance. + */ + stopRunning(): Promise; + + /** + * Kill all running services on this instance. + */ + killRunning(): Promise; + + isRunning(id: string): boolean; + + waitForBusyAction(id: string): Promise; + + waitForStopped(id: string): Promise; +} + +type HandledServiceStage = { + state: ServiceState; +} + +type RunningService = { + id: string; + session: ServiceSession; + internalSession: InternalSession; + state?: ServiceState; +}; + +export type InternalSession = { + containerId: string; + // TODO: add more useful information? +}; + +export let engine: ServiceEngine; + +let nodeId: string; +let templateManager: TemplateManager; +let serviceManager: ServiceManager; +let db: Database; +let logger: winston.Logger; + +// Service IDs that are currently running +const started: RunningService[] = []; +const startedStages: Map = new Map(); +// TODO: Save errors somewhere else? +// Could it be a memory leak if there are tons of them?? +const errors = {}; +const evtHandlers: Map[]> = new Map(); + +["push", "splice"].forEach((funcName) => { + started[funcName] = (...args: any[]) => { + const result = Array.prototype[funcName].apply(started, args); + + // Emit services change within those methods + if (isDebug()) { + logger.debug("Service registry changed"); + } + + return result; + }; +}); + +export const init = async ( + engine_: ServiceEngine, + appConfig: AppConfig, + templateManager_: TemplateManager, + serviceManager_: ServiceManager, + db_: Database, + logger_: winston.Logger, +) => { + engine = engine_; + nodeId = appConfig.getNodeId(); + templateManager = templateManager_; + serviceManager = serviceManager_; + db = db_; + logger = logger_; + + registerLoggingEventHandlers(); + gatherEngineErrors(); + await deleteGarbage(logger); + await reattachStaleContainers(logger); +} + +const deleteGarbage = async (logger: winston.Logger) => { + // TODO: delete containers that are not running and remained from last session +} + +/** + * Reattach to containers that are still running from the previous session. + * This may happen if NSM was force-stopped and not properly cleared up resources. + * + * @param logger The logger to use + */ +const reattachStaleContainers = async (logger: winston.Logger) => { + const running = await engine + .listRunning(Filters.node(nodeId)) + .then((containerIds) => + containerIds + // Filter out those that we have already started in this session, just in case + // this was started more than once a session + .filter( + (id) => + !started.find( + (runningService) => + runningService.internalSession?.containerId === id, + ), + ), + ); + + for (let containerId of running) { + const labels = await engine.getLabels(containerId); + if (!labels[StandardLabel.ServiceId]) { + // The container was in the running list, but does not have the required labels + // Should not happen, but just in case + logger.warn( + `Found a running container with id ${containerId} that does not have a service id label, stopping.`, + ); + + await engine.stop(containerId); + } + + const serviceId = labels[StandardLabel.ServiceId]; + + // We must begin a new session since the previous was interrupted + const session = await beginServiceSession(serviceId); + // Reattach and watch the container + await engine.reattach(containerId, buildRunListener(session)); + + // Save session in-memory + const info: RunningService = { + id: serviceId, + session, + internalSession: { + containerId, + }, + }; + started.push(info); + logger.info(`Reattached container ${containerId} for service ${serviceId}`); + } + + await new Promise((resolve) => whenUnlockedAll(() => resolve(null))); +} + +/** + * Registers event handlers for logging in debug mode. + */ +const registerLoggingEventHandlers = () => { + const notifyIfSuccess = ( + messageProvider: (serviceId: string) => string + ): EventHandler => { + return ({ id, error }) => { + if (error) { + return; + } + + logger.debug(messageProvider(id)); + } + } + + on("resume", + notifyIfSuccess((id) => `Service ${id} resumed`)); + on("stop", + notifyIfSuccess((id) => `Service ${id} stopped`)); +} + +const gatherEngineErrors = () => { + on("engine_err", (event) => { + errors[event.id] = event.error; + }); +} + +export const resumeService: ServiceRunner["resumeService"] = async (id) => { + if (isRunning(id)) { + throw new ServiceAlreadyRunningError(id); + } + + const service = await serviceManager.getService(id); + if (!service) { + throw new ServiceNotFoundError(id); + } + + let { + options, + env, + network, + port, + ...rest + } = service; + + const template = templateManager.getTemplate(rest.template); + if (!template) { + throw new TemplateNotFoundError(rest.template); + } + + let { defaults, env: settingsEnv } = template.settings; + // Filter env to only those that are defined in settings.yml, because those are the only ones that + // we can guarantee to be used and will not make problems when handling images. + env = { + ...Object.entries(env) + .filter(([key]) => settingsEnv && key in settingsEnv) + .reduce((obj, [key, value]) => ({ ...obj, [key]: value }), {}), + }; + + const meta = buildMetaStorage(id); + const unlock = lockBusyAction(id, "resume"); + + const runOptions: RunOptions = { + ram: options.ram ?? (defaults.ram as number), + cpu: options.cpu ?? (defaults.cpu as number), + disk: options.disk ?? (defaults.disk as number), + env: env ?? (defaults.env as { [key: string]: string }), + port, + ports: options.ports ?? [], + network, + labels: { + [StandardLabel.Nsm]: "true", + [StandardLabel.ServiceId]: id, + [StandardLabel.NodeId]: nodeId, + [StandardLabel.VolumeId]: id, + [StandardLabel.TemplateId]: template.id, + }, + }; + + // Propagate other options to env, so they can be used in image processing and building + propagateOptionsToEnv(runOptions, runOptions.env); + // Include service ID in env + runOptions.env.SERVICE_ID = id; + + // Omit the always-changing args from build env, since they would always trigger an + // image rebuild + const { SERVICE_ID, SERVICE_PORT, SERVICE_PORTS, ...buildEnv } = + runOptions.env; + + const updateImageIfChanged = async (image: string) => { + // If the image was changed by processing (e.g. it was built or rebuilt), update the image id in database + if (image != service.imageId) { + + // Update image in database if it was changed by processing + const updated = await serviceManager.updateService(service.serviceId, { imageId: image }); + if (!updated) { + throw new InternalError(`Failed to update image ID for service ${service.serviceId}`); + } + } + + return image; + } + + return new AsyncTask( + // TODO: logovat někam message z image processingu pomocí posledního parametru + processImage(service.imageId, template.id, buildEnv) + .then(updateImageIfChanged) + .then(async (image) => { + const session = await beginServiceSession(id); + // Run the container with the built image and save the container id for later use. + try { + const containerId = await engine.run( + image, + id, + runOptions, + meta, + buildRunListener(session), + ); + started.push({ + id, + session, + internalSession: { + containerId, + }, + }); + + callManagerEvent("resume", { id }); + } catch (e) { + callManagerEvent("resume", { id, error: e }); + callServiceEngineError(id, e); + } + }) + .finally(() => unlock()) + ); +} + +export const stopService: ServiceRunner["stopService"] = async (id, force) => { + const service = await serviceManager.getService(id); + if (!service) { + throw new ServiceNotFoundError(id); + } + + const runningService = getRunningService(id); + if (!runningService) { + throw new ServiceNotRunningError(id); + } + + const callEngine = async (task: () => Promise) => { + try { + await task(); + } catch (e) { + logger.error(e); + callManagerEvent("stop", { id, error: e }); + } + } + + const internalSession = runningService.internalSession; + + let awaitingPromise: Promise; + if (force) { + const pendingAction = getActionType(id); + if (pendingAction && pendingAction !== "stop") { + // the service is locked and not stopping, the force stop can't be allowed + throw new ServicePendingActionError(id, pendingAction); + } + + await callEngine(async () => engine.kill(internalSession.containerId, buildMetaStorage(id))); + // resolves immediately on kill + awaitingPromise = Promise.resolve(); + } else { + // lock only on soft stop, to allow hard-killing if any issues happen during stopping + const unlock = lockBusyAction(id, "stop"); + awaitingPromise = new Promise((resolve) => { + // wait for stop + // this is really not necessary because any busy action is unlocked on stop, but + // just in case and for the promise + on("stop", ({ id: stoppedId, error }) => { + if (stoppedId !== id) { + // This call is not for me + return false; + } + + if (isServicePending(id)) { + unlock(error); + } + resolve(); + return true; + }); + }); + + // TODO: stop strategy + const stopCmd = service.meta?.stopCmd; + await callEngine(async () => { + if (stopCmd) { + // send stop cmd if set + await engine.cmd(internalSession.containerId, stopCmd); + } else { + // send stop signal + await engine.stop(internalSession.containerId); + } + }); + } + awaitingPromise = awaitingPromise.then(() => waitForStopped(id)); + + return new AsyncTask(awaitingPromise); +} + +export const getRunningService: ServiceRunner["getRunningService"] = (id) => { + return started.find((service) => service.id === id); +} + +export const getServiceStage: ServiceRunner["getServiceStage"] = (id) => { + return startedStages.get(id); +} + +export const isRunning: ServiceRunner["isRunning"] = (id: string) => { + return getRunningService(id) != undefined; +} + +/** + * Builds the meta storage for a service, which is used for storing and retrieving internal metadata for the service. + * + * @param serviceId The ID of the service for which to build the meta storage. + */ +const buildMetaStorage = (serviceId: string): MetaStorage => { + // service id + return { + set: async (key, value) => { + return db.serviceMetaRepository.setServiceMeta(serviceId, key, value); + }, + get: async (key, def) => { + const meta = await db.serviceMetaRepository.getServiceMeta(serviceId, key); + + return meta ?? def; + }, + }; +} + +/** + * Collects all relevant run listeners and builds a composite one + * to be used directly when running/attaching service container. + * + * @param session The session for whom to create the session. + */ +const buildRunListener = (session: ActiveServiceSession): RunListener => { + const { serviceId } = session; + + // The internal run listener of this manager + const internalRunListener: RunListener = { + onStateChange: (state) => { + const stage = startedStages.get(serviceId); + if (stage) { + stage.state = state; + } else { + startedStages.set(serviceId, { state }); + } + + callManagerEvent("statechange", { id: serviceId, state }); + }, + onClose: async () => { + clearRunningServiceIfExists(serviceId); + startedStages.delete(serviceId); + // clear any busy action that may potentially still be locked + try { + unlockBusyAction(serviceId); + } catch (e) { + if (e.message && e.message.includes("No busy action")) { + // ignore, since it just means there is no busy action to unlock, so nothing to do + } + } + + callManagerEvent("stop", { id: serviceId }); + }, + }; + // Combine collected listeners + return combineRunListeners([ + internalRunListener, + // Add listener from the session + session.runListener, + ]); +} + +export const on: ServiceRunner["on"] = ( + evt: T, + h: EventHandler, +) => { + if (!evtHandlers.has(evt)) { + evtHandlers.set(evt, []); + } + evtHandlers.get(evt).push(h); +} + +const callManagerEvent = ( + e: T, + event: ServiceRunnerEvents[T], +) => { + if (!evtHandlers.has(e)) { + return; + } + const newArray = evtHandlers.get(e).filter((handler) => { + // Filter out those who returned true, which means they want to be unsubscribed after this call. + const result = handler(event); + + return typeof result != "boolean" || !result; + }); + evtHandlers.set(e, newArray); +} + +/** + * Notifies about an error that happened during internal engine calling. + * + * @param id The service ID for which the error happened + * @param error The error that happened + */ +const callServiceEngineError = (id: string, error: Error) => { + callManagerEvent("engine_err", { id, error }); +} + +const clearRunningServiceIfExists = (id: string) => { + const service = getRunningService(id); + + if (service) { + started.splice(started.indexOf(service, 1)); + } +} + +export const getLastPowerError: ServiceRunner["getLastPowerError"] = (id) => { + return errors[id]; +} + +export const getRunningServices: ServiceRunner["getRunningServices"] = () => { + return [...started]; +} + +export const waitForStopped: ServiceRunner["waitForStopped"] = async (id: string) => { + if (!isRunning(id)) { + // service not running, so we continue immediately + return; + } + + return new Promise((resolve, reject) => { + on("stop", ({ id, error }) => { + if (id !== id) { + // This call is not for me + return false; + } + + if (error) { + reject(error); + } else { + resolve(); + } + }); + }); +} + +export const stopRunning: ServiceRunner["stopRunning"] = async () => { + const tasks = started.map( + ({ id }) => + new Promise((resolve) => { + whenUnlocked(id, () => { + stopService(id) + .catch((e) => logger.error(e)) + .then(() => { + whenUnlocked(id, () => resolve(null)); + }); + }); + }), + ); + + await Promise.all(tasks); +} + +export const killRunning: ServiceRunner["killRunning"] = async () => { + await Promise.all( + started.map( + async ({ id }) => stopService(id, true).catch((e) => logger.error(e)) + ) + ) +} + +export const waitForBusyAction: ServiceRunner["waitForBusyAction"] = async (id: string) => { + return new Promise((resolve, reject) => { + whenUnlocked(id, (_, __, err) => (err ? reject(err) : resolve(null))); + }); +} \ No newline at end of file diff --git a/src/engine/session.ts b/src/engine/session.ts index 6fa5fbb..c08152c 100644 --- a/src/engine/session.ts +++ b/src/engine/session.ts @@ -7,6 +7,7 @@ import { ServiceLogRecordModel, ServiceSessionModel, } from "@nsm/database"; +import {ServiceWasNeverActiveError} from "@nsm/engine/error"; export interface SessionManager { /** @@ -25,6 +26,15 @@ export interface SessionManager { */ beginServiceSession(serviceId: string): Promise; + /** + * Retrieves the last session for a given service ID. + * + * @param serviceId The ID of the service for which to retrieve the last session. + * @return An object representing the last service session, or undefined if no sessions were found. + * @throws ServiceWasNeverActiveError if the service has never had an active session. + */ + getLastSession(serviceId: string): Promise; + /** * Lists service sessions. * @@ -192,7 +202,21 @@ const debounceBulkPush = () => { }; }; -// TODO: get service session +export const getLastSession: SessionManager["getLastSession"] = async ( + serviceId +) => { + // Service not running, so we need to retrieve last session ID + const lastSession = await listSessions({ + filter: { serviceId }, + sort: { by: "startedAt", direction: "desc" }, + page: { index: 0, size: 1 }, + }); + if (lastSession && lastSession.length > 0) { + return lastSession[0]; + } + + throw new ServiceWasNeverActiveError(); +} export const listSessions: SessionManager["listSessions"] = async ( args: ListSessionsArgs, diff --git a/src/router/v1/service/createRoute.ts b/src/router/v1/service/createRoute.ts index 5a8bbd0..07544bb 100644 --- a/src/router/v1/service/createRoute.ts +++ b/src/router/v1/service/createRoute.ts @@ -7,6 +7,8 @@ import {TemplateNotFoundError} from "@nsm/engine/error"; export default async function ({ manager, + templateManager, + runner, }: AppContext): Promise { return { url: "/service/create", @@ -20,7 +22,7 @@ export default async function ({ .end(); return; } - const template = manager.getTemplate(req.body.template); + const template = templateManager.getTemplate(req.body.template); if (!template) { throw new TemplateNotFoundError(req.body.template); } @@ -39,7 +41,7 @@ export default async function ({ const serviceId = await manager.createService(template.id, options); - await manager.resumeService(serviceId); + await runner.resumeService(serviceId); res .status(200) diff --git a/src/router/v1/service/deleteRoute.ts b/src/router/v1/service/deleteRoute.ts index bfb7f07..441c43a 100644 --- a/src/router/v1/service/deleteRoute.ts +++ b/src/router/v1/service/deleteRoute.ts @@ -2,7 +2,7 @@ import { AppContext } from "@nsm/app"; import { RouterHandler } from "../../index"; export default async function ({ - manager, + facade, }: AppContext): Promise { return { url: "/service/:id/delete", @@ -19,7 +19,7 @@ export default async function ({ return; } - await manager.deleteService(id); + await facade.deleteService(id); res.status(200).json({ status: 200, message: "Service deleted." }); }, diff --git a/src/router/v1/service/listRoute.ts b/src/router/v1/service/listRoute.ts index 4b06c19..af4c854 100644 --- a/src/router/v1/service/listRoute.ts +++ b/src/router/v1/service/listRoute.ts @@ -6,6 +6,7 @@ import z from "zod"; export default async function ({ manager, database, + appConfig }: AppContext): Promise { return { url: "/servicelist", @@ -73,7 +74,7 @@ export default async function ({ meta: { ...listOptions, // Total num of services on this node - total: await database.permaRepository.countPerma(manager.nodeId), + total: await database.permaRepository.countPerma(appConfig.getNodeId()), }, }; res.status(200).json(data).end(); diff --git a/src/router/v1/service/logsRoute.ts b/src/router/v1/service/logsRoute.ts index c957e85..af9de59 100644 --- a/src/router/v1/service/logsRoute.ts +++ b/src/router/v1/service/logsRoute.ts @@ -30,7 +30,7 @@ export default async function (ctx: AppContext): Promise { let logs: ServiceLogRecordModel[]; try { - const session = await ctx.manager.getLastSession(id); + const session = await ctx.sessionManager.getLastSession(id); logs = await ctx.sessionManager.listSessionLogs({ filter: { sessionId: session.id, diff --git a/src/router/v1/service/lookupRoute.ts b/src/router/v1/service/lookupRoute.ts index 3c18a82..3c50a17 100644 --- a/src/router/v1/service/lookupRoute.ts +++ b/src/router/v1/service/lookupRoute.ts @@ -2,7 +2,8 @@ import { AppContext } from "@nsm/app"; import { RouterHandler } from "../../index"; export default async function ({ - manager, + runner, + facade }: AppContext): Promise { return { url: "/service/:id", @@ -10,7 +11,7 @@ export default async function ({ get: async (req, res) => { const id = req.params.id; - const service = await manager.getService(id, { includeSession: true }); + const service = await facade.getServiceInfo(id, { includeSession: true }); if (!service) { res .status(404) @@ -21,8 +22,8 @@ export default async function ({ const session = service.internalSession; let stats: any; - if (session && req.query.stats === "true") { - stats = await manager.engine.stat(session.containerId); + if (session && session.containerId && req.query.stats === "true") { + stats = await runner.engine.stat(session.containerId); } else { stats = null; } diff --git a/src/router/v1/service/powerStatusRoute.ts b/src/router/v1/service/powerStatusRoute.ts index e3db385..fc1556c 100644 --- a/src/router/v1/service/powerStatusRoute.ts +++ b/src/router/v1/service/powerStatusRoute.ts @@ -1,10 +1,8 @@ -import { AppContext } from "../../../app"; +import { AppContext } from "@nsm/app"; import { RouterHandler } from "../../index"; import { isServicePending } from "@nsm/engine/asyncp"; -export default async function ({ - manager, -}: AppContext): Promise { +export default async function (ctx: AppContext): Promise { return { url: "/service/:id/powerstatus", routes: { @@ -24,7 +22,7 @@ export default async function ({ if (isServicePending(id)) { status = "PENDING"; } else { - const err = manager.getLastPowerError(id); + const err = ctx.runner.getLastPowerError(id); if (err) { status = "ERROR"; error = err; diff --git a/src/router/v1/service/rebootRoute.ts b/src/router/v1/service/rebootRoute.ts index 78628e9..3ea5233 100644 --- a/src/router/v1/service/rebootRoute.ts +++ b/src/router/v1/service/rebootRoute.ts @@ -3,7 +3,7 @@ import { RouterHandler } from "../../index"; import {KnownError, ServiceNotRunningError} from "@nsm/engine/error"; export default async function ({ - manager, + runner, }: AppContext): Promise { return { url: "/service/:id/reboot", @@ -23,7 +23,7 @@ export default async function ({ let promise: Promise; try { - const task = await manager.stopService(id, isForce); + const task = await runner.stopService(id, isForce); promise = task.promise; } catch (e) { if (e instanceof ServiceNotRunningError) { @@ -35,7 +35,7 @@ export default async function ({ } promise.then(async () => { try { - const task = await manager.resumeService(id); + const task = await runner.resumeService(id); await task.promise; } catch (e) { diff --git a/src/router/v1/service/resumeRoute.ts b/src/router/v1/service/resumeRoute.ts index 7140bbc..03e7ef3 100644 --- a/src/router/v1/service/resumeRoute.ts +++ b/src/router/v1/service/resumeRoute.ts @@ -3,6 +3,7 @@ import { RouterHandler } from "../../index"; export default async function ({ manager, + runner }: AppContext): Promise { return { url: "/service/:id/resume", @@ -19,7 +20,7 @@ export default async function ({ return; } - await manager.resumeService(id); + await runner.resumeService(id); res.status(200).json({ status: 200, diff --git a/src/router/v1/service/stopRoute.ts b/src/router/v1/service/stopRoute.ts index 9857f0c..4cae218 100644 --- a/src/router/v1/service/stopRoute.ts +++ b/src/router/v1/service/stopRoute.ts @@ -2,7 +2,7 @@ import { AppContext } from "@nsm/app"; import { RouterHandler } from "../../index"; export default async function ({ - manager, + runner }: AppContext): Promise { return { url: "/service/:id/stop", @@ -20,7 +20,7 @@ export default async function ({ return; } - await manager.stopService(id, isForce); + await runner.stopService(id, isForce); res.status(200).json({ status: 200, diff --git a/src/router/v1/status/index.ts b/src/router/v1/status/index.ts index 3388a02..c5c7ced 100644 --- a/src/router/v1/status/index.ts +++ b/src/router/v1/status/index.ts @@ -1,12 +1,17 @@ import { AppContext } from "@nsm/app"; import { RouterHandler } from "../../index"; import * as os from "os"; -import { Filters, ServiceManager } from "@nsm/engine"; +import {Filters, ServiceEngine, ServiceManager} from "@nsm/engine"; import { Database } from "@nsm/database"; -async function checkNsmResources(engine: ServiceManager, db: Database) { - const stats = await engine.engine.statAll(Filters.node(engine.nodeId)); - const servicesGlobal = await db.permaRepository.listPerma(engine.nodeId); +async function checkNsmResources( + nodeId: string, + manager: ServiceManager, + engine: ServiceEngine, + db: Database +) { + const stats = await engine.statAll(Filters.node(nodeId)); + const servicesGlobal = await db.permaRepository.listPerma(nodeId); const res = stats.reduce( (acc, s) => { acc.memory.used += s.memory.used; @@ -35,7 +40,7 @@ async function checkNsmResources(engine: ServiceManager, db: Database) { }, ); for (const s of servicesGlobal) { - const service = await engine.getService(s); + const service = await manager.getService(s); res.services.memTotal += BigInt(service.optionsRam); res.services.cpuTotal += BigInt(service.optionsCpu); res.services.diskTotal += BigInt(service.optionsDisk); @@ -57,6 +62,7 @@ async function checkNsmResources(engine: ServiceManager, db: Database) { */ export default async function ({ manager, + runner, appConfig, database, }: AppContext): Promise { @@ -66,7 +72,7 @@ export default async function ({ get: async (req, res) => { const nodeId = appConfig.getNodeId(); const all = await database.permaRepository.listPerma(nodeId); - const [free, size] = await manager.engine.calcHostUsage(); + const [free, size] = await runner.engine.calcHostUsage(); const system = { totalmem: os.totalmem(), freemem: os.freemem(), @@ -76,12 +82,15 @@ export default async function ({ res .json({ nodeId, - running: manager.getRunningServices().map((s) => s.id), + running: runner.getRunningServices().map((s) => s.id), all: all.length, system, ...(req.query.stats === "true" - ? { stats: await checkNsmResources(manager, database) } - : {}), + ? { + stats: await checkNsmResources(appConfig.getNodeId(), manager, runner.engine, database) + } + : { + }), }) .end(); }, diff --git a/tests/api/api.test.ts b/tests/api/api.test.ts index 32c7cba..881a579 100644 --- a/tests/api/api.test.ts +++ b/tests/api/api.test.ts @@ -1,5 +1,5 @@ import server from "@nsm/server"; -import { init as boot, AppBootContext, AppBootOptions } from "@nsm/app"; +import {init as boot, AppBootOptions, AppContext} from "@nsm/app"; import request from "supertest"; import { afterAll, beforeAll, describe, expect, test } from "@jest/globals"; import { isServicePending } from "@nsm/engine/asyncp"; @@ -15,9 +15,9 @@ function expectProps(obj: any, model: any[]) { } } -async function miniService(ctx: AppBootContext) { +async function miniService(ctx: AppContext) { const id = await ctx.manager.createService("test", {}); - await ctx.manager.resumeService(id); + await ctx.runner.resumeService(id); do { await new Promise((resolve) => { @@ -25,20 +25,20 @@ async function miniService(ctx: AppBootContext) { }); } while (isServicePending(id)); // Status check - if (ctx.manager.getLastPowerError(id)) { + if (ctx.runner.getLastPowerError(id)) { return undefined; } else { return id; } } -async function killMini(ctx: AppBootContext, id: string) { - await ctx.manager.stopService(id, true); - await ctx.manager.waitForStopped(id); +async function killMini(ctx: AppContext, id: string) { + await ctx.runner.stopService(id, true); + await ctx.runner.waitForStopped(id); } describe("Test v1 API models", () => { - let ctx: AppBootContext | undefined = undefined; + let ctx: AppContext | undefined = undefined; beforeAll((done) => { const options: AppBootOptions = { @@ -192,7 +192,7 @@ describe("Test v1 API models", () => { expectProps(res.body, ["status", 200, "message", undefined]); // Wait for it to be started await new Promise((resolve, reject) => { - ctx.manager.on("resume", (event) => { + ctx.runner.on("resume", (event) => { if (event.id == id) { if (event.error) { reject(event.error); @@ -218,7 +218,7 @@ describe("Test v1 API models", () => { return; } - return ctx.manager.killRunning(); + return ctx.runner.killRunning(); }, 60000); // TODO: /v1/service//options diff --git a/tests/engine/middle.test.ts b/tests/engine/middle.test.ts index 696ada9..e9a98f9 100644 --- a/tests/engine/middle.test.ts +++ b/tests/engine/middle.test.ts @@ -4,41 +4,8 @@ import { registerErrorPublisher, ServiceActionError, } from "@nsm/engine/middle"; -import * as manager from "@nsm/engine/manager"; -import { Options, ServiceManager } from "@nsm/engine/manager"; - -it("test receives action error", async () => { - let receivedError: ServiceActionError | null = null; - registerErrorPublisher({ - publishError(action: ServiceActionError): Promise { - receivedError = action; - - return Promise.resolve(); - }, - }); - - let customManager: ServiceManager = { - ...manager, - async createService(_: string, __: Options) { - throw new Error("Failed to create service"); - }, - }; - customManager = middleLayer(customManager); - - let threw = false; - try { - await customManager.createService("test-template", {}); - } catch (e) { - // Expected to throw an error - threw = true; - } - - expect(threw).toBe(true); - expect(receivedError).not.toBeNull(); - expect(receivedError?.serviceId).toBeUndefined(); - expect(receivedError?.type).toEqual("create"); - expect(receivedError?.message).toEqual("Failed to create service"); -}); +import * as runner from "@nsm/engine/runner"; +import {ServiceRunner} from "@nsm/engine/runner"; it("test sets service id in action error", async () => { let receivedError: ServiceActionError | null = null; @@ -50,17 +17,17 @@ it("test sets service id in action error", async () => { }, }); - let customManager: ServiceManager = { - ...manager, + let customRunner: ServiceRunner = { + ...runner, async resumeService(_: string) { throw new Error("Failed to resume service"); }, }; - customManager = middleLayer(customManager); + customRunner = middleLayer(customRunner); let threw = false; try { - await customManager.resumeService("test-service-id"); + await customRunner.resumeService("test-service-id"); } catch (e) { // Expected to throw an error threw = true; From 14671f39825399f504239fd0bcbd4ddd73210fe7 Mon Sep 17 00:00:00 2001 From: ZorTik Date: Sun, 14 Jun 2026 21:47:50 +0200 Subject: [PATCH 28/53] feat: tighten ServiceActionType in middle --- src/engine/middle.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/engine/middle.ts b/src/engine/middle.ts index 935c5cb..bb95737 100644 --- a/src/engine/middle.ts +++ b/src/engine/middle.ts @@ -4,12 +4,8 @@ import {AsyncTask} from "@nsm/util/promises"; import {ServiceRunner} from "@nsm/engine/runner"; export type ServiceActionType = - | "create" | "resume" - | "stop" - | "forceStop" - | "sendStopSignal" - | "delete"; + | "stop"; /** * Represents an error that occurred during a service action. From e60b3ff08704d01f5ddff55cfce9352b6c9dfd4d Mon Sep 17 00:00:00 2001 From: ZorTik Date: Sun, 14 Jun 2026 23:14:01 +0200 Subject: [PATCH 29/53] refactor --- src/app.ts | 6 ++- src/engine/engine.ts | 1 - src/engine/error.ts | 8 ++++ src/engine/facade.ts | 4 +- src/engine/index.ts | 2 +- src/engine/middle.ts | 14 +++++- src/engine/runner.ts | 69 ++++++++++++++------------- src/engine/{manager.ts => service.ts} | 68 ++++++++++---------------- src/engine/template.ts | 23 ++++++++- src/router/v1/status/index.ts | 9 ++-- src/util/services.ts | 12 ++++- 11 files changed, 129 insertions(+), 87 deletions(-) rename src/engine/{manager.ts => service.ts} (83%) diff --git a/src/app.ts b/src/app.ts index d05601c..37b4908 100644 --- a/src/app.ts +++ b/src/app.ts @@ -22,7 +22,7 @@ import loadSecurity from "@nsm/security"; import createEngine from "@nsm/engine/engine"; import { init as initImageEngine } from "@nsm/engine/image"; import * as facade from "@nsm/engine/facade"; -import * as manager from "@nsm/engine/manager"; +import * as manager from "@nsm/engine/service"; import * as runner from "@nsm/engine/runner"; import * as sessionManager from "@nsm/engine/session"; import * as templateManager from "@nsm/engine/template"; @@ -31,7 +31,7 @@ import * as logging from "./logger"; import winston from "winston"; import { Application } from "express-ws"; import fs from "fs"; -import { middleLayer } from "@nsm/engine/middle"; +import {middleLayer, registerErrorPublishersFromConfig} from "@nsm/engine/middle"; import { SessionManager } from "@nsm/engine/session"; import { mkdirResource, saveResource } from "@nsm/resources"; import path from "path"; @@ -104,6 +104,8 @@ export const init = async ( debug: process.env.DEBUG === "true", }); + await registerErrorPublishersFromConfig(appConfig); + const engine = createEngine(appConfig); logger.info(`Using engine: ${engine.name}`); diff --git a/src/engine/engine.ts b/src/engine/engine.ts index 915757d..08b665a 100644 --- a/src/engine/engine.ts +++ b/src/engine/engine.ts @@ -187,7 +187,6 @@ export type ServiceEngine = { /** * Deletes a volume by ID. - * This is NEVER called if ServiceEngine#useVolumes is false. * * @param id The volume ID. */ diff --git a/src/engine/error.ts b/src/engine/error.ts index 36634c7..de28bb6 100644 --- a/src/engine/error.ts +++ b/src/engine/error.ts @@ -58,6 +58,14 @@ export class ServicePendingActionError extends KnownError { } } +export class ServiceEngineError extends InternalError { + constructor( + public readonly cause: Error + ) { + super("An error occurred in the service engine. Cause: " + cause.message); + } +} + export class TemplateNotFoundError extends KnownError { constructor( public readonly templateId: string diff --git a/src/engine/facade.ts b/src/engine/facade.ts index e623252..efe1bff 100644 --- a/src/engine/facade.ts +++ b/src/engine/facade.ts @@ -1,9 +1,9 @@ -import {Service} from "@nsm/engine/manager"; +import {Service} from "@nsm/engine/service"; import {ServiceSession} from "@nsm/engine/session"; import {InternalSession} from "@nsm/engine/runner"; import {PermaModel} from "@nsm/database"; -import * as manager from "@nsm/engine/manager"; +import * as manager from "@nsm/engine/service"; import * as runner from "@nsm/engine/runner"; import {getActionType} from "@nsm/engine/asyncp"; diff --git a/src/engine/index.ts b/src/engine/index.ts index eff7ffb..e4aeca6 100644 --- a/src/engine/index.ts +++ b/src/engine/index.ts @@ -1,2 +1,2 @@ -export * from "./manager"; +export * from "./service"; export * from "./engine"; diff --git a/src/engine/middle.ts b/src/engine/middle.ts index bb95737..8320f97 100644 --- a/src/engine/middle.ts +++ b/src/engine/middle.ts @@ -1,7 +1,8 @@ import {currentContext} from "@nsm/app"; -import {KnownError} from "@nsm/engine/error"; +import {InternalError, KnownError} from "@nsm/engine/error"; import {AsyncTask} from "@nsm/util/promises"; import {ServiceRunner} from "@nsm/engine/runner"; +import {AppConfig} from "@nsm/config"; export type ServiceActionType = | "resume" @@ -14,6 +15,7 @@ export interface ServiceActionError { serviceId?: string; type: ServiceActionType; message: string; + internal: boolean; } export interface ErrorPublisher { @@ -40,6 +42,15 @@ export const registerErrorPublisher = (publisher: ErrorPublisher) => { publishers.push(publisher); }; +/** + * Registers error publishers based on the app configuration. + * + * @param config The application configuration used to determine which error publishers to register. + */ +export const registerErrorPublishersFromConfig = async (config: AppConfig) => { + // TODO: publishers +} + const publishError = async (action: ServiceActionError) => { try { await Promise.all(publishers.map((p) => p.publishError(action))); @@ -92,6 +103,7 @@ const handleExecutionError = async ) => Pro serviceId: serviceIdExtractor?.(args), type: actionType, message: e instanceof Error ? e.message : String(e), + internal: !(e instanceof KnownError), }; await publishError(action); diff --git a/src/engine/runner.ts b/src/engine/runner.ts index bab558d..ed8792c 100644 --- a/src/engine/runner.ts +++ b/src/engine/runner.ts @@ -21,11 +21,11 @@ import {propagateOptionsToEnv} from "@nsm/engine/docker/util/env"; import {processImage} from "@nsm/engine/image"; import { InternalError, - ServiceAlreadyRunningError, + ServiceAlreadyRunningError, ServiceEngineError, ServiceNotFoundError, ServiceNotRunningError, ServicePendingActionError, TemplateNotFoundError } from "@nsm/engine/error"; -import {ServiceManager} from "@nsm/engine/manager"; +import {ServiceManager} from "@nsm/engine/service"; import {TemplateManager} from "@nsm/engine/template"; import {Database} from "@nsm/database"; import {isDebug} from "@nsm/helpers"; @@ -356,37 +356,38 @@ export const resumeService: ServiceRunner["resumeService"] = async (id) => { return image; } - return new AsyncTask( - // TODO: logovat někam message z image processingu pomocí posledního parametru - processImage(service.imageId, template.id, buildEnv) - .then(updateImageIfChanged) - .then(async (image) => { - const session = await beginServiceSession(id); - // Run the container with the built image and save the container id for later use. - try { - const containerId = await engine.run( - image, - id, - runOptions, - meta, - buildRunListener(session), - ); - started.push({ - id, - session, - internalSession: { - containerId, - }, - }); - - callManagerEvent("resume", { id }); - } catch (e) { - callManagerEvent("resume", { id, error: e }); - callServiceEngineError(id, e); - } - }) - .finally(() => unlock()) - ); + const task = processImage(service.imageId, template.id, buildEnv) // TODO: logovat někam message z image processingu pomocí posledního parametru + .then(updateImageIfChanged) + .then(async (image) => { + const session = await beginServiceSession(id); + // Run the container with the built image and save the container id for later use. + try { + const containerId = await engine.run( + image, + id, + runOptions, + meta, + buildRunListener(session), + ); + started.push({ + id, + session, + internalSession: { + containerId, + }, + }); + + callManagerEvent("resume", { id }); + } catch (e) { + callManagerEvent("resume", { id, error: e }); + callServiceEngineError(id, e); + + throw new ServiceEngineError(e); + } + }) + .finally(() => unlock()); + + return new AsyncTask(task); } export const stopService: ServiceRunner["stopService"] = async (id, force) => { @@ -406,6 +407,8 @@ export const stopService: ServiceRunner["stopService"] = async (id, force) => { } catch (e) { logger.error(e); callManagerEvent("stop", { id, error: e }); + + throw new ServiceEngineError(e); } } diff --git a/src/engine/manager.ts b/src/engine/service.ts similarity index 83% rename from src/engine/manager.ts rename to src/engine/service.ts index 402a260..2376693 100644 --- a/src/engine/manager.ts +++ b/src/engine/service.ts @@ -4,18 +4,16 @@ import { import * as templateManager from "./template"; import crypto from "crypto"; import { randomPort as retrieveRandomPort } from "@nsm/util/port"; -import { Database, PermaModel } from "../database"; +import {Database, ImageModel, PermaModel} from "../database"; import { reqNotPending, } from "./asyncp"; import winston from "winston"; -import {resolveSequentially} from "@nsm/util/promises"; import { deleteImageIfUnused, } from "@nsm/engine/image"; import { InternalError, - InvalidMetaError, ServiceNotFoundError, TemplateNotFoundError } from "@nsm/engine/error"; @@ -43,6 +41,9 @@ export type Options = { * (optional) */ ports?: number[]; // Optional ports to expose + /** + * The optional meta attributes to set for the service. + */ meta?: { [key: string]: any }; /** * The optional environment variables (template options) to set. @@ -168,11 +169,7 @@ export interface ServiceManager { listServices(options: ListServicesOptions): Promise; } -export type Service = PermaModel & { - optionsRam: number; // From options.ram - optionsCpu: number; // From options.cpu - optionsDisk: number; // From options.disk -}; +export type Service = PermaModel; let nodeId: string; let db: Database; @@ -201,13 +198,14 @@ export const createService: ServiceManager["createService"] = async (template, o const serviceSettings = foundTemplate.settings; // Join meta supplied by user and template meta - const meta = { - ...(options.meta ?? {}), - ...(serviceSettings.meta ?? {}), - }; - if (!meta || !meta.stopCmd) { - throw new InvalidMetaError("Invalid template meta for " + template); + let meta = {}; + if (options.meta) { + meta = { ...meta, ...options.meta }; } + if (serviceSettings.meta) { + meta = { ...meta, ...serviceSettings.meta }; + } + // validate meta? and throw InvalidMetaError const serviceId = crypto.randomUUID(); // Create new unique service id // Pick random main port from the range specified in settings.yml @@ -243,23 +241,19 @@ export const createService: ServiceManager["createService"] = async (template, o } export const deleteService: ServiceManager["deleteService"] = async (id) => { - const image = await db.permaRepository - .getPerma(id) - .then((perma) => - perma.imageId - ? db.imageRepository.getImage(perma.imageId) - : undefined, - ); - await resolveSequentially( - async () => engine.deleteVolume(id), - async () => db.permaRepository.deletePerma(id), - async () => { - if (image) { - // If the image becomes unused after service deletion, delete it - await deleteImageIfUnused(image); - } - }, - ); + let image: ImageModel | undefined; + + const perma = await db.permaRepository.getPerma(id); + if (perma.imageId) { + image = await db.imageRepository.getImage(perma.imageId); + } + + await engine.deleteVolume(id); + await db.permaRepository.deletePerma(id); + if (image) { + // If the image becomes unused after service deletion, delete it + await deleteImageIfUnused(image); + } logger.debug(`Service ${id} deleted`); } @@ -300,17 +294,7 @@ export const updateOptions: ServiceManager["updateOptions"] = async (id, options } export const getService: ServiceManager["getService"] = async (from) => { - const data = typeof from === "string" ? await db.permaRepository.getPerma(from) : from; - if (!data) { - return undefined; - } - - return { - ...data, - optionsRam: data.env.SERVICE_RAM ? Number(data.env.SERVICE_RAM) : 0, - optionsCpu: data.env.SERVICE_CPU ? Number(data.env.SERVICE_CPU) : 0, - optionsDisk: data.env.SERVICE_DISK ? Number(data.env.SERVICE_DISK) : 0, - }; + return typeof from === "string" ? await db.permaRepository.getPerma(from) : from; } export const listServices: ServiceManager["listServices"] = async (options) => { diff --git a/src/engine/template.ts b/src/engine/template.ts index 02b20a1..b9e6a73 100644 --- a/src/engine/template.ts +++ b/src/engine/template.ts @@ -19,9 +19,30 @@ export type Template = { /** * The settings (definitions) object. */ - settings: any; + settings: TemplateSettings; }; +export type TemplateSettings = { + port_range: { + min: number; + max: number; + }; + defaults: { + ram: number; + cpu: number; + disk: number; + env?: { + [key: string]: string; + } + }; + meta: { + [key: string]: string; + }; + env: { + [key: string]: string; + } +} + export type TemplateManager = { /** * Prepares the environment variables for a template by validating the provided env object against diff --git a/src/router/v1/status/index.ts b/src/router/v1/status/index.ts index c5c7ced..0114361 100644 --- a/src/router/v1/status/index.ts +++ b/src/router/v1/status/index.ts @@ -3,6 +3,7 @@ import { RouterHandler } from "../../index"; import * as os from "os"; import {Filters, ServiceEngine, ServiceManager} from "@nsm/engine"; import { Database } from "@nsm/database"; +import {parseResourceOptionsSet} from "@nsm/util/services"; async function checkNsmResources( nodeId: string, @@ -41,9 +42,11 @@ async function checkNsmResources( ); for (const s of servicesGlobal) { const service = await manager.getService(s); - res.services.memTotal += BigInt(service.optionsRam); - res.services.cpuTotal += BigInt(service.optionsCpu); - res.services.diskTotal += BigInt(service.optionsDisk); + const resourceOptions = parseResourceOptionsSet(service); + + res.services.memTotal += BigInt(resourceOptions.ram); + res.services.cpuTotal += BigInt(resourceOptions.cpu); + res.services.diskTotal += BigInt(resourceOptions.disk); } if (res.memory.total > 0) { res.memory.percent = res.memory.used / res.memory.total; diff --git a/src/util/services.ts b/src/util/services.ts index c5a8425..82a52a4 100644 --- a/src/util/services.ts +++ b/src/util/services.ts @@ -1,11 +1,21 @@ +import {Service} from "@nsm/engine"; + export type NSMObjectLabels = { id: string; }; // Default labels to use in docker engine objects produced by NSM -export function constructObjectLabels({ id }: NSMObjectLabels) { +export const constructObjectLabels = ({ id }: NSMObjectLabels) => { return { nsm: "true", "nsm.id": id, }; } + +export const parseResourceOptionsSet = (service: Service) => { + return { + ram: service.env.SERVICE_RAM ? Number(service.env.SERVICE_RAM) : 0, + cpu: service.env.SERVICE_CPU ? Number(service.env.SERVICE_CPU) : 0, + disk: service.env.SERVICE_DISK ? Number(service.env.SERVICE_DISK) : 0, + }; +} \ No newline at end of file From 3c6d9e76b0124adbebdbe38f331403cb072a3d57 Mon Sep 17 00:00:00 2001 From: ZorTik Date: Mon, 15 Jun 2026 02:04:02 +0200 Subject: [PATCH 30/53] feat: stop strategy --- dev/templates/test/settings.yml | 4 +- .../templates/example/example_settings.yml | 2 +- src/database/models.ts | 2 +- src/engine/middle.ts | 2 +- src/engine/runner.ts | 114 ++++++++++++++---- 5 files changed, 94 insertions(+), 30 deletions(-) diff --git a/dev/templates/test/settings.yml b/dev/templates/test/settings.yml index 1b222b8..20bb2c3 100644 --- a/dev/templates/test/settings.yml +++ b/dev/templates/test/settings.yml @@ -9,6 +9,6 @@ defaults: disk: 2000000000 # bytes meta: # Stop command to be sent in stop signal endpoint - stopCmd: "stop" + internal/stop-command: "stop" # Optional ENV vars, and their default values -env: {} +env: {} \ No newline at end of file diff --git a/resources/templates/example/example_settings.yml b/resources/templates/example/example_settings.yml index f351972..db34c8f 100644 --- a/resources/templates/example/example_settings.yml +++ b/resources/templates/example/example_settings.yml @@ -10,7 +10,7 @@ defaults: disk: 2000000000 # bytes meta: # Stop command to be sent in stop signal endpoint - stopCmd: "stop" + internal/stop-command: "stop" # Optional ENV vars, and their default values env: STARTUP_FILE: "server.jar" diff --git a/src/database/models.ts b/src/database/models.ts index 3666fa1..578a89b 100644 --- a/src/database/models.ts +++ b/src/database/models.ts @@ -99,7 +99,7 @@ export type PermaModel = { [key: string]: any; }; meta?: { - stopCmd?: string; + [key: string]: string; }; env: { [key: string]: string; diff --git a/src/engine/middle.ts b/src/engine/middle.ts index 8320f97..4abdf4f 100644 --- a/src/engine/middle.ts +++ b/src/engine/middle.ts @@ -1,5 +1,5 @@ import {currentContext} from "@nsm/app"; -import {InternalError, KnownError} from "@nsm/engine/error"; +import {KnownError} from "@nsm/engine/error"; import {AsyncTask} from "@nsm/util/promises"; import {ServiceRunner} from "@nsm/engine/runner"; import {AppConfig} from "@nsm/config"; diff --git a/src/engine/runner.ts b/src/engine/runner.ts index ed8792c..3a037da 100644 --- a/src/engine/runner.ts +++ b/src/engine/runner.ts @@ -25,7 +25,7 @@ import { ServiceNotFoundError, ServiceNotRunningError, ServicePendingActionError, TemplateNotFoundError } from "@nsm/engine/error"; -import {ServiceManager} from "@nsm/engine/service"; +import {Service, ServiceManager} from "@nsm/engine/service"; import {TemplateManager} from "@nsm/engine/template"; import {Database} from "@nsm/database"; import {isDebug} from "@nsm/helpers"; @@ -60,6 +60,72 @@ type EventHandler = ( event: ServiceRunnerEvents[T], ) => boolean | void; +interface StopStrategyProvider { + /** + * Get the stop strategy for a service. + * + * @param service The service for which to get the stop strategy + * @returns The stop strategy for the service + */ + getStopStrategy(service: Service): Promise; +} + +class MetaStopStrategyProvider implements StopStrategyProvider { + getStopStrategy = async (service: Service) => { + const metaKey = "internal/stop-command"; + + if (service.meta && service.meta[metaKey]) { + return new StopCommandStopStrategy(service.meta[metaKey]); + } else { + return new DefaultStopStrategy(); + } + } +} + +interface StopStrategy { + /** + * Stop a service. + * + * @param service The service to stop + */ + stop(service: Service): Promise; +} + +class StopCommandStopStrategy implements StopStrategy { + constructor( + private readonly command: string, + ) { + } + + stop = async (service: Service) => { + const runningService = getRunningService(service.serviceId); + if (!runningService) { + throw new ServiceNotRunningError(service.serviceId); + } + + const callEngine = createEngineCaller( + "stop", + (e) => ({ id: service.serviceId, error: e }) + ); + await callEngine(() => engine.cmd(runningService.internalSession.containerId, this.command)); + } +} + +class DefaultStopStrategy implements StopStrategy { + stop = async (service: Service) => { + const runningService = getRunningService(service.serviceId); + if (!runningService) { + throw new ServiceNotRunningError(service.serviceId); + } + + const callEngine = createEngineCaller( + "stop", + (e) => ({ id: service.serviceId, error: e }) + ); + await callEngine(() => engine.stop(runningService.internalSession.containerId)); + } +} + interface ServiceRunnerEventBus { on(evt: T, h: EventHandler): void; } @@ -147,6 +213,7 @@ export let engine: ServiceEngine; let nodeId: string; let templateManager: TemplateManager; let serviceManager: ServiceManager; +let stopStrategyProvider: StopStrategyProvider; let db: Database; let logger: winston.Logger; @@ -183,6 +250,7 @@ export const init = async ( nodeId = appConfig.getNodeId(); templateManager = templateManager_; serviceManager = serviceManager_; + stopStrategyProvider = new MetaStopStrategyProvider(); db = db_; logger = logger_; @@ -390,6 +458,22 @@ export const resumeService: ServiceRunner["resumeService"] = async (id) => { return new AsyncTask(task); } +const createEngineCaller = ( + action: T, + onErrorEventFactory: (e: Error) => ServiceRunnerEvents[T] +) => { + return async (task: () => Promise) => { + try { + await task(); + } catch (e) { + logger.error(e); + callManagerEvent(action, onErrorEventFactory(e)); + + throw new ServiceEngineError(e); + } + } +} + export const stopService: ServiceRunner["stopService"] = async (id, force) => { const service = await serviceManager.getService(id); if (!service) { @@ -401,18 +485,7 @@ export const stopService: ServiceRunner["stopService"] = async (id, force) => { throw new ServiceNotRunningError(id); } - const callEngine = async (task: () => Promise) => { - try { - await task(); - } catch (e) { - logger.error(e); - callManagerEvent("stop", { id, error: e }); - - throw new ServiceEngineError(e); - } - } - - const internalSession = runningService.internalSession; + const callEngine = createEngineCaller("stop", (e) => ({ id, error: e })); let awaitingPromise: Promise; if (force) { @@ -422,7 +495,7 @@ export const stopService: ServiceRunner["stopService"] = async (id, force) => { throw new ServicePendingActionError(id, pendingAction); } - await callEngine(async () => engine.kill(internalSession.containerId, buildMetaStorage(id))); + await callEngine(async () => engine.kill(runningService.internalSession.containerId, buildMetaStorage(id))); // resolves immediately on kill awaitingPromise = Promise.resolve(); } else { @@ -446,17 +519,8 @@ export const stopService: ServiceRunner["stopService"] = async (id, force) => { }); }); - // TODO: stop strategy - const stopCmd = service.meta?.stopCmd; - await callEngine(async () => { - if (stopCmd) { - // send stop cmd if set - await engine.cmd(internalSession.containerId, stopCmd); - } else { - // send stop signal - await engine.stop(internalSession.containerId); - } - }); + const stopStrategy = await stopStrategyProvider.getStopStrategy(service); + await stopStrategy.stop(service); } awaitingPromise = awaitingPromise.then(() => waitForStopped(id)); From 5eb0573f75f06e35d4b94c66c5646412557f8f21 Mon Sep 17 00:00:00 2001 From: ZorTik Date: Mon, 15 Jun 2026 02:22:04 +0200 Subject: [PATCH 31/53] refactor --- dev/templates/nginx/.nsmignore | 2 + dev/templates/nginx/Dockerfile | 14 ++++++ dev/templates/nginx/settings.yml | 12 +++++ docker-compose.yml | 2 +- openapi.yml | 46 +++----------------- src/app.ts | 4 +- src/engine/facade.ts | 7 +-- src/engine/image.ts | 2 +- src/engine/runner.ts | 14 +++++- src/engine/service.ts | 2 +- src/engine/session.ts | 2 +- src/{database => persistence}/image.ts | 2 +- src/{database => persistence}/index.ts | 0 src/{database => persistence}/meta.ts | 2 +- src/{database => persistence}/models.ts | 0 src/{database => persistence}/perma.ts | 2 +- src/{database => persistence}/serviceLog.ts | 2 +- src/{database => persistence}/serviceMeta.ts | 2 +- src/{database => persistence}/session.ts | 2 +- src/router/v1/service/logsRoute.ts | 2 +- src/router/v1/status/index.ts | 2 +- tests/database/manager.test.ts | 2 +- tests/engine/image.test.ts | 2 +- 23 files changed, 66 insertions(+), 61 deletions(-) create mode 100644 dev/templates/nginx/.nsmignore create mode 100644 dev/templates/nginx/Dockerfile create mode 100644 dev/templates/nginx/settings.yml rename src/{database => persistence}/image.ts (97%) rename src/{database => persistence}/index.ts (100%) rename src/{database => persistence}/meta.ts (90%) rename src/{database => persistence}/models.ts (100%) rename src/{database => persistence}/perma.ts (97%) rename src/{database => persistence}/serviceLog.ts (93%) rename src/{database => persistence}/serviceMeta.ts (92%) rename src/{database => persistence}/session.ts (94%) diff --git a/dev/templates/nginx/.nsmignore b/dev/templates/nginx/.nsmignore new file mode 100644 index 0000000..e04c9ca --- /dev/null +++ b/dev/templates/nginx/.nsmignore @@ -0,0 +1,2 @@ +# Define list of ignored files & directories to NOT include in the container, +# as you would do in .gitignore \ No newline at end of file diff --git a/dev/templates/nginx/Dockerfile b/dev/templates/nginx/Dockerfile new file mode 100644 index 0000000..55bdfec --- /dev/null +++ b/dev/templates/nginx/Dockerfile @@ -0,0 +1,14 @@ +FROM nginx + +# Built-in args. Don't change. +ARG SERVICE_ID +ARG SERVICE_PORT +ARG SERVICE_PORTS +ARG SERVICE_RAM +ARG SERVICE_CPU +ARG SERVICE_DISK + +# Optional ones. From settings.yml + +# port1 port2 port3 +EXPOSE $SERVICE_PORTS \ No newline at end of file diff --git a/dev/templates/nginx/settings.yml b/dev/templates/nginx/settings.yml new file mode 100644 index 0000000..f08aedc --- /dev/null +++ b/dev/templates/nginx/settings.yml @@ -0,0 +1,12 @@ +name: "Nginx" +description: "Nginx template" +port_range: + min: 22222 + max: 33333 +defaults: + ram: 512000000 # bytes + cpu: 2 # cores + disk: 2000000000 # bytes +meta: {} +# Optional ENV vars, and their default values +env: {} \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 11131ef..f25d886 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ services: nsm: build: . volumes: - - "./dev/templates/test:/data/resources/templates/test:ro" + - "./dev/templates:/data/resources/templates:ro" - "./tests:/data/tests:ro" ports: - "3000:3000" diff --git a/openapi.yml b/openapi.yml index 2bc5b0b..dd5c295 100644 --- a/openapi.yml +++ b/openapi.yml @@ -186,6 +186,10 @@ components: type: object required: false description: "A map of custom variables mapped to values whose definitions are in settings.yml in template under 'env'." + meta: + type: object + required: false + description: "A map of custom optional variables. string -> string" paths: /v1/status: get: @@ -392,46 +396,6 @@ paths: application/json: schema: $ref: "#/components/schemas/Result" - /v1/service/{serviceId}/stopcmd: - post: - description: "Stop a service using cmd" - parameters: - - name: "serviceId" - in: "path" - required: true - schema: - type: "string" - responses: - "200": - description: "Stop command successfully sent." - content: - application/json: - schema: - $ref: "#/components/schemas/Result" - "400": - description: "Invalid request" - content: - application/json: - schema: - $ref: "#/components/schemas/Result" - "404": - description: "Service not found" - content: - application/json: - schema: - $ref: "#/components/schemas/Result" - "409": - description: "Conflict, service is not running" - content: - application/json: - schema: - $ref: "#/components/schemas/Result" - "500": - description: "Internal server error" - content: - application/json: - schema: - $ref: "#/components/schemas/Result" /v1/service/{serviceId}/delete: post: description: "Delete a service" @@ -556,4 +520,4 @@ paths: # TODO: /v1/service/{serviceId}/sessions # TODO: /v1/service/{serviceId}/logs -# TODO: /v1/session/{sessionId}/logs +# TODO: /v1/session/{sessionId}/logs \ No newline at end of file diff --git a/src/app.ts b/src/app.ts index 37b4908..994313a 100644 --- a/src/app.ts +++ b/src/app.ts @@ -14,10 +14,10 @@ const appConfig = loadAppConfig(); initFileStructure(appConfig); import { Router } from "express"; -import { Database } from "@nsm/database"; +import { Database } from "@nsm/persistence"; import { ServiceManager } from "@nsm/engine"; import loadAppRoutes from "@nsm/router"; -import createDbManager from "@nsm/database"; +import createDbManager from "@nsm/persistence"; import loadSecurity from "@nsm/security"; import createEngine from "@nsm/engine/engine"; import { init as initImageEngine } from "@nsm/engine/image"; diff --git a/src/engine/facade.ts b/src/engine/facade.ts index efe1bff..8ce4880 100644 --- a/src/engine/facade.ts +++ b/src/engine/facade.ts @@ -1,11 +1,10 @@ import {Service} from "@nsm/engine/service"; import {ServiceSession} from "@nsm/engine/session"; import {InternalSession} from "@nsm/engine/runner"; -import {PermaModel} from "@nsm/database"; +import {PermaModel} from "@nsm/persistence"; import * as manager from "@nsm/engine/service"; import * as runner from "@nsm/engine/runner"; -import {getActionType} from "@nsm/engine/asyncp"; export type ServiceInfo = Service & { state: State; @@ -72,13 +71,15 @@ export const getServiceInfo: Facade["getServiceInfo"] = async (from, options) => } export const getServiceState: Facade["getServiceState"] = async (id) => { - if (getActionType(id) === "stop") { + if (runner.isStopping(id)) { return "STOPPING"; } const stage = runner.getServiceStage(id); if (stage) { return stage.state.ready ? "RUNNING" : "BUILDING"; + } else if (runner.isStarting(id)) { + return "BUILDING"; } else { return "STOPPED"; } diff --git a/src/engine/image.ts b/src/engine/image.ts index a60bedc..4e75510 100644 --- a/src/engine/image.ts +++ b/src/engine/image.ts @@ -1,4 +1,4 @@ -import { Database, ImageModel } from "@nsm/database"; +import { Database, ImageModel } from "@nsm/persistence"; import winston from "winston"; import { MessageListener, ServiceEngine } from "@nsm/engine/engine"; import { TemplateManager } from "@nsm/engine/template"; diff --git a/src/engine/runner.ts b/src/engine/runner.ts index 3a037da..5facf54 100644 --- a/src/engine/runner.ts +++ b/src/engine/runner.ts @@ -27,7 +27,7 @@ import { } from "@nsm/engine/error"; import {Service, ServiceManager} from "@nsm/engine/service"; import {TemplateManager} from "@nsm/engine/template"; -import {Database} from "@nsm/database"; +import {Database} from "@nsm/persistence"; import {isDebug} from "@nsm/helpers"; import winston from "winston"; import {AppConfig} from "@nsm/config"; @@ -187,6 +187,10 @@ export interface ServiceRunner extends ServiceRunnerEventBus { isRunning(id: string): boolean; + isStarting(id: string): boolean; + + isStopping(id: string): boolean; + waitForBusyAction(id: string): Promise; waitForStopped(id: string): Promise; @@ -539,6 +543,14 @@ export const isRunning: ServiceRunner["isRunning"] = (id: string) => { return getRunningService(id) != undefined; } +export const isStarting: ServiceRunner["isStarting"] = (id: string) => { + return getActionType(id) === "resume"; +} + +export const isStopping: ServiceRunner["isStopping"] = (id: string) => { + return getActionType(id) === "stop"; +} + /** * Builds the meta storage for a service, which is used for storing and retrieving internal metadata for the service. * diff --git a/src/engine/service.ts b/src/engine/service.ts index 2376693..4729eda 100644 --- a/src/engine/service.ts +++ b/src/engine/service.ts @@ -4,7 +4,7 @@ import { import * as templateManager from "./template"; import crypto from "crypto"; import { randomPort as retrieveRandomPort } from "@nsm/util/port"; -import {Database, ImageModel, PermaModel} from "../database"; +import {Database, ImageModel, PermaModel} from "../persistence"; import { reqNotPending, } from "./asyncp"; diff --git a/src/engine/session.ts b/src/engine/session.ts index c08152c..db06772 100644 --- a/src/engine/session.ts +++ b/src/engine/session.ts @@ -6,7 +6,7 @@ import { ListSessionsArgs, ServiceLogRecordModel, ServiceSessionModel, -} from "@nsm/database"; +} from "@nsm/persistence"; import {ServiceWasNeverActiveError} from "@nsm/engine/error"; export interface SessionManager { diff --git a/src/database/image.ts b/src/persistence/image.ts similarity index 97% rename from src/database/image.ts rename to src/persistence/image.ts index 3c99fc8..5ca3658 100644 --- a/src/database/image.ts +++ b/src/persistence/image.ts @@ -1,4 +1,4 @@ -import { ImageRepository } from "@nsm/database/models"; +import { ImageRepository } from "@nsm/persistence/models"; import { optionsDiffer } from "@nsm/engine/image"; import { PrismaClient } from "@prisma/client"; diff --git a/src/database/index.ts b/src/persistence/index.ts similarity index 100% rename from src/database/index.ts rename to src/persistence/index.ts diff --git a/src/database/meta.ts b/src/persistence/meta.ts similarity index 90% rename from src/database/meta.ts rename to src/persistence/meta.ts index 5d50753..9e2b2dc 100644 --- a/src/database/meta.ts +++ b/src/persistence/meta.ts @@ -1,5 +1,5 @@ import { PrismaClient } from "@prisma/client"; -import { MetaRepository } from "@nsm/database/models"; +import { MetaRepository } from "@nsm/persistence/models"; let client: PrismaClient; diff --git a/src/database/models.ts b/src/persistence/models.ts similarity index 100% rename from src/database/models.ts rename to src/persistence/models.ts diff --git a/src/database/perma.ts b/src/persistence/perma.ts similarity index 97% rename from src/database/perma.ts rename to src/persistence/perma.ts index 0daa297..8eb939a 100644 --- a/src/database/perma.ts +++ b/src/persistence/perma.ts @@ -1,5 +1,5 @@ import { PrismaClient } from "@prisma/client"; -import { PermaModel, PermaRepository } from "@nsm/database/models"; +import { PermaModel, PermaRepository } from "@nsm/persistence/models"; let client: PrismaClient; diff --git a/src/database/serviceLog.ts b/src/persistence/serviceLog.ts similarity index 93% rename from src/database/serviceLog.ts rename to src/persistence/serviceLog.ts index 64bb6bb..1fc5a36 100644 --- a/src/database/serviceLog.ts +++ b/src/persistence/serviceLog.ts @@ -1,5 +1,5 @@ import { Prisma, PrismaClient } from "@prisma/client"; -import { ServiceLogRepository } from "@nsm/database/models"; +import { ServiceLogRepository } from "@nsm/persistence/models"; let client: PrismaClient; diff --git a/src/database/serviceMeta.ts b/src/persistence/serviceMeta.ts similarity index 92% rename from src/database/serviceMeta.ts rename to src/persistence/serviceMeta.ts index 3db62cf..c06617d 100644 --- a/src/database/serviceMeta.ts +++ b/src/persistence/serviceMeta.ts @@ -1,5 +1,5 @@ import { PrismaClient } from "@prisma/client"; -import { ServiceMetaRepository } from "@nsm/database/models"; +import { ServiceMetaRepository } from "@nsm/persistence/models"; let client: PrismaClient; diff --git a/src/database/session.ts b/src/persistence/session.ts similarity index 94% rename from src/database/session.ts rename to src/persistence/session.ts index ae3db43..bd9cf08 100644 --- a/src/database/session.ts +++ b/src/persistence/session.ts @@ -1,5 +1,5 @@ import { Prisma, PrismaClient } from "@prisma/client"; -import { SessionRepository } from "@nsm/database/models"; +import { SessionRepository } from "@nsm/persistence/models"; let client: PrismaClient; diff --git a/src/router/v1/service/logsRoute.ts b/src/router/v1/service/logsRoute.ts index af9de59..6b8c37b 100644 --- a/src/router/v1/service/logsRoute.ts +++ b/src/router/v1/service/logsRoute.ts @@ -1,7 +1,7 @@ import { AppContext } from "@nsm/app"; import { RouterHandler } from "@nsm/router"; import {ServiceWasNeverActiveError} from "@nsm/engine/error"; -import {ServiceLogRecordModel} from "@nsm/database"; +import {ServiceLogRecordModel} from "@nsm/persistence"; export default async function (ctx: AppContext): Promise { return { diff --git a/src/router/v1/status/index.ts b/src/router/v1/status/index.ts index 0114361..a411634 100644 --- a/src/router/v1/status/index.ts +++ b/src/router/v1/status/index.ts @@ -2,7 +2,7 @@ import { AppContext } from "@nsm/app"; import { RouterHandler } from "../../index"; import * as os from "os"; import {Filters, ServiceEngine, ServiceManager} from "@nsm/engine"; -import { Database } from "@nsm/database"; +import { Database } from "@nsm/persistence"; import {parseResourceOptionsSet} from "@nsm/util/services"; async function checkNsmResources( diff --git a/tests/database/manager.test.ts b/tests/database/manager.test.ts index 2c3ccd7..afce7ed 100644 --- a/tests/database/manager.test.ts +++ b/tests/database/manager.test.ts @@ -1,6 +1,6 @@ import { afterEach, beforeEach, expect, it } from "@jest/globals"; import { StartedMariaDbContainer } from "@testcontainers/mariadb"; -import getDb, { Database } from "@nsm/database"; +import getDb, { Database } from "@nsm/persistence"; import { PrismaClient } from "@prisma/client"; import { initDbContainerForTest } from "../testUtils"; diff --git a/tests/engine/image.test.ts b/tests/engine/image.test.ts index 8ec43a1..d15bb31 100644 --- a/tests/engine/image.test.ts +++ b/tests/engine/image.test.ts @@ -5,7 +5,7 @@ import {processImage } from "@nsm/engine/image"; import {prepareEnvForTemplate, Template, TemplateManager} from "@nsm/engine/template"; import {TemplateDirWatcher} from "@nsm/engine/monitoring/templateDirWatcher"; import {DeepMockProxy, mock, mockDeep} from "jest-mock-extended"; -import {Database, ImageModel} from "@nsm/database"; +import {Database, ImageModel} from "@nsm/persistence"; import {createTestLogger} from "../testUtils"; import {AppConfig} from "@nsm/config"; From d481bfed6450970d42da63030321b8fc0e4c4e26 Mon Sep 17 00:00:00 2001 From: ZorTik Date: Mon, 15 Jun 2026 13:06:03 +0200 Subject: [PATCH 32/53] fix --- src/engine/docker/action/deletev.ts | 4 +- src/engine/docker/action/kill.ts | 2 +- src/engine/engine.ts | 12 +++++- src/engine/facade.ts | 1 + src/engine/middle.ts | 9 ++++- src/engine/runner.ts | 58 +++++++++++++++++++++++++++-- src/engine/session.ts | 8 +++- src/router/v1/index.ts | 2 + 8 files changed, 86 insertions(+), 10 deletions(-) diff --git a/src/engine/docker/action/deletev.ts b/src/engine/docker/action/deletev.ts index 773496c..389c83c 100644 --- a/src/engine/docker/action/deletev.ts +++ b/src/engine/docker/action/deletev.ts @@ -11,7 +11,9 @@ export default function ( await client.getVolume(id).remove(); return true; } catch (e) { - currentContext.logger.error(e); + if (!e.message.includes("no such volume")) { + currentContext.logger.error(e); + } return false; } }; diff --git a/src/engine/docker/action/kill.ts b/src/engine/docker/action/kill.ts index 8651ea4..5ce9912 100644 --- a/src/engine/docker/action/kill.ts +++ b/src/engine/docker/action/kill.ts @@ -11,7 +11,7 @@ export default function (client: DockerClient): ServiceEngine["kill"] { return true; } catch (e) { - if (!e.message.includes("container is not running")) { + if (!e.message.includes("is not running")) { console.log(e); } return false; diff --git a/src/engine/engine.ts b/src/engine/engine.ts index 08b665a..3c9e54d 100644 --- a/src/engine/engine.ts +++ b/src/engine/engine.ts @@ -170,10 +170,9 @@ export type ServiceEngine = { * Kills a container. * * @param id Container ID - * @param meta Meta storage for this unique context * @return Success state */ - kill(id: string, meta: MetaStorage): Promise; + kill(id: string): Promise; /** * Reattaches to a container. @@ -284,6 +283,15 @@ export const Filters = { }, }; }, + + service(serviceId: string) { + return { + labels: { + ...this.nsm().labels, + [StandardLabel.ServiceId]: serviceId, + } + } + } }; /** diff --git a/src/engine/facade.ts b/src/engine/facade.ts index 8ce4880..45e168b 100644 --- a/src/engine/facade.ts +++ b/src/engine/facade.ts @@ -49,6 +49,7 @@ export const deleteService: Facade["deleteService"] = async (id) => { await runner.stopService(id, true); } + await runner.clearService(id); await manager.deleteService(id); } diff --git a/src/engine/middle.ts b/src/engine/middle.ts index 4abdf4f..53e7025 100644 --- a/src/engine/middle.ts +++ b/src/engine/middle.ts @@ -6,7 +6,8 @@ import {AppConfig} from "@nsm/config"; export type ServiceActionType = | "resume" - | "stop"; + | "stop" + | "clear"; /** * Represents an error that occurred during a service action. @@ -176,5 +177,11 @@ export const middleLayer = (runner: ServiceRunner): ServiceRunner => { "stop", argServiceIdExtractor(0), ), + + clearService: decorateFunc( + runner.clearService, + "clear", + argServiceIdExtractor(0), + ) }; }; diff --git a/src/engine/runner.ts b/src/engine/runner.ts index 5facf54..8436c7e 100644 --- a/src/engine/runner.ts +++ b/src/engine/runner.ts @@ -149,6 +149,13 @@ export interface ServiceRunner extends ServiceRunnerEventBus { */ stopService(id: string, force?: boolean): Promise>; + /** + * Clear a service, that is, delete all its resources. + * + * @param id The service ID + */ + clearService(id: string): Promise; + /** * Get list of running services on this node. */ @@ -296,16 +303,29 @@ const reattachStaleContainers = async (logger: winston.Logger) => { // The container was in the running list, but does not have the required labels // Should not happen, but just in case logger.warn( - `Found a running container with id ${containerId} that does not have a service id label, stopping.`, + `Found a running container with id ${containerId} that does not have a service id label, killing.`, ); - await engine.stop(containerId); + await engine.kill(containerId); + continue; } const serviceId = labels[StandardLabel.ServiceId]; // We must begin a new session since the previous was interrupted - const session = await beginServiceSession(serviceId); + let session: ActiveServiceSession; + try { + session = await beginServiceSession(serviceId); + } catch (e) { + if (e instanceof ServiceNotFoundError) { + logger.warn( + `Found a running container ${containerId} for service ${serviceId}, but the service was not found + in database, killing the container and clearing resources.`, + ); + await clearService(serviceId); + continue; + } + } // Reattach and watch the container await engine.reattach(containerId, buildRunListener(session)); @@ -499,7 +519,7 @@ export const stopService: ServiceRunner["stopService"] = async (id, force) => { throw new ServicePendingActionError(id, pendingAction); } - await callEngine(async () => engine.kill(runningService.internalSession.containerId, buildMetaStorage(id))); + await callEngine(async () => engine.kill(runningService.internalSession.containerId)); // resolves immediately on kill awaitingPromise = Promise.resolve(); } else { @@ -531,6 +551,36 @@ export const stopService: ServiceRunner["stopService"] = async (id, force) => { return new AsyncTask(awaitingPromise); } +export const clearService: ServiceRunner["clearService"] = async (id) => { + try { + await stopService(id, true); + } catch (e) { + if (e instanceof ServiceNotFoundError || e instanceof ServiceNotRunningError) { + // ignore + } else { + throw e; + } + } + + const containerIds = await engine.listContainers(Filters.service(id)); + for (let containerId of containerIds) { + try { + await engine.kill(containerId); + } catch (e) { + throw new ServiceEngineError(e); + } + } + + try { + const deleted = await engine.deleteVolume(id); + if (!deleted) { + logger.warn(`Failed to delete volume for service ${id}`); + } + } catch (e) { + throw new ServiceEngineError(e); + } +} + export const getRunningService: ServiceRunner["getRunningService"] = (id) => { return started.find((service) => service.id === id); } diff --git a/src/engine/session.ts b/src/engine/session.ts index db06772..1109add 100644 --- a/src/engine/session.ts +++ b/src/engine/session.ts @@ -7,7 +7,7 @@ import { ServiceLogRecordModel, ServiceSessionModel, } from "@nsm/persistence"; -import {ServiceWasNeverActiveError} from "@nsm/engine/error"; +import {ServiceNotFoundError, ServiceWasNeverActiveError} from "@nsm/engine/error"; export interface SessionManager { /** @@ -81,9 +81,15 @@ export const init = (db_: Database) => { * * @param serviceId The ID of the service for which to begin a session. * @return An object representing the active service session. + * @throws ServiceNotFoundError if the service with the given ID does not exist. */ export const beginServiceSession: SessionManager["beginServiceSession"] = async (serviceId: string): Promise => { + const perma = await db.permaRepository.getPerma(serviceId); + if (!perma) { + throw new ServiceNotFoundError(serviceId); + } + let session = await db.sessionRepository.createSession(serviceId); // Debounce the push in bulk to prevent database overhead diff --git a/src/router/v1/index.ts b/src/router/v1/index.ts index a5352fd..697d162 100644 --- a/src/router/v1/index.ts +++ b/src/router/v1/index.ts @@ -11,6 +11,7 @@ import optionsRoute from "@nsm/router/v1/service/optionsRoute"; import sessionsRoute from "@nsm/router/v1/service/sessionsRoute"; import sessionLogsRoute from "@nsm/router/v1/session/sessionLogsRoute"; import logsRoute from "@nsm/router/v1/service/logsRoute"; +import templateListRoute from "@nsm/router/v1/template/listRoute"; export default [ // v1 routes @@ -27,4 +28,5 @@ export default [ sessionsRoute, logsRoute, sessionLogsRoute, + templateListRoute, ]; From cb85d652a265eefdae76d0e632c90c3fe9171515 Mon Sep 17 00:00:00 2001 From: ZorTik Date: Mon, 15 Jun 2026 15:16:33 +0200 Subject: [PATCH 33/53] refactor: extract prepareEnvForTemplate from TemplateManager --- src/app.ts | 5 ++--- src/engine/engine.ts | 11 +++++++++-- src/engine/image.ts | 4 ++-- src/engine/template.ts | 22 ++++++++++------------ tests/engine/image.test.ts | 15 +++++++++++---- 5 files changed, 34 insertions(+), 23 deletions(-) diff --git a/src/app.ts b/src/app.ts index 994313a..748a96c 100644 --- a/src/app.ts +++ b/src/app.ts @@ -15,11 +15,10 @@ initFileStructure(appConfig); import { Router } from "express"; import { Database } from "@nsm/persistence"; -import { ServiceManager } from "@nsm/engine"; +import {initEngine, ServiceManager} from "@nsm/engine"; import loadAppRoutes from "@nsm/router"; import createDbManager from "@nsm/persistence"; import loadSecurity from "@nsm/security"; -import createEngine from "@nsm/engine/engine"; import { init as initImageEngine } from "@nsm/engine/image"; import * as facade from "@nsm/engine/facade"; import * as manager from "@nsm/engine/service"; @@ -106,7 +105,7 @@ export const init = async ( await registerErrorPublishersFromConfig(appConfig); - const engine = createEngine(appConfig); + const engine = initEngine(appConfig); logger.info(`Using engine: ${engine.name}`); initImageEngine(engine, templateManager, templateDirWatcher, database, appConfig, logger); diff --git a/src/engine/engine.ts b/src/engine/engine.ts index 3c9e54d..d6fc199 100644 --- a/src/engine/engine.ts +++ b/src/engine/engine.ts @@ -319,7 +319,14 @@ export const combineRunListeners = (listeners: RunListener[]): RunListener => { }; }; -export default function (appConfig: AppConfig): ServiceEngineI { +/** + * Initializes the service engine based on the configuration. + * + * @param appConfig The application configuration to use for initializing the engine. + * @returns The initialized service engine instance. + * @throws Error if the engine ID specified in the configuration is invalid. + */ +export const initEngine = (appConfig: AppConfig): ServiceEngineI => { let engine = getSingleton("engine"); if (!engine) { const engineId = process.env.NSM_ENGINE ?? "docker"; @@ -335,4 +342,4 @@ export default function (appConfig: AppConfig): ServiceEngineI { cast: undefined, // Being set in manager ...engine, }; -} +} \ No newline at end of file diff --git a/src/engine/image.ts b/src/engine/image.ts index 4e75510..053306f 100644 --- a/src/engine/image.ts +++ b/src/engine/image.ts @@ -1,7 +1,7 @@ import { Database, ImageModel } from "@nsm/persistence"; import winston from "winston"; import { MessageListener, ServiceEngine } from "@nsm/engine/engine"; -import { TemplateManager } from "@nsm/engine/template"; +import { prepareEnvForTemplate, TemplateManager } from "@nsm/engine/template"; import { TemplateDirWatcher } from "@nsm/engine/monitoring/templateDirWatcher"; import { AppConfig } from "@nsm/config"; @@ -52,7 +52,7 @@ export const processImage = async ( ) => { const template = templateManager.getTemplate(templateId); // Checks if the provided options are still compatible with the template - buildOptions = templateManager.prepareEnvForTemplate(template, buildOptions); + buildOptions = prepareEnvForTemplate(template, buildOptions); if (!id) { // No image specified, need to build or pick a new one diff --git a/src/engine/template.ts b/src/engine/template.ts index b9e6a73..eef5d40 100644 --- a/src/engine/template.ts +++ b/src/engine/template.ts @@ -44,18 +44,6 @@ export type TemplateSettings = { } export type TemplateManager = { - /** - * Prepares the environment variables for a template by validating the provided env object against - * the template's settings and filling in default values where necessary. It checks for required options, validates - * types, and returns a new env object that can be used when creating a service from the template. - * - * @param template The template or template ID for which to prepare the environment variables - * @param env The environment variables provided by the user, which may be incomplete or have incorrect types - * @return A new env object that has been validated and filled with default values according to the template's settings - * @throws Error if a required option is missing or if an option has an invalid type - */ - prepareEnvForTemplate(template: Template | string, env: any): any; - /** * Returns a template by ID. * @@ -102,6 +90,16 @@ export const getAllTemplates = () => { .filter((template) => template !== null); }; +/** + * Prepares the environment variables for a template by validating the provided env object against + * the template's settings and filling in default values where necessary. It checks for required options, validates + * types, and returns a new env object that can be used when creating a service from the template. + * + * @param template The template or template ID for which to prepare the environment variables + * @param env The environment variables provided by the user, which may be incomplete or have incorrect types + * @return A new env object that has been validated and filled with default values according to the template's settings + * @throws Error if a required option is missing or if an option has an invalid type + */ export const prepareEnvForTemplate = ( template: Template | string, env: any, diff --git a/tests/engine/image.test.ts b/tests/engine/image.test.ts index d15bb31..a4bdad4 100644 --- a/tests/engine/image.test.ts +++ b/tests/engine/image.test.ts @@ -2,7 +2,7 @@ import {expect, it} from "@jest/globals"; import {ServiceEngine} from "@nsm/engine"; import {init as initImageEngine} from "@nsm/engine/image"; import {processImage } from "@nsm/engine/image"; -import {prepareEnvForTemplate, Template, TemplateManager} from "@nsm/engine/template"; +import {Template, TemplateManager} from "@nsm/engine/template"; import {TemplateDirWatcher} from "@nsm/engine/monitoring/templateDirWatcher"; import {DeepMockProxy, mock, mockDeep} from "jest-mock-extended"; import {Database, ImageModel} from "@nsm/persistence"; @@ -15,10 +15,20 @@ it("reuses image with same options", async () => { name: "idk", description: "idk more", settings: { + port_range: { + min: 1000, + max: 2000, + }, env: { option1: "", option2: "", }, + defaults: { + cpu: 1, + disk: 100000, + ram: 512000000, + }, + meta: {} }, }; @@ -29,9 +39,6 @@ it("reuses image with same options", async () => { imageId ?? "generated-image-id-" + (Math.random() * 1000000).toFixed(0)); const templateManagerMock = mock(); - templateManagerMock - .prepareEnvForTemplate - .mockImplementation((template, env) => prepareEnvForTemplate(template, env)); templateManagerMock .getTemplate .mockImplementation((id) => id === "test-template" ? template : null) From 4d168df455ae20f923601fc3e72331eb5db3ab79 Mon Sep 17 00:00:00 2001 From: ZorTik Date: Mon, 15 Jun 2026 15:28:19 +0200 Subject: [PATCH 34/53] feat: nested env config filling --- src/config.ts | 37 ++++++++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/src/config.ts b/src/config.ts index af82545..b14c63b 100644 --- a/src/config.ts +++ b/src/config.ts @@ -95,10 +95,41 @@ export class YamlAppConfig implements AppConfig { saveResource("config.yml", "config.yml", true, currentPaths.config); const config = loadYamlFile(path.join(currentPaths.config, "config.yml")); - for (let key in YamlAppConfig.schema.shape) { + + return YamlAppConfig.fillDataFromEnv(YamlAppConfig.schema.shape, config); + }; + + + /** + * Recursively fills config data from environment variables. + * + * @param shape The shape of the config schema, used to determine which keys to look for in env variables. + * @param config The config object to fill with env variables. + * @param envPrefix The prefix to use for env variables, default is "CONFIG_". For nested objects, the prefix will be extended with the parent key in uppercase followed by an underscore. + * @returns The config object filled with env variables where applicable. + */ + private static fillDataFromEnv = ( + shape: z.ZodObject, + config: any, + envPrefix?: string, + ) => { + const prefix = envPrefix ?? "CONFIG_"; + + for (let key in shape) { + const envKey = prefix + key.toUpperCase(); + + // Recursively fill nested objects + if (shape[key] instanceof z.ZodObject) { + config[key] = YamlAppConfig.fillDataFromEnv( + shape[key].shape, + config[key] || {}, + envKey + "_", + ); + continue; + } + // Overwrite with env variable if exists. // Sync - const envKey = "CONFIG_" + key.toUpperCase(); if (process.env[envKey]) { config[key] = process.env[envKey]; } else if (config[key]) { @@ -106,7 +137,7 @@ export class YamlAppConfig implements AppConfig { } } return config; - }; + } } export const loadAppConfig = (): AppConfig => { From 93987c19d08c80a9c26364ab0c9a8d4bdece957f Mon Sep 17 00:00:00 2001 From: ZorTik Date: Mon, 15 Jun 2026 17:23:31 +0200 Subject: [PATCH 35/53] feat: new template system --- resources/config.yml | 3 + src/app.ts | 9 +- src/config.ts | 21 +++ src/engine/docker/index.ts | 2 + .../repository/filesystem}/image.ts | 21 +-- .../monitoring/templateDirWatcher.ts | 4 +- .../repository/filesystem}/monitoring/util.ts | 0 .../docker/repository/filesystem/template.ts | 45 ++++++ src/engine/docker/template.ts | 137 ++++++++++++++++++ src/engine/engine.ts | 83 ++++++++++- src/engine/error.ts | 11 ++ src/engine/runner.ts | 10 +- src/engine/service.ts | 15 +- src/engine/template.ts | 81 +++++------ src/persistence/image.ts | 2 +- src/router/middlewares/catchKnownErrors.ts | 2 + src/router/v1/service/createRoute.ts | 2 +- tests/engine/image.test.ts | 18 +-- 18 files changed, 377 insertions(+), 89 deletions(-) rename src/engine/{ => docker/repository/filesystem}/image.ts (89%) rename src/engine/{ => docker/repository/filesystem}/monitoring/templateDirWatcher.ts (96%) rename src/engine/{ => docker/repository/filesystem}/monitoring/util.ts (100%) create mode 100644 src/engine/docker/repository/filesystem/template.ts create mode 100644 src/engine/docker/template.ts diff --git a/resources/config.yml b/resources/config.yml index 53b08fd..67c20a0 100644 --- a/resources/config.yml +++ b/resources/config.yml @@ -12,3 +12,6 @@ docker_host: "unix:///var/run/docker.sock" # Override resources path if needed. # By default, an explicit system-specific data path is used. # resources_path: '/srv/resources' +repositories: + - id: local + type: filesystem \ No newline at end of file diff --git a/src/app.ts b/src/app.ts index 748a96c..6bffeed 100644 --- a/src/app.ts +++ b/src/app.ts @@ -19,13 +19,11 @@ import {initEngine, ServiceManager} from "@nsm/engine"; import loadAppRoutes from "@nsm/router"; import createDbManager from "@nsm/persistence"; import loadSecurity from "@nsm/security"; -import { init as initImageEngine } from "@nsm/engine/image"; import * as facade from "@nsm/engine/facade"; import * as manager from "@nsm/engine/service"; import * as runner from "@nsm/engine/runner"; import * as sessionManager from "@nsm/engine/session"; import * as templateManager from "@nsm/engine/template"; -import * as templateDirWatcher from "@nsm/engine/monitoring/templateDirWatcher"; import * as logging from "./logger"; import winston from "winston"; import { Application } from "express-ws"; @@ -36,8 +34,8 @@ import { mkdirResource, saveResource } from "@nsm/resources"; import path from "path"; import { AppConfig } from "@nsm/config"; import { ServiceRunner } from "@nsm/engine/runner"; -import {TemplateManager} from "@nsm/engine/template"; import {Facade} from "@nsm/engine/facade"; +import {TemplateManager} from "@nsm/engine/template"; // Passed context to the routes export type AppContext = { @@ -105,14 +103,13 @@ export const init = async ( await registerErrorPublishersFromConfig(appConfig); - const engine = initEngine(appConfig); + const engine = await initEngine(ctx); logger.info(`Using engine: ${engine.name}`); - initImageEngine(engine, templateManager, templateDirWatcher, database, appConfig, logger); + templateManager.init(engine); sessionManager.init(database); await ctx.manager.init(appConfig, database, engine, logger); - templateDirWatcher.watchTemplateDirChanges(logger); await runner.init(engine, appConfig, templateManager, manager, database, logger); ctx.runner = currentContext.runner = middleLayer(runner); diff --git a/src/config.ts b/src/config.ts index b14c63b..151ad1f 100644 --- a/src/config.ts +++ b/src/config.ts @@ -3,6 +3,7 @@ import path from "path"; import { saveResource } from "@nsm/resources"; import z from "zod"; import envPaths, {Paths} from "env-paths"; +import {TemplateRepositoryConfig} from "@nsm/engine"; export const currentPaths: Paths = envPaths("nsm"); @@ -22,6 +23,8 @@ export interface AppConfig { getTemplateBuildDir(template: string): string; getTempPath(): string; + + getTemplateRepositoryConfigs(): TemplateRepositoryConfig[]; } /** @@ -38,6 +41,12 @@ export class YamlAppConfig implements AppConfig { auth: z.string(), docker_host: z.string(), resources_path: z.string().optional(), + repositories: z.array( + z.object({ + id: z.string(), + type: z.string(), + }).passthrough() + ) }) .strict(); @@ -83,6 +92,18 @@ export class YamlAppConfig implements AppConfig { return currentPaths.temp; } + getTemplateRepositoryConfigs(): TemplateRepositoryConfig[] { + const repositories: any[] = this.data["repositories"]; + + return repositories.map((repo) => { + return { + id: repo.id, + type: repo.type, + config: repo, + }; + }); + } + private validate = () => { const result = YamlAppConfig.schema.safeParse(this.data); if (!result.success) { diff --git a/src/engine/docker/index.ts b/src/engine/docker/index.ts index dae08ec..057d3ca 100644 --- a/src/engine/docker/index.ts +++ b/src/engine/docker/index.ts @@ -17,6 +17,7 @@ import statAll from "./action/statall"; import calcHostUsage from "./action/calcHostUsage"; import listRunning from "./action/listRunning"; import { AppConfig } from "@nsm/config"; +import {DockerTemplateRepositoryRegistry} from "@nsm/engine/docker/template"; export default function buildDockerEngine(appConfig: AppConfig) { // Default engine implementation @@ -25,6 +26,7 @@ export default function buildDockerEngine(appConfig: AppConfig) { engine.name = "Docker"; engine.dockerClient = client; engine.rws = {}; + engine.templateRepositoryRegistry = new DockerTemplateRepositoryRegistry(engine); // engine.cast - Being replaced in manager. engine.build = build(client); engine.run = run(engine, client); diff --git a/src/engine/image.ts b/src/engine/docker/repository/filesystem/image.ts similarity index 89% rename from src/engine/image.ts rename to src/engine/docker/repository/filesystem/image.ts index 053306f..9dc6e4a 100644 --- a/src/engine/image.ts +++ b/src/engine/docker/repository/filesystem/image.ts @@ -1,8 +1,8 @@ import { Database, ImageModel } from "@nsm/persistence"; import winston from "winston"; import { MessageListener, ServiceEngine } from "@nsm/engine/engine"; -import { prepareEnvForTemplate, TemplateManager } from "@nsm/engine/template"; -import { TemplateDirWatcher } from "@nsm/engine/monitoring/templateDirWatcher"; +import { Template } from "@nsm/engine/template"; +import { TemplateDirWatcher } from "@nsm/engine/docker/repository/filesystem/monitoring/templateDirWatcher"; import { AppConfig } from "@nsm/config"; type BuildOptionsMap = { @@ -10,7 +10,6 @@ type BuildOptionsMap = { }; let engine: ServiceEngine; -let templateManager: TemplateManager; let templateDirWatcher: TemplateDirWatcher; let appConfig: AppConfig; let db: Database; @@ -18,14 +17,12 @@ let logger: winston.Logger; export const init = ( engine_: ServiceEngine, - templateManager_: TemplateManager, templateDirWatcher_: TemplateDirWatcher, db_: Database, appConfig_: AppConfig, logger_: winston.Logger, ) => { engine = engine_; - templateManager = templateManager_; templateDirWatcher = templateDirWatcher_; db = db_; appConfig = appConfig_; @@ -46,23 +43,19 @@ export const init = ( */ export const processImage = async ( id: string | undefined | null, - templateId: string, + template: Template, buildOptions: BuildOptionsMap, messageListener?: MessageListener, ) => { - const template = templateManager.getTemplate(templateId); - // Checks if the provided options are still compatible with the template - buildOptions = prepareEnvForTemplate(template, buildOptions); - if (!id) { // No image specified, need to build or pick a new one - id = await pickImageOrBuild(templateId, buildOptions); + id = await pickImageOrBuild(template.id, buildOptions); } const imageModel = await getImage(id); - if (imageModel.templateId != templateId) { + if (imageModel.templateId != template.id) { throw new Error( - `Image ${id} is based on template ${imageModel.templateId}, but template ${templateId} was expected`, + `Image ${id} is based on template ${imageModel.templateId}, but template ${template.id} was expected`, ); } @@ -76,7 +69,7 @@ export const processImage = async ( logger.info( `The target options differ, finding or building a new compatible image...`, ); - id = await pickImageOrBuild(templateId, buildOptions); + id = await pickImageOrBuild(template.id, buildOptions); // If the image becomes unused after the switch, delete it await deleteImageIfUnused(imageModel); diff --git a/src/engine/monitoring/templateDirWatcher.ts b/src/engine/docker/repository/filesystem/monitoring/templateDirWatcher.ts similarity index 96% rename from src/engine/monitoring/templateDirWatcher.ts rename to src/engine/docker/repository/filesystem/monitoring/templateDirWatcher.ts index 0069a5a..c69c28d 100644 --- a/src/engine/monitoring/templateDirWatcher.ts +++ b/src/engine/docker/repository/filesystem/monitoring/templateDirWatcher.ts @@ -1,11 +1,11 @@ -import { debounce } from "@nsm/engine/monitoring/util"; +import { debounce } from "@nsm/engine/docker/repository/filesystem/monitoring/util"; import { hashElement } from "folder-hash"; import { getFilteredPaths } from "@nsm/engine/ignore"; -import { getAllTemplates } from "@nsm/engine/template"; import winston from "winston"; import chokidar, { FSWatcher } from "chokidar"; import path from "path"; import {getTemplateBuildDir, getTemplatesPath} from "@nsm/filestructure"; +import {getAllTemplates} from "@nsm/engine/docker/repository/filesystem/template"; export type TemplateDirWatcher = { /** diff --git a/src/engine/monitoring/util.ts b/src/engine/docker/repository/filesystem/monitoring/util.ts similarity index 100% rename from src/engine/monitoring/util.ts rename to src/engine/docker/repository/filesystem/monitoring/util.ts diff --git a/src/engine/docker/repository/filesystem/template.ts b/src/engine/docker/repository/filesystem/template.ts new file mode 100644 index 0000000..ab5a74f --- /dev/null +++ b/src/engine/docker/repository/filesystem/template.ts @@ -0,0 +1,45 @@ +import path from "path"; +import {getTemplatesPath} from "@nsm/filestructure"; +import fs from "fs"; +import {loadYamlFile} from "@nsm/util/yaml"; +import {Template} from "@nsm/engine/template"; + +export type FileSystemTemplateManager = { + /** + * Returns a template by ID. + * + * @param id The ID of the template + * @return The template, or null if not exists + */ + getTemplate(id: string): Template | null; + + getAllTemplates(): Template[]; +}; + +export const getTemplate = (id: string): Template | null => { + const settingsPath = path.join(getTemplatesPath(), id, "settings.yml"); + if (!fs.existsSync(settingsPath)) { + return null; + } + const settings = loadYamlFile(settingsPath); + return { + id, + name: settings.name, + description: settings.description, + settings, + }; +}; + +export const getAllTemplates = () => { + if (!fs.existsSync(getTemplatesPath())) { + return []; + } + + return fs + .readdirSync(getTemplatesPath()) + .filter((file) => + fs.statSync(path.join(getTemplatesPath(), file)).isDirectory(), + ) + .map((id) => getTemplate(id)) + .filter((template) => template !== null); +}; \ No newline at end of file diff --git a/src/engine/docker/template.ts b/src/engine/docker/template.ts new file mode 100644 index 0000000..6342d74 --- /dev/null +++ b/src/engine/docker/template.ts @@ -0,0 +1,137 @@ +import { + BuildOptionsMap, MessageListener, ServiceEngine, + TemplateRepository, + TemplateRepositoryConfig, + TemplateRepositoryRegistry +} from "@nsm/engine"; +import {AppContext} from "@nsm/app"; +import {init as initImageEngine, processImage} from "@nsm/engine/docker/repository/filesystem/image"; +import {getAllTemplates} from "@nsm/engine/docker/repository/filesystem/template"; +import {TemplateNotFoundError, TemplateRepositoryConfigurationError} from "@nsm/engine/error"; +import * as templateDirWatcher from "@nsm/engine/docker/repository/filesystem/monitoring/templateDirWatcher"; +import path from "path"; +import fs from "fs"; +import {loadYamlFile} from "@nsm/util/yaml"; +import {Template} from "@nsm/engine/template"; +import winston from "winston"; + +type RepositoryRegistration = { + id: string; + repository: TemplateRepository; +} + +/** + * A template repository that loads templates from the filesystem. + * The template dir is determined from the app config. + * + * @author ZorTik + */ +class FilesystemTemplateRepository implements TemplateRepository { + private readonly templateCache: Map; + private readonly templateHashCache: Map; + + private templatesPath: string; + private logger: winston.Logger; + + constructor( + private readonly engine: ServiceEngine, + ) { + this.templateCache = new Map(); + this.templateHashCache = new Map(); + } + + async init(ctx: AppContext) { + this.templatesPath = ctx.appConfig.getTemplatesPath(); + this.logger = ctx.logger; + + initImageEngine(this.engine, templateDirWatcher, ctx.database, ctx.appConfig, ctx.logger); + templateDirWatcher.watchTemplateDirChanges(ctx.logger); + } + + async buildImage( + templateId: string, + options: BuildOptionsMap, + imageId?: string, + messageListener?: MessageListener + ) { + const template = await this.getTemplate(templateId); + if (template) { + return processImage(imageId, template, options, messageListener); + } else { + throw new TemplateNotFoundError(templateId); + } + } + + async getTemplate(id: string) { + if (this.templateCache.has(id) + && this.templateHashCache.has(id) + // template didn't change, so we can be sure that settings.yml didn't as well + && this.templateHashCache.get(id) === templateDirWatcher.getTemplateHash(id)) { + return this.templateCache.get(id); + } + + const settingsPath = path.join(this.templatesPath, id, "settings.yml"); + if (!fs.existsSync(settingsPath)) { + return undefined; + } + + const settings = loadYamlFile(settingsPath); + const template: Template = { + id, + name: settings.name, + description: settings.description, + settings, + }; + this.templateCache.set(id, template); + + let hash: string; + try { + hash = templateDirWatcher.getTemplateHash(template.id); + } catch (e) { + this.logger.warn(`Failed to get hash for template ${id}: ${e.message}`); + this.templateHashCache.delete(id); + } + if (hash) { + this.templateHashCache.set(id, hash); + } + return template; + } + + async getAllTemplates() { + return getAllTemplates(); + } +} + +export class DockerTemplateRepositoryRegistry implements TemplateRepositoryRegistry { + private readonly repositories: RepositoryRegistration[]; + + constructor( + private readonly engine: ServiceEngine, + ) { + this.repositories = []; + } + + async saveRepository(config: TemplateRepositoryConfig) { + let repository: TemplateRepository; + if (config.type === "filesystem") { + repository = new FilesystemTemplateRepository(this.engine); + } else { + throw new TemplateRepositoryConfigurationError(config.id, `Unsupported repository type: ${config.type}`); + } + + this.repositories.push({ + id: config.id, + repository: repository, + }); + } + + getRepository(id: string) { + const registration = this.repositories.find((r) => r.id === id); + + return registration ? registration.repository : undefined; + } + + getAllRepositories() { + return this.repositories.map((registration) => registration.repository); + } +} \ No newline at end of file diff --git a/src/engine/engine.ts b/src/engine/engine.ts index d6fc199..73afe9a 100644 --- a/src/engine/engine.ts +++ b/src/engine/engine.ts @@ -1,7 +1,9 @@ import DockerClient from "dockerode"; import buildDockerEngine from "./docker"; import {getSingleton} from "../depend"; -import {AppConfig} from "@nsm/config"; +import {Template} from "@nsm/engine/template"; +import {AppContext} from "@nsm/app"; +import {TemplateRepositoryConfigurationError} from "@nsm/engine/error"; /** * The options for running a service. @@ -102,6 +104,63 @@ export type MetaStorage = { get: (key: string, def?: T) => Promise; }; +export type BuildOptionsMap = { + [key: string]: string; +}; + +export interface TemplateRepository { + init(ctx: AppContext): Promise; + + buildImage( + templateId: string, + options: BuildOptionsMap, + imageId?: string, + messageListener?: MessageListener + ): Promise; + + /** + * Gets the template by ID. + * + * @param id The template ID + * @return The template, or undefined if not exists + */ + getTemplate(id: string): Promise