diff --git a/README.md b/README.md index 120a96d..57d5b4b 100644 --- a/README.md +++ b/README.md @@ -89,6 +89,7 @@ const config = loadConfig({ // mergeProcessEnv?: boolean // automatically merge process.env values into the resulting config object + // keys ending in _SECRET are decrypted if possible, and left unchanged if decryption fails // defaults to true }); ``` @@ -131,6 +132,7 @@ docker run --rm -it -v `pwd`:/src -w /src ghcr.io/zyno-io/node-config exec -e pr The environment is resolved from `-e` if provided, otherwise from the `APP_ENV` environment variable. With an environment set, loads `.env`, `.env.local`, `.env.`, and `.env..local`. Without either, loads `.env` and `.env.local`. The subprocess inherits the current environment. Values from `.env` files are merged in, but existing environment variables take precedence (matching the behavior of `sh`/`shenv`). +Inherited environment keys ending in `_SECRET` are decrypted if possible, and left unchanged if decryption fails. Be sure the decryption key is set as `CONFIG_DECRYPTION_SECRET` in your environment (or pass `-k`). @@ -146,10 +148,22 @@ npx config-cli concat .env .env.production # or compose from an environment name -> loads .env + .env. npx config-cli concat -e production + +# output as JSON or YAML +npx config-cli concat -e production --format json +npx config-cli concat -e production --format yaml + +# prefix output keys +npx config-cli concat -e production --prefix pre_ +npx config-cli concat -e production --format yaml --prefix top. +npx config-cli concat -e production --format json --prefix top.pre_ ``` Unlike `exec`/`shenv`, the `-e` form excludes developer `.local` files (`.env.local`, `.env..local`) by default since they should not reach a deploy; pass `--local` to include them. +The default output format is dotenv. `--format json` and `--format yaml` reformat the merged key/value data without decrypting or otherwise changing values. +Prefixes are literal for dotenv output. For JSON and YAML output, prefixes containing `.` create nested objects; the final segment prefixes the config keys, so `--prefix top.pre_` produces keys under `top` named `pre_`. + Drop keys you don't want in the output with one or more `-x/--exclude` regular expressions — for example, to strip build-time front-end vars and the encryption key from a backend deploy blob: ``` diff --git a/package.json b/package.json index 847dfaf..576c467 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@zyno-io/config", - "version": "1.13.0", + "version": "1.14.0", "description": "Runtime configuration encryption helpers", "license": "ISC", "author": "Zyno Consulting ", diff --git a/src/cli.program.ts b/src/cli.program.ts index f1bcea7..a50a8c8 100644 --- a/src/cli.program.ts +++ b/src/cli.program.ts @@ -3,10 +3,13 @@ import { Command } from 'commander'; import { writeFileSync } from 'fs'; import { Decryptor, Encryptor } from './crypto'; -import { fileExists, generateConfigKeyPair, getDecryptionKeyFromEnv } from './helpers'; +import { decryptProcessEnvSecrets, fileExists, generateConfigKeyPair, getDecryptionKeyFromEnv } from './helpers'; import { keyMatches, readContentFromFile, transformContent } from './reader'; import { ConfigData } from './types'; +type ConcatOutputFormat = 'dotenv' | 'json' | 'yaml'; +type FormattedConfigData = { [key: string]: string | FormattedConfigData }; + export const program = new Command(); program.enablePositionalOptions(); @@ -183,10 +186,11 @@ program const key = options.key ?? getDecryptionKeyFromEnv(); const env = loadEnvFromFiles(files, key); + const inheritedEnv = decryptProcessEnvSecrets(process.env, new Decryptor(key)); const result = spawnSync(args[0], args.slice(1), { stdio: 'inherit', - env: { ...env, ...process.env } + env: { ...env, ...inheritedEnv } }); if (result.error) { @@ -208,6 +212,8 @@ program ) .option('-e, --env ', 'Compose the file list from an environment name (.env + .env.) instead of listing files') .option('--local', 'Also include .local files (developer overrides; excluded by default)') + .option('-f, --format ', 'Output format: dotenv, json, or yaml', 'dotenv') + .option('-p, --prefix ', 'Prefix output keys') .option( '-x, --exclude ', 'Exclude keys matching this regular expression (repeatable, e.g. -x ^VITE_)', @@ -223,7 +229,8 @@ program } const exclude = (options.exclude as string[]).map(pattern => new RegExp(pattern)); - console.log(concatEnvFiles(resolved, exclude)); + const format = getConcatOutputFormat(options.format); + console.log(formatConfigData(concatEnvFiles(resolved, exclude), format, options.prefix)); }); // helpers @@ -288,7 +295,7 @@ function exportFiles(files: string[], key: string) { } } -function concatEnvFiles(files: string[], exclude: RegExp[]): string { +function concatEnvFiles(files: string[], exclude: RegExp[]): ConfigData { const merged: ConfigData = {}; for (const file of files) { @@ -316,7 +323,84 @@ function concatEnvFiles(files: string[], exclude: RegExp[]): string { } } - return Object.entries(merged) + return merged; +} + +function getConcatOutputFormat(format: string): ConcatOutputFormat { + if (format === 'dotenv' || format === 'json' || format === 'yaml') { + return format; + } + + throw new Error(`Unsupported concat output format: ${format}`); +} + +function formatConfigData(data: ConfigData, format: ConcatOutputFormat, prefix = ''): string { + const formatted = applyConcatPrefix(data, format, prefix); + + if (format === 'json') { + return JSON.stringify(formatted, null, 4); + } + + if (format === 'yaml') { + return formatYamlData(formatted); + } + + return Object.entries(formatted) .map(([key, value]) => `${key}=${value}`) .join('\n'); } + +function applyConcatPrefix(data: ConfigData, format: ConcatOutputFormat, prefix: string): FormattedConfigData { + if (!prefix || Object.keys(data).length === 0) { + return data; + } + + if (format === 'dotenv' || !prefix.includes('.')) { + return prefixConfigKeys(data, prefix); + } + + const parts = prefix.split('.'); + const keyPrefix = parts.pop() ?? ''; + let formatted: FormattedConfigData = prefixConfigKeys(data, keyPrefix); + + for (const part of parts.reverse()) { + formatted = { [part]: formatted }; + } + + return formatted; +} + +function prefixConfigKeys(data: ConfigData, prefix: string): ConfigData { + const prefixed: ConfigData = {}; + + for (const [key, value] of Object.entries(data)) { + prefixed[`${prefix}${key}`] = value; + } + + return prefixed; +} + +function formatYamlData(data: FormattedConfigData, indent = 0): string { + return Object.entries(data) + .map(([key, value]) => { + const prefix = `${' '.repeat(indent)}${formatYamlKey(key)}:`; + if (typeof value === 'string') { + return `${prefix} ${formatYamlString(value)}`; + } + + return `${prefix}\n${formatYamlData(value, indent + 1)}`; + }) + .join('\n'); +} + +function formatYamlKey(key: string): string { + if (/^[A-Za-z_][A-Za-z0-9_]*$/.test(key)) { + return key; + } + + return formatYamlString(key); +} + +function formatYamlString(value: string): string { + return `'${value.replace(/'/g, "''")}'`; +} diff --git a/src/helpers.ts b/src/helpers.ts index fc730d5..a203b74 100644 --- a/src/helpers.ts +++ b/src/helpers.ts @@ -1,6 +1,10 @@ import { generateKeyPairSync } from 'crypto'; import { existsSync } from 'fs'; +import type { ConfigData } from './types'; + +import { Decryptor } from './crypto'; + export function generateConfigKeyPair() { const { privateKey, publicKey } = generateKeyPairSync('x25519', { publicKeyEncoding: { @@ -33,3 +37,26 @@ export function getPath(path: string) { export function fileExists(path: string) { return existsSync(getPath(path)); } + +export function decryptProcessEnvSecrets(env: NodeJS.ProcessEnv, decryptor: Decryptor): ConfigData { + const result: ConfigData = {}; + + for (const [key, value] of Object.entries(env)) { + if (value === undefined) { + continue; + } + + if (!key.endsWith('_SECRET')) { + result[key] = value; + continue; + } + + try { + result[key] = decryptor.decryptValueIfEncrypted(value); + } catch { + result[key] = value; + } + } + + return result; +} diff --git a/src/index.ts b/src/index.ts index 25d5179..7c03f6f 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,7 +1,7 @@ import { parse } from 'dotenv'; import { Decryptor, Encryptor } from './crypto'; -import { fileExists, getDecryptionKeyFromEnv } from './helpers'; +import { decryptProcessEnvSecrets, fileExists, getDecryptionKeyFromEnv } from './helpers'; import { readContentFromFile, transformContent } from './reader'; import { ConfigData, DefaultLoadOptions, LoadOptions } from './types'; @@ -55,7 +55,7 @@ export function loadConfig(options?: LoadOptions): T { } if (options.mergeProcessEnv !== false) { - Object.assign(config, process.env); + Object.assign(config, decryptProcessEnvSecrets(process.env, decryptor)); } return config as T; diff --git a/tests/api.spec.ts b/tests/api.spec.ts index 4bdcdd4..fa56670 100644 --- a/tests/api.spec.ts +++ b/tests/api.spec.ts @@ -41,6 +41,33 @@ describe('API', () => { } }); + it('should decrypt encrypted process.env _SECRET values without throwing on failures', async () => { + const { privateKey, publicKey } = generateConfigKeyPair(); + const encrypted = encryptConfigData(publicKey, { + ENV_SECRET: 'from process env', + ENV_TOKEN: 'encrypted but not a secret key' + }); + + process.env.ENV_SECRET = encrypted.ENV_SECRET; + process.env.ENV_TOKEN = encrypted.ENV_TOKEN; + process.env.BAD_SECRET = '$$[not-base64]'; + + try { + const config = loadConfig({ + file: `${__dirname}/fixtures/sample.env`, + key: privateKey + }); + + assert.strictEqual(config.ENV_SECRET, 'from process env'); + assert.strictEqual(config.ENV_TOKEN, encrypted.ENV_TOKEN); + assert.strictEqual(config.BAD_SECRET, '$$[not-base64]'); + } finally { + delete process.env.ENV_SECRET; + delete process.env.ENV_TOKEN; + delete process.env.BAD_SECRET; + } + }); + it('should load a config with no encryption', async () => { const config = loadConfig({ file: `${__dirname}/fixtures/sample.env` diff --git a/tests/cli.spec.ts b/tests/cli.spec.ts index 85e6881..1decd75 100644 --- a/tests/cli.spec.ts +++ b/tests/cli.spec.ts @@ -3,6 +3,7 @@ import assert from 'node:assert'; import { describe, it, beforeEach, after, mock } from 'node:test'; import { generateConfigKeyPair } from '../src/helpers'; +import { encryptConfigData } from '../src/index'; let importCounter = 0; async function getProgram() { @@ -11,6 +12,10 @@ async function getProgram() { return program; } +function readFixture(path: string) { + return readFileSync(`${__dirname}/fixtures/${path}`, 'utf8').trimEnd(); +} + describe('CLI', () => { beforeEach(() => { mock.restoreAll(); @@ -413,6 +418,48 @@ describe('CLI', () => { delete process.env.CONFIG_PATH; } }); + + it('should decrypt inherited process.env _SECRET values without throwing on failures', async () => { + const program = await getProgram(); + const exitCodes: number[] = []; + const exitMock = mock.method(process, 'exit', (code: number) => { + exitCodes.push(code); + throw new Error('process.exit'); + }); + const { privateKey, publicKey } = generateConfigKeyPair(); + const encrypted = encryptConfigData(publicKey, { + CLI_PROCESS_SECRET: 'from process env', + CLI_PROCESS_TOKEN: 'encrypted but not a secret key' + }); + const badSecret = '$$[not-base64]'; + const script = ` + if (process.env.CLI_PROCESS_SECRET !== ${JSON.stringify('from process env')}) process.exit(11); + if (process.env.CLI_PROCESS_TOKEN !== ${JSON.stringify(encrypted.CLI_PROCESS_TOKEN)}) process.exit(12); + if (process.env.CLI_PROCESS_BAD_SECRET !== ${JSON.stringify(badSecret)}) process.exit(13); + process.exit(0); + `; + + process.env.CONFIG_PATH = `${__dirname}/fixtures`; + process.env.CLI_PROCESS_SECRET = encrypted.CLI_PROCESS_SECRET; + process.env.CLI_PROCESS_TOKEN = encrypted.CLI_PROCESS_TOKEN; + process.env.CLI_PROCESS_BAD_SECRET = badSecret; + writeFileSync(`${__dirname}/fixtures/.env`, ''); + + try { + assert.throws(() => program.parse(['exec', '-k', privateKey, '--', 'node', '-e', script], { from: 'user' }), { + message: 'process.exit' + }); + + assert.strictEqual(exitCodes[0], 0, 'spawned process should receive the expected environment'); + } finally { + exitMock.mock.restore(); + rmSync(`${__dirname}/fixtures/.env`, { force: true }); + delete process.env.CONFIG_PATH; + delete process.env.CLI_PROCESS_SECRET; + delete process.env.CLI_PROCESS_TOKEN; + delete process.env.CLI_PROCESS_BAD_SECRET; + } + }); }); describe('concat', () => { @@ -453,6 +500,33 @@ describe('CLI', () => { } }); + const formatCases = [ + { name: 'dotenv', args: [], extension: 'env' }, + { name: 'json', args: ['--format', 'json'], extension: 'json' }, + { name: 'yaml', args: ['--format', 'yaml'], extension: 'yaml' } + ]; + const prefixCases = [ + { name: 'without a prefix', args: [], fixture: 'concat.expected' }, + { name: 'with a key prefix', args: ['--prefix', 'pre_'], fixture: 'concat.prefix-key.expected' }, + { name: 'with a dotted nesting prefix', args: ['--prefix', 'top.'], fixture: 'concat.prefix-nested.expected' }, + { name: 'with a dotted nesting and key prefix', args: ['--prefix', 'top.pre_'], fixture: 'concat.prefix-nested-key.expected' } + ]; + + for (const prefixCase of prefixCases) { + for (const formatCase of formatCases) { + it(`should output ${formatCase.name} ${prefixCase.name} from fixtures`, async () => { + const program = await getProgram(); + const f = `${__dirname}/fixtures/concat.format.env`; + + const logMock = mock.method(console, 'log', () => {}); + program.parse(['concat', f, ...formatCase.args, ...prefixCase.args], { from: 'user' }); + + const output = logMock.mock.calls.map(call => call.arguments[0]).join('\n'); + assert.strictEqual(output, readFixture(`${prefixCase.fixture}.${formatCase.extension}`)); + }); + } + } + it('should exclude keys matching --exclude patterns', async () => { const program = await getProgram(); const f = `${__dirname}/fixtures/concat.exclude.test`; diff --git a/tests/fixtures/concat.expected.env b/tests/fixtures/concat.expected.env new file mode 100644 index 0000000..63de7d4 --- /dev/null +++ b/tests/fixtures/concat.expected.env @@ -0,0 +1,4 @@ +PLAIN=hello +EMPTY= +TOKEN_SECRET=$$[ZW5jcnlwdGVk] +WEIRD-KEY=needs quoting diff --git a/tests/fixtures/concat.expected.json b/tests/fixtures/concat.expected.json new file mode 100644 index 0000000..493b4c5 --- /dev/null +++ b/tests/fixtures/concat.expected.json @@ -0,0 +1,6 @@ +{ + "PLAIN": "hello", + "EMPTY": "", + "TOKEN_SECRET": "$$[ZW5jcnlwdGVk]", + "WEIRD-KEY": "needs quoting" +} diff --git a/tests/fixtures/concat.expected.yaml b/tests/fixtures/concat.expected.yaml new file mode 100644 index 0000000..3a0e323 --- /dev/null +++ b/tests/fixtures/concat.expected.yaml @@ -0,0 +1,4 @@ +PLAIN: 'hello' +EMPTY: '' +TOKEN_SECRET: '$$[ZW5jcnlwdGVk]' +'WEIRD-KEY': 'needs quoting' diff --git a/tests/fixtures/concat.format.env b/tests/fixtures/concat.format.env new file mode 100644 index 0000000..63de7d4 --- /dev/null +++ b/tests/fixtures/concat.format.env @@ -0,0 +1,4 @@ +PLAIN=hello +EMPTY= +TOKEN_SECRET=$$[ZW5jcnlwdGVk] +WEIRD-KEY=needs quoting diff --git a/tests/fixtures/concat.prefix-key.expected.env b/tests/fixtures/concat.prefix-key.expected.env new file mode 100644 index 0000000..715653d --- /dev/null +++ b/tests/fixtures/concat.prefix-key.expected.env @@ -0,0 +1,4 @@ +pre_PLAIN=hello +pre_EMPTY= +pre_TOKEN_SECRET=$$[ZW5jcnlwdGVk] +pre_WEIRD-KEY=needs quoting diff --git a/tests/fixtures/concat.prefix-key.expected.json b/tests/fixtures/concat.prefix-key.expected.json new file mode 100644 index 0000000..424f1f0 --- /dev/null +++ b/tests/fixtures/concat.prefix-key.expected.json @@ -0,0 +1,6 @@ +{ + "pre_PLAIN": "hello", + "pre_EMPTY": "", + "pre_TOKEN_SECRET": "$$[ZW5jcnlwdGVk]", + "pre_WEIRD-KEY": "needs quoting" +} diff --git a/tests/fixtures/concat.prefix-key.expected.yaml b/tests/fixtures/concat.prefix-key.expected.yaml new file mode 100644 index 0000000..905a9d3 --- /dev/null +++ b/tests/fixtures/concat.prefix-key.expected.yaml @@ -0,0 +1,4 @@ +pre_PLAIN: 'hello' +pre_EMPTY: '' +pre_TOKEN_SECRET: '$$[ZW5jcnlwdGVk]' +'pre_WEIRD-KEY': 'needs quoting' diff --git a/tests/fixtures/concat.prefix-nested-key.expected.env b/tests/fixtures/concat.prefix-nested-key.expected.env new file mode 100644 index 0000000..b96e2ab --- /dev/null +++ b/tests/fixtures/concat.prefix-nested-key.expected.env @@ -0,0 +1,4 @@ +top.pre_PLAIN=hello +top.pre_EMPTY= +top.pre_TOKEN_SECRET=$$[ZW5jcnlwdGVk] +top.pre_WEIRD-KEY=needs quoting diff --git a/tests/fixtures/concat.prefix-nested-key.expected.json b/tests/fixtures/concat.prefix-nested-key.expected.json new file mode 100644 index 0000000..4f6a17d --- /dev/null +++ b/tests/fixtures/concat.prefix-nested-key.expected.json @@ -0,0 +1,8 @@ +{ + "top": { + "pre_PLAIN": "hello", + "pre_EMPTY": "", + "pre_TOKEN_SECRET": "$$[ZW5jcnlwdGVk]", + "pre_WEIRD-KEY": "needs quoting" + } +} diff --git a/tests/fixtures/concat.prefix-nested-key.expected.yaml b/tests/fixtures/concat.prefix-nested-key.expected.yaml new file mode 100644 index 0000000..040b873 --- /dev/null +++ b/tests/fixtures/concat.prefix-nested-key.expected.yaml @@ -0,0 +1,5 @@ +top: + pre_PLAIN: 'hello' + pre_EMPTY: '' + pre_TOKEN_SECRET: '$$[ZW5jcnlwdGVk]' + 'pre_WEIRD-KEY': 'needs quoting' diff --git a/tests/fixtures/concat.prefix-nested.expected.env b/tests/fixtures/concat.prefix-nested.expected.env new file mode 100644 index 0000000..3cee1fb --- /dev/null +++ b/tests/fixtures/concat.prefix-nested.expected.env @@ -0,0 +1,4 @@ +top.PLAIN=hello +top.EMPTY= +top.TOKEN_SECRET=$$[ZW5jcnlwdGVk] +top.WEIRD-KEY=needs quoting diff --git a/tests/fixtures/concat.prefix-nested.expected.json b/tests/fixtures/concat.prefix-nested.expected.json new file mode 100644 index 0000000..22850ba --- /dev/null +++ b/tests/fixtures/concat.prefix-nested.expected.json @@ -0,0 +1,8 @@ +{ + "top": { + "PLAIN": "hello", + "EMPTY": "", + "TOKEN_SECRET": "$$[ZW5jcnlwdGVk]", + "WEIRD-KEY": "needs quoting" + } +} diff --git a/tests/fixtures/concat.prefix-nested.expected.yaml b/tests/fixtures/concat.prefix-nested.expected.yaml new file mode 100644 index 0000000..7903e0c --- /dev/null +++ b/tests/fixtures/concat.prefix-nested.expected.yaml @@ -0,0 +1,5 @@ +top: + PLAIN: 'hello' + EMPTY: '' + TOKEN_SECRET: '$$[ZW5jcnlwdGVk]' + 'WEIRD-KEY': 'needs quoting'