From e2a612bdcadfe76f0ede8e439c8b98c1e7dbe59c Mon Sep 17 00:00:00 2001 From: Joseph McCarron Date: Mon, 6 Oct 2025 16:50:07 -0500 Subject: [PATCH 01/17] init claude configuration directories, update install --- .claude/settings.json | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .claude/settings.json diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 0000000..e8f289d --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,3 @@ +{ + "enableAllProjectMcpServers": false +} \ No newline at end of file From 9779fd9603ea222704257b788071f6c8d9b1682c Mon Sep 17 00:00:00 2001 From: Joseph McCarron Date: Tue, 7 Oct 2025 13:53:13 -0500 Subject: [PATCH 02/17] install skyflow-node --- package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index d1ce103..79c7ada 100644 --- a/package.json +++ b/package.json @@ -22,7 +22,8 @@ "@types/node": "^22.13.15", "axios": "^1.8.4", "commander": "^13.1.0", - "inquirer": "^12.5.0" + "inquirer": "^12.5.0", + "skyflow-node": "^2.0.0" }, "devDependencies": { "@types/inquirer": "^9.0.7", From 04b9b6c5d9e121952ac073e9dae5cffc17274ece Mon Sep 17 00:00:00 2001 From: Joseph McCarron Date: Tue, 7 Oct 2025 14:38:29 -0500 Subject: [PATCH 03/17] implement insert records, deidentify, reidentify --- dist/commands/deidentify.js | 189 ++ dist/commands/insert.js | 164 ++ dist/commands/reidentify.js | 158 ++ dist/index.js | 16 +- dist/utils/logger.js | 10 +- dist/utils/skyflow.js | 162 ++ docs/implementation-plan-skyflow-v2.md | 299 +++ docs/skyflow-node/readme.md | 3401 ++++++++++++++++++++++++ docs/skyflow-node/samples/README.md | 138 + src/commands/deidentify.ts | 220 ++ src/commands/insert.ts | 177 ++ src/commands/reidentify.ts | 190 ++ src/index.ts | 18 +- src/types.ts | 39 + src/utils/logger.ts | 10 + src/utils/skyflow.ts | 194 ++ 16 files changed, 5380 insertions(+), 5 deletions(-) create mode 100644 dist/commands/deidentify.js create mode 100644 dist/commands/insert.js create mode 100644 dist/commands/reidentify.js create mode 100644 dist/utils/skyflow.js create mode 100644 docs/implementation-plan-skyflow-v2.md create mode 100644 docs/skyflow-node/readme.md create mode 100644 docs/skyflow-node/samples/README.md create mode 100644 src/commands/deidentify.ts create mode 100644 src/commands/insert.ts create mode 100644 src/commands/reidentify.ts create mode 100644 src/utils/skyflow.ts diff --git a/dist/commands/deidentify.js b/dist/commands/deidentify.js new file mode 100644 index 0000000..5c1ef27 --- /dev/null +++ b/dist/commands/deidentify.js @@ -0,0 +1,189 @@ +"use strict"; +var __importDefault = (this && this.__importDefault) || function (mod) { + return (mod && mod.__esModule) ? mod : { "default": mod }; +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.deidentifyCommand = void 0; +const inquirer_1 = __importDefault(require("inquirer")); +const skyflow_node_1 = require("skyflow-node"); +const skyflow_1 = require("../utils/skyflow"); +const logger_1 = require("../utils/logger"); +// Map of entity aliases to DetectEntities enum values +const ENTITY_MAP = { + SSN: skyflow_node_1.DetectEntities.SSN, + CREDIT_CARD: skyflow_node_1.DetectEntities.CREDIT_CARD, + CREDIT_CARD_NUMBER: skyflow_node_1.DetectEntities.CREDIT_CARD, + EMAIL: skyflow_node_1.DetectEntities.EMAIL_ADDRESS, + EMAIL_ADDRESS: skyflow_node_1.DetectEntities.EMAIL_ADDRESS, + PHONE_NUMBER: skyflow_node_1.DetectEntities.PHONE_NUMBER, + PHONE: skyflow_node_1.DetectEntities.PHONE_NUMBER, + NAME: skyflow_node_1.DetectEntities.NAME, + DOB: skyflow_node_1.DetectEntities.DOB, + DATE_OF_BIRTH: skyflow_node_1.DetectEntities.DOB, + ACCOUNT_NUMBER: skyflow_node_1.DetectEntities.ACCOUNT_NUMBER, + DRIVER_LICENSE: skyflow_node_1.DetectEntities.DRIVER_LICENSE, + PASSPORT_NUMBER: skyflow_node_1.DetectEntities.PASSPORT_NUMBER, + PASSPORT: skyflow_node_1.DetectEntities.PASSPORT_NUMBER, +}; +const AVAILABLE_ENTITIES = Object.keys(ENTITY_MAP).join(', '); +const deidentifyCommand = (program) => { + program + .command('deidentify') + .description('Detect and redact sensitive data from text using Skyflow Detect API') + .option('--text ', 'Text to deidentify (or pipe from stdin)') + .option('--entities ', `Comma-separated entity types to detect (e.g., SSN,CREDIT_CARD). Available: ${AVAILABLE_ENTITIES}`) + .option('--token-type ', 'Token format: vault_token (stored), entity_only (labels), random_token (ephemeral)', 'vault_token') + .option('--output ', 'Output format: text or json', 'text') + .option('--vault-id ', 'Vault ID (or set SKYFLOW_VAULT_ID)') + .option('--cluster-id ', 'Cluster ID (or set SKYFLOW_VAULT_URL)') + .option('--environment ', 'Environment: PROD, SANDBOX, STAGE, DEV', 'PROD') + .action(async (options) => { + var _a; + try { + // Get text from option or stdin + let textInput = options.text; + if (!textInput) { + // Check if stdin is being piped + if (!process.stdin.isTTY) { + (0, logger_1.logVerbose)('Reading text from stdin...'); + textInput = await readStdin(); + } + else { + // Prompt for text + const answers = await inquirer_1.default.prompt([ + { + type: 'editor', + name: 'text', + message: 'Enter text to deidentify:', + validate: (input) => { + if (!input) + return 'Text is required'; + return true; + }, + }, + ]); + textInput = answers.text; + } + } + // Parse entities if provided + let entityList; + if (options.entities) { + const rawEntities = options.entities.split(',').map((e) => e.trim().toUpperCase()); + entityList = rawEntities.map((entity) => { + const mapped = ENTITY_MAP[entity]; + if (!mapped) { + throw new Error(`Unknown entity type: ${entity}\nAvailable entities: ${AVAILABLE_ENTITIES}`); + } + return mapped; + }); + (0, logger_1.logVerbose)(`Detecting entities: ${entityList.join(', ')}`); + } + else { + // Use common entities by default + entityList = [ + skyflow_node_1.DetectEntities.SSN, + skyflow_node_1.DetectEntities.CREDIT_CARD, + skyflow_node_1.DetectEntities.EMAIL_ADDRESS, + skyflow_node_1.DetectEntities.PHONE_NUMBER, + skyflow_node_1.DetectEntities.NAME, + skyflow_node_1.DetectEntities.DOB, + ]; + (0, logger_1.logVerbose)('Using default entity detection (SSN, CREDIT_CARD, EMAIL_ADDRESS, PHONE_NUMBER, NAME, DOB)'); + } + // Parse token type + let tokenType; + switch ((_a = options.tokenType) === null || _a === void 0 ? void 0 : _a.toLowerCase()) { + case 'vault_token': + tokenType = skyflow_node_1.TokenType.VAULT_TOKEN; + break; + case 'entity_only': + tokenType = skyflow_node_1.TokenType.ENTITY_ONLY; + break; + case 'entity_unique_counter': + tokenType = skyflow_node_1.TokenType.ENTITY_UNIQUE_COUNTER; + break; + default: + tokenType = skyflow_node_1.TokenType.VAULT_TOKEN; + (0, logger_1.logVerbose)(`Unknown token type '${options.tokenType}', using vault_token`); + } + (0, logger_1.logVerbose)(`Using token type: ${options.tokenType || 'vault_token'}`); + // Resolve vault ID + const vaultId = (0, skyflow_1.resolveVaultId)(options.vaultId); + (0, logger_1.logVerbose)(`Using vault ID: ${vaultId}`); + // Initialize Skyflow client + const verbose = program.opts().verbose || false; + const skyflowClient = (0, skyflow_1.initializeSkyflowClient)(vaultId, options.clusterId, options.environment, verbose); + // Create deidentify request + const deidentifyRequest = new skyflow_node_1.DeidentifyTextRequest(textInput); + (0, logger_1.logVerbose)('Created deidentify request'); + // Configure options + const deidentifyOptions = new skyflow_node_1.DeidentifyTextOptions(); + deidentifyOptions.setEntities(entityList); + const tokenFormat = new skyflow_node_1.TokenFormat(); + tokenFormat.setDefault(tokenType); + deidentifyOptions.setTokenFormat(tokenFormat); + // Execute deidentify + (0, logger_1.logVerbose)('Executing deidentify operation...'); + const response = await skyflowClient + .detect(vaultId) + .deidentifyText(deidentifyRequest, deidentifyOptions); + // Display results + if (options.output === 'json') { + console.log(JSON.stringify(response, null, 2)); + } + else { + console.log('\nDeidentified Text:'); + console.log('─'.repeat(60)); + console.log(response.processedText); + console.log('─'.repeat(60)); + if (response.entities && response.entities.length > 0) { + console.log(`\nDetected ${response.entities.length} sensitive entities:\n`); + response.entities.forEach((entity, index) => { + console.log(`${index + 1}. ${entity.entity}`); + console.log(` Original: "${entity.value}"`); + console.log(` Token: ${entity.token}`); + if (entity.textIndex) { + console.log(` Position: ${entity.textIndex.start}-${entity.textIndex.end}`); + } + if (entity.scores) { + const confidence = Object.values(entity.scores)[0]; + console.log(` Confidence: ${(confidence * 100).toFixed(1)}%`); + } + console.log(); + }); + } + else { + console.log('\nNo sensitive entities detected.'); + } + if (response.wordCount !== undefined) { + console.log(`Word count: ${response.wordCount}`); + } + if (response.charCount !== undefined) { + console.log(`Character count: ${response.charCount}`); + } + } + } + catch (error) { + (0, skyflow_1.handleSkyflowError)(error); + } + }); +}; +exports.deidentifyCommand = deidentifyCommand; +/** + * Read input from stdin + */ +const readStdin = () => { + return new Promise((resolve, reject) => { + let data = ''; + process.stdin.setEncoding('utf8'); + process.stdin.on('data', (chunk) => { + data += chunk; + }); + process.stdin.on('end', () => { + resolve(data.trim()); + }); + process.stdin.on('error', (error) => { + reject(error); + }); + }); +}; diff --git a/dist/commands/insert.js b/dist/commands/insert.js new file mode 100644 index 0000000..769d4f0 --- /dev/null +++ b/dist/commands/insert.js @@ -0,0 +1,164 @@ +"use strict"; +var __importDefault = (this && this.__importDefault) || function (mod) { + return (mod && mod.__esModule) ? mod : { "default": mod }; +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.insertCommand = void 0; +const inquirer_1 = __importDefault(require("inquirer")); +const skyflow_node_1 = require("skyflow-node"); +const skyflow_1 = require("../utils/skyflow"); +const logger_1 = require("../utils/logger"); +const insertCommand = (program) => { + program + .command('insert') + .description('Insert sensitive data into a Skyflow vault table') + .option('--table ', 'Table name to insert data into') + .option('--data ', 'JSON data to insert (or pipe from stdin)') + .option('--return-tokens', 'Return tokens for inserted data', false) + .option('--continue-on-error', 'Continue if some records fail', false) + .option('--upsert-column ', 'Column name for upsert operations') + .option('--vault-id ', 'Vault ID (or set SKYFLOW_VAULT_ID)') + .option('--cluster-id ', 'Cluster ID (or set SKYFLOW_VAULT_URL)') + .option('--environment ', 'Environment: PROD, SANDBOX, STAGE, DEV', 'PROD') + .action(async (options) => { + var _a, _b; + try { + // Prompt for missing required options + if (!options.table) { + const answers = await inquirer_1.default.prompt([ + { + type: 'input', + name: 'table', + message: 'Enter table name:', + validate: (input) => { + if (!input) + return 'Table name is required'; + return true; + }, + }, + ]); + options.table = answers.table; + } + // Get data from option or stdin + let dataInput = options.data; + if (!dataInput) { + // Check if stdin is being piped + if (!process.stdin.isTTY) { + (0, logger_1.logVerbose)('Reading data from stdin...'); + dataInput = await readStdin(); + } + else { + // Prompt for data + const answers = await inquirer_1.default.prompt([ + { + type: 'editor', + name: 'data', + message: 'Enter JSON data to insert:', + validate: (input) => { + if (!input) + return 'Data is required'; + try { + JSON.parse(input); + return true; + } + catch (_a) { + return 'Invalid JSON format'; + } + }, + }, + ]); + dataInput = answers.data; + } + } + // Parse JSON data + let insertData; + try { + const parsed = JSON.parse(dataInput); + // Ensure data is an array + insertData = Array.isArray(parsed) ? parsed : [parsed]; + (0, logger_1.logVerbose)(`Parsed ${insertData.length} record(s) to insert`); + } + catch (error) { + throw new Error(`Invalid JSON data: ${error instanceof Error ? error.message : String(error)}`); + } + // Resolve vault ID + const vaultId = (0, skyflow_1.resolveVaultId)(options.vaultId); + (0, logger_1.logVerbose)(`Using vault ID: ${vaultId}`); + // Initialize Skyflow client + const verbose = program.opts().verbose || false; + const skyflowClient = (0, skyflow_1.initializeSkyflowClient)(vaultId, options.clusterId, options.environment, verbose); + // Create insert request + const insertRequest = new skyflow_node_1.InsertRequest(options.table, insertData); + (0, logger_1.logVerbose)(`Created insert request for table: ${options.table}`); + // Configure insert options + const insertOptions = new skyflow_node_1.InsertOptions(); + if (options.returnTokens) { + insertOptions.setReturnTokens(true); + (0, logger_1.logVerbose)('Configured to return tokens'); + } + if (options.continueOnError) { + insertOptions.setContinueOnError(true); + (0, logger_1.logVerbose)('Configured to continue on error'); + } + if (options.upsertColumn) { + insertOptions.setUpsertColumn(options.upsertColumn); + (0, logger_1.logVerbose)(`Configured upsert on column: ${options.upsertColumn}`); + } + // Execute insert + (0, logger_1.logVerbose)('Executing insert operation...'); + const response = await skyflowClient + .vault(vaultId) + .insert(insertRequest, insertOptions); + // Display results + console.log('\nInsert completed successfully!\n'); + if (response.insertedFields && response.insertedFields.length > 0) { + console.log('Inserted records:'); + response.insertedFields.forEach((record, index) => { + console.log(`\nRecord ${index + 1}:`); + if (record.skyflowId) { + console.log(` Skyflow ID: ${record.skyflowId}`); + } + if (options.returnTokens) { + Object.keys(record).forEach((key) => { + if (key !== 'skyflowId') { + console.log(` ${key}: ${record[key]}`); + } + }); + } + }); + } + if (response.errors && response.errors.length > 0) { + console.log('\nErrors:'); + response.errors.forEach((error, index) => { + console.log(`\nError ${index + 1}:`); + console.log(` ${JSON.stringify(error, null, 2)}`); + }); + } + console.log(`\nTotal records processed: ${insertData.length}`); + console.log(`Successful: ${((_a = response.insertedFields) === null || _a === void 0 ? void 0 : _a.length) || 0}`); + console.log(`Failed: ${((_b = response.errors) === null || _b === void 0 ? void 0 : _b.length) || 0}`); + } + catch (error) { + (0, skyflow_1.handleSkyflowError)(error); + } + }); +}; +exports.insertCommand = insertCommand; +/** + * Read input from stdin + */ +const readStdin = () => { + return new Promise((resolve, reject) => { + let data = ''; + process.stdin.setEncoding('utf8'); + process.stdin.on('data', (chunk) => { + data += chunk; + }); + process.stdin.on('end', () => { + resolve(data.trim()); + }); + process.stdin.on('error', (error) => { + reject(error); + }); + }); +}; diff --git a/dist/commands/reidentify.js b/dist/commands/reidentify.js new file mode 100644 index 0000000..26bae8b --- /dev/null +++ b/dist/commands/reidentify.js @@ -0,0 +1,158 @@ +"use strict"; +var __importDefault = (this && this.__importDefault) || function (mod) { + return (mod && mod.__esModule) ? mod : { "default": mod }; +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.reidentifyCommand = void 0; +const inquirer_1 = __importDefault(require("inquirer")); +const skyflow_node_1 = require("skyflow-node"); +const skyflow_1 = require("../utils/skyflow"); +const logger_1 = require("../utils/logger"); +// Map of entity aliases to DetectEntities enum values +const ENTITY_MAP = { + SSN: skyflow_node_1.DetectEntities.SSN, + CREDIT_CARD: skyflow_node_1.DetectEntities.CREDIT_CARD, + CREDIT_CARD_NUMBER: skyflow_node_1.DetectEntities.CREDIT_CARD, + EMAIL: skyflow_node_1.DetectEntities.EMAIL_ADDRESS, + EMAIL_ADDRESS: skyflow_node_1.DetectEntities.EMAIL_ADDRESS, + PHONE_NUMBER: skyflow_node_1.DetectEntities.PHONE_NUMBER, + PHONE: skyflow_node_1.DetectEntities.PHONE_NUMBER, + NAME: skyflow_node_1.DetectEntities.NAME, + DOB: skyflow_node_1.DetectEntities.DOB, + DATE_OF_BIRTH: skyflow_node_1.DetectEntities.DOB, + ACCOUNT_NUMBER: skyflow_node_1.DetectEntities.ACCOUNT_NUMBER, + DRIVER_LICENSE: skyflow_node_1.DetectEntities.DRIVER_LICENSE, + PASSPORT_NUMBER: skyflow_node_1.DetectEntities.PASSPORT_NUMBER, + PASSPORT: skyflow_node_1.DetectEntities.PASSPORT_NUMBER, +}; +const AVAILABLE_ENTITIES = Object.keys(ENTITY_MAP).join(', '); +const reidentifyCommand = (program) => { + program + .command('reidentify') + .description('Restore original values from tokenized text using Skyflow Detect API') + .option('--text ', 'Tokenized text to reidentify (or pipe from stdin)') + .option('--plain-text ', `Comma-separated entities to return as plain text (e.g., SSN,CREDIT_CARD). Available: ${AVAILABLE_ENTITIES}`) + .option('--masked ', 'Comma-separated entities to return masked') + .option('--redacted ', 'Comma-separated entities to keep redacted') + .option('--output ', 'Output format: text or json', 'text') + .option('--vault-id ', 'Vault ID (or set SKYFLOW_VAULT_ID)') + .option('--cluster-id ', 'Cluster ID (or set SKYFLOW_VAULT_URL)') + .option('--environment ', 'Environment: PROD, SANDBOX, STAGE, DEV', 'PROD') + .action(async (options) => { + try { + // Get text from option or stdin + let textInput = options.text; + if (!textInput) { + // Check if stdin is being piped + if (!process.stdin.isTTY) { + (0, logger_1.logVerbose)('Reading text from stdin...'); + textInput = await readStdin(); + } + else { + // Prompt for text + const answers = await inquirer_1.default.prompt([ + { + type: 'editor', + name: 'text', + message: 'Enter tokenized text to reidentify:', + validate: (input) => { + if (!input) + return 'Text is required'; + return true; + }, + }, + ]); + textInput = answers.text; + } + } + // Parse entity options + const parseEntities = (entityString) => { + if (!entityString) + return undefined; + const rawEntities = entityString.split(',').map((e) => e.trim().toUpperCase()); + return rawEntities.map((entity) => { + const mapped = ENTITY_MAP[entity]; + if (!mapped) { + throw new Error(`Unknown entity type: ${entity}\nAvailable entities: ${AVAILABLE_ENTITIES}`); + } + return mapped; + }); + }; + const plainTextEntities = parseEntities(options.plainText); + const maskedEntities = parseEntities(options.masked); + const redactedEntities = parseEntities(options.redacted); + if (plainTextEntities) { + (0, logger_1.logVerbose)(`Plain text entities: ${plainTextEntities.join(', ')}`); + } + if (maskedEntities) { + (0, logger_1.logVerbose)(`Masked entities: ${maskedEntities.join(', ')}`); + } + if (redactedEntities) { + (0, logger_1.logVerbose)(`Redacted entities: ${redactedEntities.join(', ')}`); + } + // If no options specified, default to plain text for all entities + const hasEntityOptions = plainTextEntities || maskedEntities || redactedEntities; + if (!hasEntityOptions) { + (0, logger_1.logVerbose)('No entity options specified, returning all as plain text'); + } + // Resolve vault ID + const vaultId = (0, skyflow_1.resolveVaultId)(options.vaultId); + (0, logger_1.logVerbose)(`Using vault ID: ${vaultId}`); + // Initialize Skyflow client + const verbose = program.opts().verbose || false; + const skyflowClient = (0, skyflow_1.initializeSkyflowClient)(vaultId, options.clusterId, options.environment, verbose); + // Create reidentify request + const reidentifyRequest = new skyflow_node_1.ReidentifyTextRequest(textInput); + (0, logger_1.logVerbose)('Created reidentify request'); + // Configure options + const reidentifyOptions = new skyflow_node_1.ReidentifyTextOptions(); + if (plainTextEntities) { + reidentifyOptions.setPlainTextEntities(plainTextEntities); + } + if (maskedEntities) { + reidentifyOptions.setMaskedEntities(maskedEntities); + } + if (redactedEntities) { + reidentifyOptions.setRedactedEntities(redactedEntities); + } + // Execute reidentify + (0, logger_1.logVerbose)('Executing reidentify operation...'); + const response = await skyflowClient + .detect(vaultId) + .reidentifyText(reidentifyRequest, reidentifyOptions); + // Display results + if (options.output === 'json') { + console.log(JSON.stringify(response, null, 2)); + } + else { + console.log('\nReidentified Text:'); + console.log('─'.repeat(60)); + console.log(response.processedText); + console.log('─'.repeat(60)); + console.log('\nOriginal sensitive data has been restored.'); + } + } + catch (error) { + (0, skyflow_1.handleSkyflowError)(error); + } + }); +}; +exports.reidentifyCommand = reidentifyCommand; +/** + * Read input from stdin + */ +const readStdin = () => { + return new Promise((resolve, reject) => { + let data = ''; + process.stdin.setEncoding('utf8'); + process.stdin.on('data', (chunk) => { + data += chunk; + }); + process.stdin.on('end', () => { + resolve(data.trim()); + }); + process.stdin.on('error', (error) => { + reject(error); + }); + }); +}; diff --git a/dist/index.js b/dist/index.js index 2a0fcba..1dfb86f 100755 --- a/dist/index.js +++ b/dist/index.js @@ -4,6 +4,9 @@ Object.defineProperty(exports, "__esModule", { value: true }); const commander_1 = require("commander"); const createVault_1 = require("./commands/createVault"); const configure_1 = require("./commands/configure"); +const insert_1 = require("./commands/insert"); +const deidentify_1 = require("./commands/deidentify"); +const reidentify_1 = require("./commands/reidentify"); const config_1 = require("./utils/config"); const logger_1 = require("./utils/logger"); // Create the CLI program @@ -22,14 +25,23 @@ program // Register commands (0, configure_1.configureCommand)(program); (0, createVault_1.createVaultCommand)(program); +(0, insert_1.insertCommand)(program); +(0, deidentify_1.deidentifyCommand)(program); +(0, reidentify_1.reidentifyCommand)(program); // Error handler for authentication program.hook('preAction', async (thisCommand, actionCommand) => { + const commandName = actionCommand.name(); // Skip authentication for configure command - if (actionCommand.name() === 'configure') { + if (commandName === 'configure') { + return; + } + // Skip authentication for SDK commands (they handle their own auth) + const sdkCommands = ['insert', 'deidentify', 'reidentify']; + if (sdkCommands.includes(commandName)) { return; } try { - // Load and validate configuration + // Load and validate configuration for API commands (0, config_1.loadConfig)(); } catch (error) { diff --git a/dist/utils/logger.js b/dist/utils/logger.js index 9dd2721..a3e3348 100644 --- a/dist/utils/logger.js +++ b/dist/utils/logger.js @@ -3,7 +3,7 @@ * Simple logger utility that controls output based on verbosity level */ Object.defineProperty(exports, "__esModule", { value: true }); -exports.errorLog = exports.verboseWarn = exports.verboseLog = exports.isVerboseMode = exports.setVerbose = void 0; +exports.logError = exports.logVerbose = exports.errorLog = exports.verboseWarn = exports.verboseLog = exports.isVerboseMode = exports.setVerbose = void 0; let isVerbose = false; /** * Set the global verbose flag @@ -49,3 +49,11 @@ const errorLog = (message, details) => { } }; exports.errorLog = errorLog; +/** + * Alias for verboseLog for consistency + */ +exports.logVerbose = exports.verboseLog; +/** + * Alias for errorLog for consistency + */ +exports.logError = exports.errorLog; diff --git a/dist/utils/skyflow.js b/dist/utils/skyflow.js new file mode 100644 index 0000000..597ada1 --- /dev/null +++ b/dist/utils/skyflow.js @@ -0,0 +1,162 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.resolveVaultId = exports.handleSkyflowError = exports.initializeSkyflowClient = exports.parseEnvironment = exports.extractClusterIdFromUrl = exports.loadSkyflowCredentials = void 0; +const skyflow_node_1 = require("skyflow-node"); +const config_1 = require("./config"); +const logger_1 = require("./logger"); +/** + * Load Skyflow credentials from configuration or environment + */ +const loadSkyflowCredentials = () => { + // Check for API key in environment first + const apiKey = process.env.SKYFLOW_API_KEY; + if (apiKey) { + return { apiKey }; + } + // Check for credentials file path in environment + const credentialsPath = process.env.SKYFLOW_CREDENTIALS_PATH; + if (credentialsPath) { + return { path: credentialsPath }; + } + // Check for credentials string in environment + const credentialsString = process.env.SKYFLOW_CREDENTIALS; + if (credentialsString) { + return { credentialsString }; + } + // Fall back to bearer token from config + const config = (0, config_1.loadConfig)(); + if (config.bearerToken) { + return { token: config.bearerToken }; + } + throw new Error('No Skyflow credentials found. Please set one of:\n' + + ' - SKYFLOW_API_KEY environment variable\n' + + ' - SKYFLOW_CREDENTIALS_PATH environment variable\n' + + ' - SKYFLOW_CREDENTIALS environment variable\n' + + ' - Or run: sky configure'); +}; +exports.loadSkyflowCredentials = loadSkyflowCredentials; +/** + * Extract cluster ID from vault URL + * Example: https://ebfc9bee4242.vault.skyflowapis.com -> ebfc9bee4242 + */ +const extractClusterIdFromUrl = (vaultUrl) => { + const match = vaultUrl.match(/https?:\/\/([^.]+)\.vault\.skyflowapis/); + if (!match || !match[1]) { + throw new Error(`Invalid vault URL format: ${vaultUrl}`); + } + return match[1]; +}; +exports.extractClusterIdFromUrl = extractClusterIdFromUrl; +/** + * Parse environment string to Env enum + */ +const parseEnvironment = (env) => { + const envUpper = (env || 'PROD').toUpperCase(); + switch (envUpper) { + case 'PROD': + case 'PRODUCTION': + return skyflow_node_1.Env.PROD; + case 'SANDBOX': + return skyflow_node_1.Env.SANDBOX; + case 'STAGE': + case 'STAGING': + return skyflow_node_1.Env.STAGE; + case 'DEV': + case 'DEVELOPMENT': + return skyflow_node_1.Env.DEV; + default: + (0, logger_1.logVerbose)(`Unknown environment '${env}', defaulting to PROD`); + return skyflow_node_1.Env.PROD; + } +}; +exports.parseEnvironment = parseEnvironment; +/** + * Initialize Skyflow client with vault configuration + */ +const initializeSkyflowClient = (vaultId, clusterId, environment, verbose = false) => { + (0, logger_1.logVerbose)('Initializing Skyflow client...'); + // Load credentials + const credentials = (0, exports.loadSkyflowCredentials)(); + (0, logger_1.logVerbose)('Credentials loaded successfully'); + // If no cluster ID provided, try to get from config or environment + let resolvedClusterId = clusterId; + if (!resolvedClusterId) { + const vaultUrl = process.env.SKYFLOW_VAULT_URL; + if (vaultUrl) { + resolvedClusterId = (0, exports.extractClusterIdFromUrl)(vaultUrl); + (0, logger_1.logVerbose)(`Cluster ID extracted from vault URL: ${resolvedClusterId}`); + } + else { + throw new Error('Cluster ID not provided. Please provide --cluster-id or set SKYFLOW_VAULT_URL environment variable'); + } + } + // Parse environment + const env = (0, exports.parseEnvironment)(environment); + (0, logger_1.logVerbose)(`Using environment: ${environment || 'PROD'}`); + // Create vault configuration + const vaultConfig = { + vaultId, + clusterId: resolvedClusterId, + env, + credentials, + }; + // Create Skyflow configuration + const skyflowConfig = { + vaultConfigs: [vaultConfig], + skyflowCredentials: credentials, + logLevel: verbose ? skyflow_node_1.LogLevel.INFO : skyflow_node_1.LogLevel.ERROR, + }; + // Initialize and return client + const client = new skyflow_node_1.Skyflow(skyflowConfig); + (0, logger_1.logVerbose)('Skyflow client initialized successfully'); + return client; +}; +exports.initializeSkyflowClient = initializeSkyflowClient; +/** + * Handle Skyflow-specific errors with user-friendly messages + */ +const handleSkyflowError = (error) => { + if (error instanceof skyflow_node_1.SkyflowError) { + const errorInfo = error.error; + const message = error.message; + // Build detailed error message + const errorParts = ['Skyflow API Error:']; + if (errorInfo === null || errorInfo === void 0 ? void 0 : errorInfo.http_code) { + errorParts.push(` HTTP Code: ${errorInfo.http_code}`); + } + if (message) { + errorParts.push(` Message: ${message}`); + } + if ((errorInfo === null || errorInfo === void 0 ? void 0 : errorInfo.details) && Array.isArray(errorInfo.details)) { + errorParts.push(` Details: ${errorInfo.details.join(', ')}`); + } + if (errorInfo === null || errorInfo === void 0 ? void 0 : errorInfo.request_ID) { + errorParts.push(` Request ID: ${errorInfo.request_ID}`); + } + (0, logger_1.logError)(errorParts.join('\n')); + process.exit(1); + } + else if (error instanceof Error) { + (0, logger_1.logError)(`Error: ${error.message}`); + process.exit(1); + } + else { + (0, logger_1.logError)(`Unknown error: ${String(error)}`); + process.exit(1); + } +}; +exports.handleSkyflowError = handleSkyflowError; +/** + * Get vault ID from options or environment + */ +const resolveVaultId = (vaultId) => { + if (vaultId) { + return vaultId; + } + const envVaultId = process.env.SKYFLOW_VAULT_ID; + if (envVaultId) { + return envVaultId; + } + throw new Error('Vault ID not provided. Please provide --vault-id or set SKYFLOW_VAULT_ID environment variable'); +}; +exports.resolveVaultId = resolveVaultId; diff --git a/docs/implementation-plan-skyflow-v2.md b/docs/implementation-plan-skyflow-v2.md new file mode 100644 index 0000000..7fa91fa --- /dev/null +++ b/docs/implementation-plan-skyflow-v2.md @@ -0,0 +1,299 @@ +# Implementation Plan: Skyflow Node v2 SDK Integration + +## Overview +This document outlines the plan to integrate the skyflow-node v2 SDK into the CLI, adding three new commands: `insert`, `deidentify`, and `reidentify`. + +## Phase 1: SDK Setup & Configuration + +### 1.1 Install Dependencies +- SDK already installed (`skyflow-node: ^2.3.0`) +- Verify package.json has correct version + +### 1.2 Create Shared SDK Utilities (`src/utils/skyflow.ts`) +- `initializeSkyflowClient()` - Initialize Skyflow client from config +- `loadSkyflowCredentials()` - Load credentials from config/environment +- `handleSkyflowError()` - Centralized error handling for SkyflowError + +### 1.3 Update Types (`src/types.ts`) +- Add interfaces for Insert, Deidentify, Reidentify operations +- Add DetectEntity enums and token type definitions + +### 1.4 Extend Configuration +- Update `~/.skyflow/config.json` schema to include: + - `clusterId` (extracted from vault URL) + - `environment` (PROD, SANDBOX, STAGE, DEV) + - Credentials storage options + +## Phase 2: Command Implementation + +### 2.1 Insert Command (`src/commands/insert.ts`) +**Purpose**: Insert sensitive data into vault table + +**Required Options**: +- `--table ` - Target table name +- `--data ` - JSON data to insert + +**Optional Options**: +- `--return-tokens` - Return tokens for inserted data +- `--continue-on-error` - Continue if some records fail +- `--upsert-column ` - Column name for upsert operations + +**Flow**: +1. Parse JSON data or accept from stdin +2. Initialize Skyflow client with vault config +3. Create `InsertRequest` with table and data +4. Configure `InsertOptions` based on flags +5. Execute insert and display tokens/IDs + +**Example Usage**: +```bash +sky-cli insert --table credit_cards --data '{"card_number":"4111111111111111"}' --return-tokens +``` + +### 2.2 Deidentify Command (`src/commands/deidentify.ts`) +**Purpose**: Detect and redact sensitive data from text + +**Required Options**: +- `--text ` - Text to deidentify (or accept from stdin) + +**Optional Options**: +- `--entities ` - Comma-separated entity types to detect (SSN, CREDIT_CARD, EMAIL, etc.) +- `--token-type ` - Token format: vault_token, entity_only, random_token +- `--output ` - Output format: text, json + +**Flow**: +1. Accept text from flag or stdin +2. Initialize Detect API client +3. Create `DeidentifyTextRequest` +4. Configure entities (SSN, CREDIT_CARD, EMAIL, etc.) +5. Set token format (VAULT_TOKEN, ENTITY_ONLY, RANDOM_TOKEN) +6. Execute and display processed text + entity metadata + +**Example Usage**: +```bash +sky-cli deidentify --text "My SSN is 123-45-6789" --entities SSN,CREDIT_CARD +echo "sensitive data" | sky-cli deidentify --entities EMAIL,PHONE_NUMBER +``` + +### 2.3 Reidentify Command (`src/commands/reidentify.ts`) +**Purpose**: Restore original values from tokenized text + +**Required Options**: +- `--text ` - Tokenized text to reidentify (or accept from stdin) + +**Optional Options**: +- `--plain-text ` - Entities to return as plain text +- `--masked ` - Entities to return masked +- `--redacted ` - Entities to keep redacted + +**Flow**: +1. Accept tokenized text from flag or stdin +2. Initialize Detect API client +3. Create `ReidentifyTextRequest` +4. Configure entity display options +5. Execute and display restored text + +**Example Usage**: +```bash +sky-cli reidentify --text "My SSN is [SSN_0ykQWPA]" --plain-text SSN +echo "[token] data" | sky-cli reidentify +``` + +## Phase 3: Integration & Testing + +### 3.1 Update Main CLI (`src/index.ts`) +- Register new commands: `insert`, `deidentify`, `reidentify` +- Add pre-action hooks for credential validation + +### 3.2 Add Prompts +- Interactive mode for missing required options +- Entity selection prompt for deidentify (multi-select) + +### 3.3 Error Handling +- Implement SkyflowError handling +- User-friendly error messages +- Proper exit codes + +### 3.4 Documentation +- Update README with new command examples +- Add usage examples for each command +- Document configuration requirements + +## Phase 4: Enhancements + +### 4.1 Batch Operations +- Support file input for bulk insert operations +- Process multiple texts for deidentify/reidentify + +### 4.2 Output Formatting +- JSON output mode (`--json`) +- Table format for structured data +- Quiet mode for scripting + +### 4.3 Advanced Features +- Date shifting transformations for deidentify +- Regex allow/restrict lists +- Upsert support for insert + +## Technical Details + +### Authentication Priority (from SDK docs) +1. Credentials in vault config +2. Skyflow credentials in client config +3. `SKYFLOW_CREDENTIALS` environment variable + +### Client Initialization Pattern +```typescript +import { Skyflow, VaultConfig, Env, LogLevel } from 'skyflow-node'; + +const vaultConfig: VaultConfig = { + vaultId: config.vaultId, + clusterId: config.clusterId, + env: Env.PROD, + credentials: { apiKey: config.apiKey } +}; + +const skyflowClient = new Skyflow({ + vaultConfigs: [vaultConfig], + logLevel: verbose ? LogLevel.INFO : LogLevel.ERROR +}); +``` + +### Entity Types Available +- SSN +- CREDIT_CARD (alias: CREDIT_CARD_NUMBER) +- EMAIL +- PHONE_NUMBER +- NAME +- DOB +- ACCOUNT_NUMBER +- DRIVER_LICENSE +- PASSPORT_NUMBER +- And more... + +### Token Types +- `VAULT_TOKEN` - Stored in vault, returns token reference +- `ENTITY_ONLY` - Returns entity type only (e.g., `[SSN]`) +- `RANDOM_TOKEN` - Generates random token without vault storage + +### Insert Operation Details +```typescript +import { InsertRequest, InsertOptions, InsertResponse } from 'skyflow-node'; + +const insertData = [{ field: 'value' }]; +const insertReq = new InsertRequest('table_name', insertData); +const insertOptions = new InsertOptions(); +insertOptions.setReturnTokens(true); +insertOptions.setContinueOnError(true); + +const response: InsertResponse = await skyflowClient + .vault(vaultId) + .insert(insertReq, insertOptions); +``` + +### Deidentify Operation Details +```typescript +import { + DeidentifyTextRequest, + DeidentifyTextOptions, + DetectEntities, + TokenFormat, + TokenType +} from 'skyflow-node'; + +const request = new DeidentifyTextRequest('text to process'); +const options = new DeidentifyTextOptions(); +options.setEntities([DetectEntities.SSN, DetectEntities.CREDIT_CARD]); + +const tokenFormat = new TokenFormat(); +tokenFormat.setDefault(TokenType.VAULT_TOKEN); +options.setTokenFormat(tokenFormat); + +const response = await skyflowClient + .detect(vaultId) + .deidentifyText(request, options); +``` + +### Reidentify Operation Details +```typescript +import { + ReidentifyTextRequest, + ReidentifyTextOptions, + DetectEntities +} from 'skyflow-node'; + +const request = new ReidentifyTextRequest('[token] text'); +const options = new ReidentifyTextOptions(); +options.setPlainTextEntities([DetectEntities.SSN]); +options.setMaskedEntities([DetectEntities.CREDIT_CARD]); + +const response = await skyflowClient + .detect(vaultId) + .reidentifyText(request, options); +``` + +### Error Handling Pattern +```typescript +import { SkyflowError } from 'skyflow-node'; + +try { + // Skyflow operation +} catch (error) { + if (error instanceof SkyflowError) { + console.error('Skyflow Error:', { + code: error.error?.http_code, + message: error.message, + details: error.error?.details, + requestId: error.error?.request_ID + }); + } else { + console.error('Unexpected Error:', error); + } +} +``` + +## Configuration Schema Extension + +Update `~/.skyflow/config.json` to include: + +```json +{ + "accountId": "existing field", + "apiKey": "existing field", + "vaultId": "existing field", + "vaultUrl": "existing field", + "clusterId": "extracted from vaultUrl", + "environment": "PROD", + "credentials": { + "type": "apiKey", + "value": "..." + } +} +``` + +## Estimated Implementation Timeline + +1. Create `src/utils/skyflow.ts` utilities - **30 minutes** +2. Implement insert command - **45 minutes** +3. Implement deidentify command - **45 minutes** +4. Implement reidentify command - **30 minutes** +5. Integration, testing, documentation - **60 minutes** + +**Total**: ~3.5 hours for core functionality + +## Dependencies + +Current `package.json` already includes: +```json +{ + "skyflow-node": "^2.3.0" +} +``` + +No additional dependencies required for core functionality. + +## References + +- Local SDK Documentation: `/docs/skyflow-node/readme.md` +- Local SDK Samples: `/docs/skyflow-node/samples/` +- GitHub Samples (v2.0.0): https://github.com/skyflowapi/skyflow-node/tree/2.0.0/samples diff --git a/docs/skyflow-node/readme.md b/docs/skyflow-node/readme.md new file mode 100644 index 0000000..b0d87bf --- /dev/null +++ b/docs/skyflow-node/readme.md @@ -0,0 +1,3401 @@ + +# Skyflow Node.js SDK + +SDK for the Skyflow Data Privacy Vault. + +[![CI](https://img.shields.io/static/v1?label=CI&message=passing&color=green?style=plastic&logo=github)](https://github.com/skyflowapi/skyflow-node/actions) +[![GitHub release](https://badge.fury.io/js/skyflow-node.svg)](https://www.npmjs.com/package/skyflow-node) +[![License](https://img.shields.io/github/license/skyflowapi/skyflow-node)](https://github.com/skyflowapi/skyflow-node/blob/main/LICENSE) + + +## Table of contents + +- [Skyflow Node.js SDK](#skyflow-nodejs-sdk) + - [Table of contents](#table-of-contents) +- [Overview](#overview) +- [Install](#install) + - [Requirements](#requirements) + - [Import / Require](#import--require) + - [Require](#require) + - [ES modules](#es-modules) + - [All imports](#all-imports) + - [Migrate from V1 to V2](#migrate-from-v1-to-v2) + - [Authentication Options](#authentication-options) + - [V1 (Old): Passing the auth function below as a parameter to the getBearerToken key.](#v1-old-passing-the-auth-function-below-as-a-parameter-to-the-getbearertoken-key) + - [V2 (New): Passing one of the following:](#v2-new-passing-one-of-the-following) + - [Notes:](#notes) + - [Initializing the client](#initializing-the-client) + - [V1 (Old)](#v1-old) + - [V2 (New)](#v2-new) + - [Key Changes:](#key-changes) + - [Request \& Response Structure](#request--response-structure) + - [V1 (Old) - Request Building](#v1-old---request-building) + - [V2 (New) - Request Building](#v2-new---request-building) + - [V1 (Old) - Response Structure](#v1-old---response-structure) + - [V2 (New) - Response Structure](#v2-new---response-structure) + - [Request Options](#request-options) + - [V1 (Old)](#v1-old-1) + - [V2 (New)](#v2-new-1) + - [Error Structure](#error-structure) + - [V1 (Old) - Error Structure](#v1-old---error-structure) + - [V2 (New) - Error Structure](#v2-new---error-structure) + - [Quickstart](#quickstart) + - [Authenticate](#authenticate) + - [Initialize the client](#initialize-the-client) + - [Insert data into the vault](#insert-data-into-the-vault) + - [Vault](#vault) + - [Insert data into the vault](#insert-data-into-the-vault-1) + - [Construct an insert request](#construct-an-insert-request) + - [Insert call example with `continueOnError` option](#insert-call-example-with-continueonerror-option) + - [Detokenize](#detokenize) + - [Construct a detokenize request](#construct-a-detokenize-request) + - [An example of a detokenize call](#an-example-of-a-detokenize-call) + - [An example of a detokenize call with `continueOnError` option:](#an-example-of-a-detokenize-call-with-continueonerror-option) + - [Tokenize](#tokenize) + - [Construct a tokenize request](#construct-a-tokenize-request) + - [An example of Tokenize call](#an-example-of-tokenize-call) + - [Get](#get) + - [Construct a get request](#construct-a-get-request) + - [Get by skyflow IDs](#get-by-skyflow-ids) + - [Get tokens](#get-tokens) + - [An example of get call to retrieve tokens using Skyflow IDs:](#an-example-of-get-call-to-retrieve-tokens-using-skyflow-ids) + - [Get by column name and column values](#get-by-column-name-and-column-values) + - [An example of get call to retrieve data using column name and column values:](#an-example-of-get-call-to-retrieve-data-using-column-name-and-column-values) + - [Redaction Types](#redaction-types) + - [Update](#update) + - [Construct an update request](#construct-an-update-request) + - [An example of update call](#an-example-of-update-call) + - [Delete](#delete) + - [Construct a delete request](#construct-a-delete-request) + - [An example of delete call](#an-example-of-delete-call) + - [Query](#query) + - [Construct a query request](#construct-a-query-request) + - [An example of query call](#an-example-of-query-call) + - [Detect](#detect) + - [Deidentify Text](#deidentify-text) + - [An example of a deidentify text call](#an-example-of-a-deidentify-text-call) + - [Reidentify Text](#reidentify-text) + - [An example of a reidentify text call](#an-example-of-a-reidentify-text-call) + - [Deidentify File](#deidentify-file) + - [An example of a deidentify file](#an-example-of-a-deidentify-file) + - [Get run](#get-run) + - [An example of a get run function](#an-example-of-a-get-run-function) + - [Connections](#connections) + - [Invoke a connection](#invoke-a-connection) + - [Construct an invoke connection request](#construct-an-invoke-connection-request) + - [An example of Invoke Connection](#an-example-of-invoke-connection) + - [Authenticate with bearer tokens](#authenticate-with-bearer-tokens) + - [Generate a bearer token](#generate-a-bearer-token) + - [Example:](#example) + - [Generate bearer tokens with context](#generate-bearer-tokens-with-context) + - [Example:](#example-1) + - [Generate scoped bearer tokens](#generate-scoped-bearer-tokens) + - [Example:](#example-2) + - [Generate signed data tokens](#generate-signed-data-tokens) + - [Example:](#example-3) + - [Bearer token expiry edge case](#bearer-token-expiry-edge-case) + - [Example:](#example-4) + - [Logging](#logging) + - [Reporting a Vulnerability](#reporting-a-vulnerability) + +# Overview + +- Authenticate using a Skyflow service account and generate bearer tokens for secure access. +- Perform Vault API operations such as inserting, retrieving, and tokenizing sensitive data with ease. +- Invoke connections to third-party APIs without directly handling sensitive data, ensuring compliance and data protection. + +# Install + +#### Requirements + +- Node 12.22.12 and above + +```sh +npm install skyflow-node +``` + +#### Import / Require + +Depending on your project setup, you may use either the `require` method (common in Node.js projects) or the `import` statement (common in projects using ES modules). + +##### Require + +```typescript +const { Skyflow } = require('skyflow-node'); +``` + +##### ES modules + +```typescript +import { Skyflow } from 'skyflow-node'; +``` + +##### All imports + +```typescript +import { + Skyflow, // Vault client + isExpired, // JWT auth helpers + LogLevel, // logging options +} from 'skyflow-node' +``` + +## Migrate from V1 to V2 + +This guide outlines the steps required to migrate the Node SDK from version 1 (V1) to version 2 (V2). + +--- + +### Authentication Options + +In V2, multiple authentication options have been introduced. You can now provide credentials in the following ways: + +- **API Key** +- **Environment Variable** (`SKYFLOW_CREDENTIALS`) (**Recommended**) +- **Path to Credentials JSON File** +- **Stringified JSON of Credentials** +- **Bearer Token** + +These options allow you to choose the authentication method that best suits your use case. + +#### V1 (Old): Passing the auth function below as a parameter to the getBearerToken key. + + +```javascript +// sample function to retrieve a bearer token from an environment variable +// customize this according to your environment and security posture +const auth = function () { + return new Promise((resolve, reject) => { + resolve(process.env.VAULT_BEARER_TOKEN); + }); +}; +``` + +#### V2 (New): Passing one of the following: + +```javascript +// Option 1: API Key (Recommended) +const credentials = { apiKey: "" }; + +// Option 2: Environment Variables (Recommended) +// Set SKYFLOW_CREDENTIALS in your environment + +// Option 3: Credentials File +const credentials = { path: "" }; // Replace with the path to credentials file + + +// Option 4: Stringified JSON +const credentials = { credentialsString: JSON.stringify(process.env.SKYFLOW_CREDENTIALS) }; + +// Option 5: Bearer Token +const credentials = { token: "" }; +``` + +### Notes: +- Use only ONLY authentication method. +- API Key or Environment Variables are recommended for production use. +- Secure storage of credentials is essential. +- For overriding behavior and priority order of credentials, please refer to [Initialize the client](#initialize-the-client) section in [Quickstart](#quickstart). + +--- + +### Initializing the client + +V2 introduces TypeScript support and multi-vault support, allowing you to configure multiple vaults during client initialization. + +In V2, the log level is tied to each individual client instance. + +During client initialization, you can pass the following parameters: + +- **`vaultId`** and **`clusterId`**: These values are derived from the vault ID & vault URL. +- **`env`**: Specify the environment (e.g., SANDBOX or PROD). +- **`credentials`**: The necessary authentication credentials. + + +#### V1 (Old) +```javascript +// Initialize the Skyflow Vault client + +const vault = Skyflow.init({ + // Id of the vault that the client should connect to. + vaultID: 'string', + // URL of the vault that the client should connect to. + + vaultURL: 'string', + // Helper function generates a Skyflow bearer token. + getBearerToken: auth, +}); +``` + +#### V2 (New) +```javascript +// Step 1: Configure Bearer Token Credentials +const credentials: Credentials = { apiKey: '' }; + +// Step 2: Configure Vault +const primaryVaultConfig: VaultConfig = { + vaultId: '', // Primary vault + clusterId: '', // ID from your vault URL e.g., https://{clusterId}.vault.skyflowapis.com + env: Env.PROD, // Deployment environment (PROD by default) + credentials: credentials, // Authentication method +}; + +// Step 3: Configure Skyflow Client +const skyflowConfig: SkyflowConfig = { + vaultConfigs: [primaryVaultConfig], + skyflowCredentials: credentials, // Skyflow credentials will be used if no individual credentials are passed + logLevel: LogLevel.INFO, // Recommended to use LogLevel.ERROR in production environment. +}; + +// Initialize Skyflow Client +const skyflowClient: Skyflow = new Skyflow(skyflowConfig); +``` + +### Key Changes: +- `vaultURL` replaced with `clusterId`. +- Added environment specification (`env`). +- Instance-specific log levels. +- TypeScript support with type definitions + +--- + +### Request & Response Structure + +In V2, with the introduction of TypeScript support, you can now pass an **InsertRequest** of type **InsertRequest**. This request need +- **`tableName`**: The name of the table. +- **`insertData`**: An array of objects containing the data to be inserted +The response will be of type InsertResponse, which contains insertedFields and errors. + +#### V1 (Old) - Request Building +```javascript +const result = skyflow.insert({ + records: [ + { + fields: { + card_number: '411111111111111', + expiry_date: '11/22', + fullname: 'firstNameTest', + }, + table: 'cards', + }, + ], + }); +``` + +#### V2 (New) - Request Building +```typescript +// Prepare Insertion Data +const insertData: Record[] = [ + { card_number: '4111111111111112' } // Example sensitive data +]; + +// Create Insert Request +const insertReq: InsertRequest = new InsertRequest( + '', // Replace with your actual table name + insertData +); + +// Perform Secure Insertion +const response: InsertResponse = await skyflowClient + .vault("") + .insert(insertReq); +``` + +#### V1 (Old) - Response Structure +```json +{ + "records": [ + { + "table": "cards", + "fields": { + "card_number": "f37186-e7e2-466f-91e5-48e2bcbc1", + "expiry_date": "1989cb56-63a-4482-adf-1f74cd1a5" + } + } + ] +} +``` + +#### V2 (New) - Response Structure +```javascript +InsertResponse( + insertedFields : [ + { + skyflowId : "ID1", + "": "", // removed tokens key + "": "" + }, + { + skyflowId : "ID2", + "": "", + "": "" + } + ], + errors: null +); +``` + +--- + +### Request Options + +In V2, we have introduced inbuilt **InsertOptions** classes. These allow you to use setters to configure options instead of passing a plain object with key-value pairs. + + +#### V1 (Old) +```javascript +const options = { + tokens: true, + // other options +}; +``` + +#### V2 (New) +```javascript +const insertOptions: InsertOptions = new InsertOptions(); +insertOptions.setReturnTokens(true); // Optional: Get tokens for inserted data +insertOptions.setContinueOnError(true); // Optional: Continue on partial errors +``` + +--- + +### Error Structure +In V2, we have enriched the error details to provide better debugging capabilities. +The error response now includes: +- **`http_status`**: The HTTP status code. . +- **`grpc_code`**: The gRPC code associated with the error. +- **`details & message`**: A detailed description of the error. +- **`request_ID`**: A unique request identifier for easier debugging. + + +#### V1 (Old) - Error Structure +```javascript +{ + code: string | number, + description: string +} +``` + +#### V2 (New) - Error Structure +```javascript +{ + http_status?: string | number | null, + grpc_code?: string | number | null, + http_code: string | number | null, + message: string, + request_ID?: string | null, + details?: Array | null, +} +``` + +## Quickstart +Get started quickly with the essential steps: authenticate, initialize the client, and perform a basic vault operation. This section provides a minimal setup to help you integrate the SDK efficiently. + +### Authenticate +You can use an API key to authenticate and authorize requests to an API. For authenticating via bearer tokens and different supported bearer token types, refer to the [Authenticate with bearer tokens](#authenticate-with-bearer-tokens) section. + +```javascript +// create a new credentials object +const credentials = { apiKey: "" }; //add your API key in credentials +``` + +### Initialize the client + +To get started, you must first initialize the skyflow client. While initializing the skyflow client, you can specify different types of credentials. +1. **API keys** +A unique identifier used to authenticate and authorize requests to an API. + +2. **Bearer tokens** +A temporary access token used to authenticate API requests, typically included in the +Authorization header. + +3. **Service account credentials file path** +The file path pointing to a JSON file containing credentials for a service account, used +for secure API access. + +4. **Service account credentials string** +JSON-formatted string containing service account credentials, often used as an alternative to a file for programmatic authentication. + +Note: Only one type of credential can be used at a time. If multiple credentials are provided, the last one added will take precedence. + + +```javascript +import { + Credentials, + Env, + LogLevel, + Skyflow, + VaultConfig, + SkyflowConfig, +} from 'skyflow-node'; + +/* +Example program to initialize the Skyflow client with various configurations. +The Skyflow client facilitates secure interactions with the Skyflow vault, +such as securely managing sensitive data. +*/ + +// Step 1: Define the primary credentials for authentication. +// Note: Only one type of credential can be used at a time. You can choose between: +// - API key +// - Bearer token +// - A credentials string (JSON-formatted) +// - A file path to a credentials file. + +// Initialize primary credentials using a Bearer token for authentication. +const primaryCredentials: Credentials = { ///////////////// + token: '', // Replace with your actual authentication token. +}; + +// Step 2: Configure the primary vault details. +// VaultConfig stores all necessary details to connect to a specific Skyflow vault. + +const primaryVaultConfig: VaultConfig = { + vaultId: '', // Replace with your primary vaulID + clusterId: '', // Replace with the cluster ID (partthe vault URL, e.g., https://{clusterId}.vault.skyflowapis.com). + env: Env.PROD, // Set the environment (PRSANDBOX, STAGE, DEV). + credentials: primaryCredentials // Attach the primcredentials to this vault configuration. +}; + +// Step 3: Create credentials as a JSON object (if a Bearer Token is not provided). +// Demonstrates an alternate approach to authenticate with Skyflow using a credentials object. +const skyflowCredentials: object = { + clientID: '', // Replace with your Client ID. + clientName: '', // Replace with your Client Name. + tokenURI: '', // Replace with the Token URI. + keyID: '', // Replace with your Key ID. + privateKey: '' // Replace with your Private Key. +} + +// Step 4: Convert the JSON object to a string and use it as credentials. +// This approach allows the use of dynamically generated or pre-configured credentials. +const credentialsString: JSON.stringify(skyflowCredentials), // Converts JSON object to string for use as credentials. + +// Step 5: Define secondary credentials (API key-based authentication as an example). +// Demonstrates a different type of authentication mechanism for Skyflow vaults. +const secondaryCredentials: Credentials = { + apiKey: '', // Replace with your API Key for authentication. +} + +// Step 6: Configure the secondary vault details. +// A secondary vault configuration can be used for operations involving multiple vaults. +const secondaryVaultConfig: VaultConfig = { + vaultId: '', // Replace with your secondary vault's ID. + clusterId: '', // Replace with the corresponding cluster ID. + env: Env.PROD, // Set the environment for this vault. + credentials: secondaryCredentials // Attach the secondary credentials to this configuration. +} + +// Step 7: Define tertiary credentials using a path to a credentials JSON file. +// This method demonstrates an alternative authentication method. +const tertiaryCredentials: Credentials = { + path: '' // Replace with the path to your credentials file. +} + +// Step 8: Configure the tertiary vault details. +const tertiaryVaultConfig: VaultConfig = { + vaultId: '', // Replace with the tertiary vault ID. + clusterId: '', // Replace with the corresponding cluster ID. + env: Env.PROD, // Set the environment for this vault. + credentials: tertiaryCredentials // Attach the tertiary credentials. +} + +// Step 9: Build and initialize the Skyflow client after creating Skyflow Config +// Skyflow client is configured with multiple vaults and credentials. + +const skyflowConfig: SkyflowConfig = { + vaultConfigs: [primaryVaultConfig, secondaryVaultConfig, tertiaryVaultConfig], // Add the primary, secondary and tertiary vault configurations. + skyflowCredentials: skyflowCredentials, // Add JSON-formatted credentials if applicable. + logLevel: LogLevel.INFO // Recommended to use LogLevel.ERROR in production environment. +}; + +// Step 10: Initialize Skyflow Client +const skyflowClient: Skyflow = new Skyflow(skyflowConfig); + +// The Skyflow client is now fully initialized. +// Use the `skyflowClient` object to perform secure operations such as: +// - Inserting data +// - Retrieving data +// - Deleting data +// within the configured Skyflow vaults. + +``` + +Notes +- If both Skyflow common credentials and individual credentials at the configuration level are specified, the individual credentials at the configuration level will take precedence. +- If neither Skyflow common credentials nor individual configuration-level credentials are provided, the SDK attempts to retrieve credentials from the ```SKYFLOW_CREDENTIALS``` environment variable. +- All Vault operations require a client instance. + +### Insert data into the vault +To insert data into your vault, use the `insert` method. The `InsertRequest` class creates an insert request, which includes the values to be inserted as a list of records. Below is a simple example to get started. For advanced options, check out [Insert data into the vault](#insert-data-into-the-vault-1) section. + +```javascript +import { + InsertOptions, + InsertRequest, + SkyflowError, + InsertResponse +} from 'skyflow-node'; + +/* +* This example demonstrates how to insert sensitive data (e.g., cardinformation) into a Skyflow vault using the Skyflow client. +* +* 1. Initializes the Skyflow client. +* 2. Prepares a record with sensitive data (e.g., card number and cardholdername). +* 3. Creates an insert request for inserting the data into the Skyflow vault. +* 4. Prints the response of the insert operation. +*/ + +try{ + // Step 1: Initialize data to be inserted into the Skyflow vault + const insertData: Record[] = [ + { + card_number: '4111111111111112', // Replace with actual card number (sensitive data) + cardholder_name: 'John Doe', // Replace with actual cardholder name (sensitive data) + } + ]; + + // Step 2: Create Insert Request + const insertReq: InsertRequest = new InsertRequest( + 'table1', // Specify the table in the vault where the data will inserted + insertData, // Attach the data (records) to be inserted + + ); + + // Step 3: Configure Insertion Options + const insertOptions: InsertOptions = new InsertOptions(); + insertOptions.setReturnTokens(true); // Optional: Specify if tokens should be returned upon successful insertion + insertOptions.setContinueOnError(true); // Optional: Continue on partial errors + + // Step 4: Perform the insert operation using the Skyflow client + const insertResponse: InsertResponse = await skyflowClient + .vault(primaryVaultConfig.vaultId) + .insert(insertReq, insertOptions); + + // Step 5: Print the response from the insert operation + console.log('Insert response: ', insertResponse); + +} catch(error) { + // Step 6: Comprehensive Error Handling + if (error instanceof SkyflowError) { + console.error('Skyflow Specific Error:', { + code: error.error?.http_code, + message: error.message, + details: error.error?.details + }); + } else { + console.error('Unexpected Error:', error); + } +} +``` +Skyflow returns tokens for the record that was just inserted. + +```javascript +InsertResponse { + insertedFields: { + skyflowId: 'a8f3ed5d-55eb-4f32-bf7e-2dbf4b9d9097', + card_number: '5484-7829-1702-9110', + cardholder_name: 'b2308e2a-c1f5-469b-97b7-1f193159399b' + }, + errors: null +} +``` + +--- + +## Vault +The [Vault](https://github.com/skyflowapi/skyflow-node/tree/v2/src/vault) performs operations on the vault such as inserting records, detokenizing tokens, retrieving tokens for list of `skyflow_id`'s and to invoke the Connection. + +### Insert data into the vault + +Apart from using the `insert` method to insert data into your vault covered in [Quickstart](#quickstart), you can also pass options to `insert` method, such as returning tokenized data, upserting records, or continuing the operation in case of errors. + +#### Construct an insert request + +```typescript +import { + InsertOptions, + InsertRequest, + SkyflowError, + InsertResponse +} from 'skyflow-node'; + +// Example program to demonstrate inserting data into a Skyflow vault, +// along with corresponding InsertRequest schema. + +try { + // Initialize Skyflow client + // Step 1: Prepare the data to be inserted into the Skyflow vault + const insertData: Record[] = [ + { + : '', // Replace with actual fielname and value + : '', // Replace with actual fielname and value + }, + { + : '', // Replace with actual fielname and value + : '', // Replace with actual fielname and value + }, + ] + + // Step 2: Build an InsertRequest object with the table name and the data to insert + const insertReq: InsertRequest = new InsertRequest( + 'table1', // Specify the table in the vault where the data will be inserted + insertData, // Attach the data (records) to be inserted + ); + + // Step 3: Perform the insert operation using the Skyflow client + const insertResponse: InsertResponse = await skyflowClient + .vault('') + .insert(insertReq, insertOptions); + // Replace with your actual vault ID + + // Step 4: Print the response from the insert operation + console.log('Insert response: ', insertResponse); +} catch(error) { + // Step 5: Comprehensive Error Handling + if (error instanceof SkyflowError) { + console.error('Skyflow Specific Error:', { + code: error.error?.http_code, + message: error.message, + details: error.error?.details + }); + } else { + console.error('Unexpected Error:', error); + } +} +``` + +#### Insert call example with `continueOnError` option +The `continueOnError` flag is a boolean that determines whether insert operation should proceed despite encountering partial errors. Set to `true` to allow the process to continue even if some errors occur. + +```typescript +import { + InsertOptions, + InsertRequest, + SkyflowError, + InsertResponse +} from 'skyflow-node'; + +/* +This example demonstrates how to insert sensitive data (e.g., card information) into a Skyflow vault using the Skyflow client. + +1. Initializes the Skyflow client. +2. Prepares a record with sensitive data (e.g., card number and cardholder name). +3. Creates an insert request for inserting the data into the Skyflow vault. +4. Specifies options to continue on error and return tokens. +5. Prints the response of the insert operation. +*/ + +try { + // Initialize Skyflow client + // Step 1: Initialize a list to hold the data records to be inserted into the vault + const insertData: Record[] = [ + // Step 2: Create the first record with card number and cardholder name + { + card_number: '4111111111111111', // Replace with actual card number (sensitive data) + cardholder_name: 'John Doe', // Replace with actual cardholder name (sensitive data) + }, + //Step 3: Create the second record with card number and cardholder name + { + card_numbe: '4111111111111111', // Replace with actual card number (sensitive data) + cardholder_name: 'John Doe', // Replace with actual cardholder name (sensitive data) + } + ]; + + // Step 4: Create Insert Request + const insertReq: InsertRequest = new InsertRequest( + 'table1', // Specify the table in the vault where the data will inserted + insertData, // Attach the data (records) to be inserted + ); + + // Step 5: Configure Insertion Options + const insertOptions: InsertOptions = new InsertOptions(); + insertOptions.setReturnTokens(true); // Optional: Specify if tokens should be returned upon successful insertion + insertOptions.setContinueOnError(true); // Optional: Specify to continue inserting records even if an error occurs for some records partial errors + + // Step 6: Perform the insert operation using the Skyflow client + const insertResponse: InsertResponse = await skyflowClient + .vault('9f27764a10f7946fe56b3258e117') + .insert(insertReq, insertOptions); + // Replace the vault ID "9f27764a10f7946fe56b3258e117" with your actual Skyflow vault ID + + // Step 7: Print the response from the insert operation + console.log('Insert response: ', insertResponse); +} catch(error) { + // Step 8: Comprehensive Error Handling + if (error instanceof SkyflowError) { + console.error('Skyflow Specific Error:', { + code: error.error?.http_code, + message: error.message, + details: error.error?.details + }); + } else { + console.error('Unexpected Error:', error); + } +} +``` + +Sample Response +```typescript +InsertResponse { + insertedFields: { + [ + { + card_number: '5484-7829-1702-9110', + requestIndex: 0, + skyflowId: '9fac9201-7b8a-4446-93f8-5244e1213bd1', + cardholder_name: 'b2308e2a-c1f5-469b-97b7-1f193159399' + } + ], + errors: [ + { + requestIndex: 1, + error: 'Insert failed. Column card_numbe is invalid. Specify a valid column.' + } + ] + } +} +``` + +**Insert call example with `upsert` option** +An upsert operation checks for a record based on a unique column's value. If a match exists, the record is updated; otherwise, a new record is inserted. + +```typescript +import { + InsertOptions, + InsertRequest, + SkyflowError, + InsertResponse +} from 'skyflow-node'; + +/* +This example demonstrates how to insert sensitive data (e.g., card information) into a Skyflow vault using the Skyflow client. + +1. Initializes the Skyflow client. +2. Prepares a record with sensitive data (e.g., card number and cardholder name). +3. Creates an insert request for inserting the data into the Skyflow vault. +4. Specifies the field (cardholder_name) for upsert operations. +5. Prints the response of the insert operation. +*/ + +try { + // Initialize Skyflow client + // Step 1: Initialize a list to hold the data records for the insert/upsert operation + const insertData: Record[] = [ + // Step 2: Create a record with the field 'cardholder_name' to insert or upsert + { + cardholder_name: 'John Doe', // Replace with actual cardholder name + } + ] + + // Step 3: Create Insert Request + const insertReq: InsertRequest = new InsertRequest( + 'table1', // Specify the table in the vault where the data will be inserted + insertData, // Attach the data (records) to be inserted + ); + + // Step 4: Set upsert column by configuring the insertion options + const insertOptions: InsertOptions = new InsertOptions(); + insertOptions.setReturnTokens(true); // Optional: Specify if tokens should be returned upon successful insertion + insertOptions.setUpsertColumn('cardholder_name'); + + // Step 5: Perform the insert/upsert operation using the Skyflow client + const insertResponse: InsertResponse = await skyflowClient + .vault(primaryVaultConfig.vaultId) + .insert(insertReq, insertOptions); + + // Step 6: Print the response from the insert operation + console.log('Insert response: ', insertResponse); +} catch(error) { + // Step 7: Comprehensive Error Handling + if (error instanceof SkyflowError) { + console.error('Skyflow Specific Error:', { + code: error.error?.http_code, + message: error.message, + details: error.error?.details + }); + } else { + console.error('Unexpected Error:', error); + } +} +``` + +Skyflow returns tokens, with `upsert` support, for the record you just inserted. + +```typescript +InsertResponse { + insertedFields: [ + { + skyflowId: "9fac9201-7b8a-4446-93f8-5244e1213bd1", + cardholder_name: "73ce45ce-20fd-490e-9310-c1d4f603ee83" + } + ], + errors: null +} +``` + +### Detokenize + +To retrieve tokens from your vault, use the `detokenize` method. The `DetokenizeRequest` class requires a list of detokenization data as input. Additionally, you can provide optional parameters, such as the redaction type and the option to continue on error. + +#### Construct a detokenize request +```typescript +import { + DetokenizeOptions, + DetokenizeRequest, + DetokenizeResponse, + DetokenizeData, + SkyflowError, +} from 'skyflow-node'; + +/* +This example demonstrates how to detokenize sensitive data from tokens stored in a Skyflow vault, along with corresponding DetokenizeRequest schema. +*/ + +try { + // Step 1: Prepare Detokenization Data + const detokenizeData: DetokenizeData[] = [ + { + token: "token1", // Token to be detokenized + redactionType: RedactionType.PLAIN_TEXT, // Redaction type + }, + { + token: "token2", // Token to be detokenized + redactionType: RedactionType.PLAIN_TEXT, // Redaction type + }, + ]; + + // Step 2: Create the DetokenizeRequest object with the tokens data + const detokenizeRequest: DetokenizeRequest = new DetokenizeRequest( + detokenizeData + ); + + // Step 3: Configure Detokenize Options + const detokenizeOptions: DetokenizeOptions = new DetokenizeOptions(); + detokenizeOptions.setContinueOnError(true); // Continue processing on errors + detokenizeOptions.setDownloadURL(false); // Disable download URL generation + + // Step 4: Perform Detokenization + const response: DetokenizeResponse = await skyflowClient + .vault(primaryVaultConfig.vaultId) + .detokenize(detokenizeRequest, detokenizeOptions); + + // Handle Successful Response + console.log('Detokenization response:', response); +} catch(error) { + // Comprehensive Error Handling + if (error instanceof SkyflowError) { + console.error('Skyflow Specific Error:', { + code: error.error?.http_code, + message: error.message, + details: error.error?.details, + }); + } else { + console.error('Unexpected Error:', error); + } +} +``` + +Notes: + +- `redactionType` defaults to `RedactionType.PLAIN_TEXT`. +- `continueOnError` default valus is `False`. + +#### An [example](https://github.com/skyflowapi/skyflow-node/blob/v2/samples/vault-api/detokenzie-records.ts) of a detokenize call + +```typescript +import { + DetokenizeOptions, + DetokenizeRequest, + DetokenizeResponse, + DetokenizeData, + SkyflowError, +} from 'skyflow-node'; + +/* +1. Initializes the Skyflow client. +2. Creates a list of tokens (e.g., credit card tokens) that represent the sensitive data. +3. Builds a detokenization request using the provided tokens and specifies how the redacted data should be returned. +4. Calls the Skyflow vault to detokenize the tokens and retrieves the detokenized data. +5. Prints the detokenization response, which contains the detokenized values or errors. +*/ + +try { + // Step 1: Prepare Detokenization Data + const detokenizeData: DetokenizeData[] = [ + { + token: "9738-1683-0486-1480", // Replace with your actual token value + redactionType: RedactionType.PLAIN_TEXT, // Redaction type + }, + { + token: "6184-6357-8409-6668", // Replace with your actual token value + redactionType: RedactionType.PLAIN_TEXT, // Redaction type + }, + ]; + + // Step 2: Create the DetokenizeRequest object with the tokens data + const detokenizeRequest: DetokenizeRequest = new DetokenizeRequest( + detokenizeData + ); + + // Step 3: Configure Detokenize Options + const detokenizeOptions: DetokenizeOptions = new DetokenizeOptions(); + detokenizeOptions.setContinueOnError(false); // Stop the process if any token cannot be detokenized + + // Step 4: Perform Detokenization + const response: DetokenizeResponse = await skyflowClient + .vault(primaryVaultConfig.vaultId) + .detokenize(detokenizeRequest, detokenizeOptions); + + // Handle Successful Response + console.log('Detokenization response:', response); +} catch(error) { + // Comprehensive Error Handling + if (error instanceof SkyflowError) { + console.error('Skyflow Specific Error:', { + code: error.error?.http_code, + message: error.message, + details: error.error?.details, + }); + } else { + console.error('Unexpected Error:', error); + } +} +``` + +Sample response: + +```typescript +DetokenizeResponse { + detokenizedFields: [ + {token: '9738-1683-0486-1480', value: '4111111111111115', type: 'STRING'}, + {token: '6184-6357-8409-6668', value: '4111111111111119', type: 'STRING'}, + ], + errors: null +} +``` + +#### An example of a detokenize call with `continueOnError` option: + +```typescript +import { + DetokenizeOptions, + DetokenizeRequest, + DetokenizeResponse, + DetokenizeData, + SkyflowError, +} from 'skyflow-node'; + +/* +1. Initializes the Skyflow client. +2. Creates a list of tokens (e.g., credit card tokens) that represent the sensitive data. +3. Builds a detokenization request using the provided tokens and specifies how the redacted data should be returned. +4. Calls the Skyflow vault to detokenize the tokens and retrieves the detokenized data. +5. Prints the detokenization response, which contains the detokenized values or errors. +*/ + +try { + // Step 1: Prepare Detokenization Data + const detokenizeData: DetokenizeData[] = [ + { + token: "9738-1683-0486-1480", // Replace with your actual token value + redactionType: RedactionType.PLAIN_TEXT, // Redaction type + }, + { + token: "6184-6357-8409-6668", // Replace with your actual token value + redactionType: RedactionType.PLAIN_TEXT, // Redaction type + }, + { + token: "4914-9088-2814-3840", // Replace with your actual token value + redactionType: RedactionType.PLAIN_TEXT, // Redaction type + }, + ]; + + // Step 2: Create the DetokenizeRequest object with the tokens and redaction type + const detokenizeRequest: DetokenizeRequest = new DetokenizeRequest( + detokenizeData, + redactionType + ); + + // Step 3: Configure Detokenize Options + const detokenizeOptions: DetokenizeOptions = new DetokenizeOptions(); + detokenizeOptions.setContinueOnError(true); // Continue even if some tokens cannot be detokenized + + // Step 5: Perform Detokenization + const response: DetokenizeResponse = await skyflowClient + .vault(primaryVaultConfig.vaultId) + .detokenize(detokenizeRequest, detokenizeOptions); + + // Handle Successful Response + console.log('Detokenization response:', response); +} catch(error) { + // Comprehensive Error Handling + if (error instanceof SkyflowError) { + console.error('Skyflow Specific Error:', { + code: error.error?.http_code, + message: error.message, + details: error.error?.details, + }); + } else { + console.error('Unexpected Error:', error); + } +} +``` + +Sample response: + +```typescript +DetokenizeResponse { + detokenizedFields: [ + {token: '9738-1683-0486-1480', value: '4111111111111115', type: 'STRING'}, + {token: '6184-6357-8409-6668', value: '4111111111111119', type: 'STRING'} + ], + errors: [ + { + token: '4914-9088-2814-3840', + error: 'Token Not Found' + } + ] +} +``` + +### Tokenize + +Tokenization replaces sensitive data with unique identifier tokens. This approach protects sensitive information by securely storing the original data while allowing the use of tokens within your application. + +To tokenize data, use the `tokenize` method. The `TokenizeRequest` class creates a tokenize request. In this request, you specify the values parameter, which is a list of column values objects. Each column value contains two properties: `value` and `columnGroup`. + +#### Construct a tokenize request + +```typescript +import { + TokenizeRequest, + TokenizeResponse, + SkyflowError, + TokenizeRequestType +} from 'skyflow-node'; + +try { + // Initialize Skyflow Client + // Step 1: Prepare Tokenization Data + const columnvalues: Array = [ + { value: "", columnGroup: "" }, // Replace and with actual data + { value: "", columnGroup: "" } // Replace and with actual data + ]; + + // Step 2: Build the TokenizeRequest with the column value2 + const tokenReq: TokenizeRequest = new TokenizeRequest(columnvalues); + + // Step 3: Call the Skyflow vault to tokenize the sensitive data + const response: TokenizeResponse = await skyflowClient + .vault("") + .tokenize(tokenReq); + // Replace with your actual Skyflow vault ID + + // Step 4: Print the tokenization response, which contains the generated tokens or errors + console.log('Tokenization Result:', response); +} catch(error) { + // Step 5: Handle any errors that occur during the tokenization process + if (error instanceof SkyflowError) { + console.error('Skyflow Specific Error:', { + code: error.error?.http_code, + message: error.message, + details: error.error?.details, + }); + } else { + console.error('Unexpected Error:', error); + } +} +``` + +#### An [example](https://github.com/skyflowapi/skyflow-node/blob/v2/samples/vault-api/tokenize-records.ts) of Tokenize call + +```typescript +import { + TokenizeRequest, + TokenizeResponse, + SkyflowError, + TokenizeRequestType +} from 'skyflow-node'; + +/* +This example demonstrates how to tokenize sensitive data (e.g., credit card information) using the Skyflow client. + +1. Initializes the Skyflow client. +2. Creates a column value for sensitive data (e.g., credit card number). +3. Builds a tokenize request with the column value to be tokenized. +4. Sends the request to the Skyflow vault for tokenization. +5. Prints the tokenization response, which includes the token or errors. +*/ + +try { + // Initialize Skyflow Client + // Step 1: Prepare Tokenization Data + const columnvalues: Array = [ + { value: "4111111111111111", columnGroup: "card_number_cg" } + ]; + + // Step 2: Build the TokenizeRequest with the column value2 + const tokenReq: TokenizeRequest = new TokenizeRequest(columnvalues); + + // Step 3: Call the Skyflow vault to tokenize the sensitive data + const response: TokenizeResponse = await skyflowClient + .vault(primaryVaultConfig.vaultId) + .tokenize(tokenReq); + // Replace primaryVaultConfig.vaultId with your actual Skyflow vault ID + + // Step 4: Print the tokenization response, which contains the generated tokens or errors + console.log('Tokenization Result:', response); +} catch(error) { + if (error instanceof SkyflowError) { + console.error('Skyflow Specific Error:', { + code: error.error?.http_code, + message: error.message, + details: error.error?.details, + }); + } else { + console.error('Unexpected Error:', error); + } +} +``` + +Sample response: + +```typescript +TokenizeResponse { + tokens: [ + { + token: '5479-4229-4622-1393' + } + ], + errors: null +} +``` + +### Get + +To retrieve data using Skyflow IDs or unique column values, use the get method. The `GetRequest` class creates a get request, where you specify parameters such as the table name, redaction type, Skyflow IDs, column names, column values. If you specify Skyflow IDs, you can't use column names and column values, and the inverse is true—if you specify column names and column values, you can't use Skyflow IDs. And `GetOptions` class creates a get options object through which you specify whether to return tokens or not. + +#### Construct a get request + +```typescript +import { + GetOptions, + GetRequest, + GetColumnRequest, + SkyflowError, + GetResponse +} from 'skyflow-node'; + +try { + // Initialize Skyflow client + // Step 1: Initialize a list of Skyflow IDs to retrieve records (replace with actual Skyflow IDs) + const getIds: Array = ['', '']; + + // Step 2: Create a GetRequest to retrieve records by Skyflow ID + const getRequest: GetRequest = new GetRequest( + 'table1', // Replace with your actual table name + getIds + ); + + // Step 3: Configure Get Options and specify not to return tokens + const getOptions: GetOptions = new GetOptions(); + getOptions.setReturnTokens(false); // Optional: Set to false to avoid returning tokens + + // Step 4: Send the request to the Skyflow vault and retrieve the records + const getResponse: GetResponse = await skyflowClient + .vault('') + .get(getRequest, getOptions); + // Replace with your actual Skyflow vault ID + + console.log('Data retrieval successful:', getResponse); + + // Step 5: Create another GetRequest to retrieve records by Skyflow ID with tokenized values + const getTokensRequest: GetRequest = new GetRequest( + 'table1', // Replace with your actual table name + getIds + ); + + // Step 6: Configure Get Options and specify to return tokens + const getOptions: GetOptions = new GetOptions(); + getOptions.setReturnTokens(true); // Optional: Set to True to return tokenized values + + // Step 7: Send the request to the Skyflow vault and retrieve the tokenized records + const getTokensResponse: GetResponse = await skyflowClient + .vault('') + .get(getRequest, getOptions); + // Replace with your actual Skyflow vault ID + + console.log('Data retrieval successful:', getTokensResponse); + + // Prepare Column-Based Retrieval Data + const columnValues: Array = [ + '', // Example Unique Column value 1 + '', // Example Unique Column value 2 + ]; + const tableName: string = 'table-name'; // Replace with your actual table name + const columnName: string = 'column-name'; // Column name configured as unique in the schema + + const getRequest: GetColumnRequest = new GetColumnRequest( + tableName, + columnName, + columnValues // Column values of the records to return + ); + + // Step 8: Configure Get Options and specify to return tokens + const getOptions: GetOptions = new GetOptions(); + getOptions.setReturnTokens(true); // Optional: Set to True to return tokenized values + + // Send the request to the Skyflow vault and retrieve the filtered records + const response: GetResponse = await skyflowClient + .vault(primaryVaultConfig.vaultId) + .get(getRequest, getOptions); + + console.log('Column-based retrieval successful:', response); +} catch(error) { + // Handle any errors that occur during the retrieval process + if (error instanceof SkyflowError) { + console.error('Skyflow Specific Error:', { + code: error.error?.http_code, + message: error.message, + details: error.error?.details, + }); + } else { + console.error('Unexpected Error:', error); + } +} +``` + +#### Get by skyflow IDs +Retrieve specific records using skyflow `ids`. Ideal for fetching exact records when IDs are known. + +```typescript +import { + GetOptions, + GetRequest, + SkyflowError, + GetResponse, + RedactionType +} from 'skyflow-node'; + +/* +This example demonstrates how to retrieve data from the Skyflow vault using a list of Skyflow IDs. + +1. Initializes the Skyflow client with a given vault ID. +2. Creates a request to retrieve records based on Skyflow IDs. +3. Specifies that the response should not return tokens. +4. Uses plain text redaction type for the retrieved records. +5. Prints the response to display the retrieved records. +*/ + +try { + // Initialize Skyflow client + // Step 1: Initialize a list of Skyflow IDs to retrieve records (replace with actual Skyflow IDs) + const getIds: Array = ['a581d205-1969-4350-acbe-a2a13eb871a6', '5ff887c3-b334-4294-9acc-70e78ae5164a']; + + // Step 2: Create a GetRequest to retrieve records by Skyflow ID + const getRequest: GetRequest = new GetRequest( + 'table1', // Replace with your actual table name + getIds + ); + + // Step 3: Configure Get Options and specify not to return tokens and redaction type + const getOptions: GetOptions = new GetOptions(); + getOptions.setReturnTokens(false); // Optional: Set to false to avoid returning tokens + getOptions.setRedactionType(RedactionType.PLAIN_TEXT); + + // Step 4: Send the request to the Skyflow vault and retrieve the records + const getResponse: GetResponse = await skyflowClient + .vault(primaryVaultConfif.vaultId) + .get(getRequest, getOptions); + // Replace with your actual Skyflow vault ID + + console.log('Data retrieval successful:', getResponse); +} catch(error) { + // Step 5: Handle any errors that occur during the retrieval process + if (error instanceof SkyflowError) { + console.error('Skyflow Specific Error:', { + code: error.error?.http_code, + message: error.message, + details: error.error?.details, + }); + } else { + console.error('Unexpected Error:', error); + } +} +``` + +Sample response: + +```typescript +GetResponse { + data: [ + { + card_number: '4555555555555553', + email: 'john.doe@gmail.com', + name: 'john doe', + skyflow_id: 'a581d205-1969-4350-acbe-a2a13eb871a6' + }, + { + card_number: '4555555555555559', + email: 'jane.doe@gmail.com', + name: 'jane doe', + skyflow_id: '5ff887c3-b334-4294-9acc-70e78ae5164a' + } + ], + errors: null +} +``` + +#### Get tokens +Return tokens for records. Ideal for securely processing sensitive data while maintaining data privacy. + +#### An [example](https://github.com/skyflowapi/skyflow-node/blob/v2/samples/vault-api/get-records.ts) of get call to retrieve tokens using Skyflow IDs: + +```typescript +import { + GetOptions, + GetRequest, + SkyflowError, + GetResponse, + RedactionType +} from 'skyflow-node'; + +/* +This example demonstrates how to retrieve data from the Skyflow vault and return tokens along with the records. + +1. Initializes the Skyflow client with a given vault ID. +2. Creates a request to retrieve records based on Skyflow IDs and ensures tokens are returned. +3. Prints the response to display the retrieved records along with the tokens. +*/ + +try { + // Initialize Skyflow client + // Step 1: Initialize a list of Skyflow IDs to retrieve records (replace with actual Skyflow IDs) + const getIds: Array = ['a581d205-1969-4350-acbe-a2a13eb871a6', '5ff887c3-b334-4294-9acc-70e78ae5164a']; + + // Step 2: Create a GetRequest to retrieve records by Skyflow ID + const getRequest: GetRequest = new GetRequest( + 'table1', // Replace with your actual table name + getIds + ); + + // Step 3: Configure Get Options and specify not to return tokens and redaction type + const getOptions: GetOptions = new GetOptions(); + getOptions.setReturnTokens(false); // Optional: Set to true to get tokens + + // Step 4: Send the request to the Skyflow vault and retrieve the records + const getResponse: GetResponse = await skyflowClient + .vault(primaryVaultConfif.vaultId) + .get(getRequest, getOptions); + // Replace with your actual Skyflow vault ID + + console.log('Data retrieval successful:', getResponse); +} catch(error) { + // Step 5: Handle any errors that occur during the retrieval process + if (error instanceof SkyflowError) { + console.error('Skyflow Specific Error:', { + code: error.error?.http_code, + message: error.message, + details: error.error?.details, + }); + } else { + console.error('Unexpected Error:', error); + } +} +``` + +Sample response: + +```typescript +GetResponse { + data: [ + { + card_number: '3998-2139-0328-0697', + email: 'c9a6c9555060@82c092e7.bd52', + name: '82c092e7-74c0-4e60-bd52-c9a6c9555060', + skyflow_id: 'a581d205-1969-4350-acbe-a2a13eb871a6' + }, + { + card_number: '3562-0140-8820-7499', + email: '6174366e2bc6@59f82e89.93fc', + name: '59f82e89-138e-4f9b-93fc-6174366e2bc6', + skyflow_id: '5ff887c3-b334-4294-9acc-70e78ae5164a' + } + ], + errors: null +} +``` + +#### Get by column name and column values +Retrieve records by unique column values. Ideal for querying data without knowing Skyflow IDs, using alternate unique identifiers. + +#### An [example](https://github.com/skyflowapi/skyflow-node/blob/v2/samples/vault-api/get-column-values.ts) of get call to retrieve data using column name and column values: + +```typescript +import { + GetOptions, + GetRequest, + SkyflowError, + GetResponse, + RedactionType, + GetColumnRequest +} from 'skyflow-node'; + +/* +This example demonstrates how to retrieve data from the Skyflow vault based on column values. + +1. Initializes the Skyflow client with a given vault ID. +2. Creates a request to retrieve records based on specific column values (e.g., email addresses). +3. Prints the response to display the retrieved records after redacting sensitive data based on the specified redaction type. +*/ + +try { + // Initialize Skyflow client + // Step 1: Initialize a list of column values (email addresses in this case) + const columnValues: Array = [ + 'john.doe@gmail.com', // Example email address + 'jane.doe@gmail.com' , // Example email address + ]; + const tableName: string = 'table1'; // Replace with your actual table name + const columnName: string = 'email'; // Column name configured as unique in the schema + + // Step 2: Create a GetRequest to retrieve records based on column values + const getRequest: GetColumnRequest = new GetColumnRequest( + tableName, + columnName, + columnValues // Column values of the records to return + ); + + // Step 3: Configure Get Options and specify redaction type + const getOptions: GetOptions = new GetOptions(); + getOptions.setRedactionType(RedactionType.PLAIN_TEXT); // Optional: Set the redaction type (e.g., PLAIN_TEXT) + + // Step 4: Send the request to the Skyflow vault and retrieve the filtered records + const response: GetResponse = await skyflowClient + .vault(primaryVaultConfig.vaultId) + .get(getRequest, getOptions); + + console.log('Column-based retrieval successful:', response); +} catch(error) { + // Step 5: Handle any errors that occur during the retrieval process + if (error instanceof SkyflowError) { + console.error('Skyflow Specific Error:', { + code: error.error?.http_code, + message: error.message, + details: error.error?.details, + }); + } else { + console.error('Unexpected Error:', error); + } +} +``` + +Sample response: + +```typescript +GetResponse { + data: [ + { + card_number: '4555555555555553', + email: 'john.doe@gmail.com', + name: 'john doe', + skyflow_id: 'a581d205-1969-4350-acbe-a2a13eb871a6' + }, + { + card_number: '4555555555555559', + email: 'jane.doe@gmail.com', + name: 'jane doe', + skyflow_id: '5ff887c3-b334-4294-9acc-70e78ae5164a' + } + ], + errors: null +} +``` + +#### Redaction Types +Redaction types determine how sensitive data is displayed when retrieved from the vault. + +**Available Redaction Types** + +- `DEFAULT`: Applies the vault-configured default redaction setting. +- `REDACTED`: Completely removes sensitive data from view. +- `MASKED`: Partially obscures sensitive information. +- `PLAIN_TEXT`: Displays the full, unmasked data. + +**Choosing the Right Redaction Type** +- Use `REDACTED` for scenarios requiring maximum data protection to prevent exposure of sensitive information. +- Use `MASKED` to provide partial visibility of sensitive data for less critical use cases. +- Use `PLAIN_TEXT` for internal, authorized access where full data visibility is necessary. + +### Update + +To update data in your vault, use the `update` method. The `UpdateRequest` class is used to create an update request, where you specify parameters such as the table name, data (as a dictionary). The `UpdateOptions` class is used to configure update options to returnTokens, tokens, and tokenMode. If `returnTokens` is set to True, Skyflow returns tokens for the updated records. If `returnTokens` is set to False, Skyflow returns IDs for the updated records. + +#### Construct an update request + +```typescript +import { + UpdateRequest, + UpdateOptions, + UpdateResponse, + SkyflowError, + TokenMode +} from 'skyflow-node'; + +// This example demonstrates how to update records in the Skyflow vault by providing new data and/or tokenized values, along with the corresponding UpdateRequest schema. + +try { + // Initialize Skyflow client + // Step 1: Prepare the data to update in the vault + // Use a dictionary to store the data that will be updated in the specified table + const updateData: Record = { + skyflowId: 'your-skyflow-id', // Skyflow ID for identifying the record to update + COLUMN_NAME1: '' //Example of a column name and its value to update + COLUMN_NAME2: ''// Another example of a column name and its value to update + }; + + // Step 2: Prepare the tokens (if necessary) for certain columns that require tokenization + const tokens: Record = { + COLUMN_NAME_2: '' // Example of a column name that should be tokenized + } + + // Step 3: Create an UpdateRequest to specify the update operation + const updateReq: UpdateRequest = new UpdateRequest( + 'sensitive_data_table', // Replace with your actual table name + updateData + ); + + // Step 4: Configure Update Options + const updateOptions: UpdateOptions = new UpdateOptions(); + updateOptions.setReturnTokens(true); // Specify whether to return tokens in the response + updatedOptions.setTokens(tokens); // The tokens associated with specific columns + updateOptions.setTokenMode(TokenMode.ENABLE); // Specifies the tokenization mode (ENABLE means tokenization is applied) + + // Step 5: Send the request to the Skyflow vault and update the record + const response: UpdateResponse = await skyflowClient + .vault(primaryVaultConfig.vaultId) + .update(updateReq, updateOptions); + + // Step 6: Print the response to confirm the update result + console.log('Update successful:', response); +} catch(error) { + // Step 7: Handle any errors that occur during the update operation + if (error instanceof SkyflowError) { + console.error('Skyflow Specific Error:', { + code: error.error?.http_code, + message: error.message, + details: error.error?.details + }); + } else { + console.error('Unexpected Error:', error); + } +} +``` + +#### An [example](https://github.com/skyflowapi/skyflow-node/blob/v2/samples/vault-api/update-record.ts) of update call + +```typescript +import { + UpdateRequest, + UpdateOptions, + UpdateResponse, + SkyflowError, + TokenMode +} from 'skyflow-node'; + +/* +This example demonstrates how to update a record in the Skyflow vault with specified data and tokens. + +1. Initializes the Skyflow client with a given vault ID. +2. Constructs an update request with data to modify and tokens to include. +3. Sends the request to update the record in the vault. +4. Prints the response to confirm the success or failure of the update operation. +*/ + +try { + // Initialize Skyflow client + // Step 1: Prepare the data to update in the vault + // Use a dictionary to store the data that will be updated in the specified table + const updateData: Record = { + skyflowId: '5b699e2c-4301-4f9f-bcff-0a8fd3057413', // Skyflow ID for identifying the record to update + name: 'john doe' //Example of a column name and its value to update + card_number: '4111111111111115'// Another example of a column name and its value to update + }; + + // Step 2: Prepare the tokens (if necessary) for certain columns that require tokenization + const tokens: Record = { + name: '72b8ffe3-c8d3-4b4f-8052-38b2a7405b5a' // Example of a column name that should be tokenized + } + + // Step 3: Create an UpdateRequest to specify the update operation + const updateReq: UpdateRequest = new UpdateRequest( + 'table1', // Replace with your actual table name + updateData + ); + + // Step 4: Configure Update Options + const updateOptions: UpdateOptions = new UpdateOptions(); + updatedOptions.setTokens(tokens); // The tokens associated with specific columns + updateOptions.setTokenMode(TokenMode.ENABLE); // Specifies the tokenization mode (ENABLE means tokenization is applied) + + // Step 5: Send the request to the Skyflow vault and update the record + const response: UpdateResponse = await skyflowClient + .vault(primaryVaultConfig.vaultId) + .update(updateReq, updateOptions); + + // Step 6: Print the response to confirm the update result + console.log('Update successful:', response); +} catch(error) { + // Step 7: Handle any errors that occur during the update operation + if (error instanceof SkyflowError) { + console.error('Skyflow Specific Error:', { + code: error.error?.http_code, + message: error.message, + details: error.error?.details + }); + } else { + console.error('Unexpected Error:', error); + } +} +``` + +Sample response: + +- When `returnTokens` is set to `True` +```typescript +UpdateResponse { + updatedField: { + skyflowId: '5b699e2c-4301-4f9f-bcff-0a8fd3057413', + name: '72b8ffe3-c8d3-4b4f-8052-38b2a7405b5a', + card_number: '4131-1751-0217-8491' + }, + errors: null +} +``` + +- When `returnTokens` is set to `False` +```typescript +UpdateResponse { + updatedField: { + skyflowId: '5b699e2c-4301-4f9f-bcff-0a8fd3057413', + }, + errors: null +} +``` + +### Delete + +To delete records using Skyflow IDs, use the `delete` method. The `DeleteRequest` class accepts a list of Skyflow IDs that you want to delete, as shown below: + +#### Construct a delete request + +```typescript +import { + DeleteRequest, + DeleteResponse, + SkyflowError +} from 'skyflow-node'; + +/* +This example demonstrates how to delete records from a Skyflow vault using specified Skyflow IDs, along with corresponding DeleteRequest schema. +*/ + +try { + // Initialize Skyflow client + // Step 1: Prepare a list of Skyflow IDs for the records to delete + // The list stores the Skyflow IDs of the records that need to be deleted from the vault + const deleteIds: Array = ['', '', '']; // Replace with actual Skyflow IDs + const tableName: string = ''; // Replace with the actual table name from which to delete + + // Step 2: Create a DeleteRequest to define the delete operation + const deleteRequest: DeleteRequest = new DeleteRequest( + tableName, + deleteIds + ); + + // Step 3: Send the delete request to the Skyflow vault + const response: DeleteResponse = await skyflowClient + .vault(primaryVaultConfig.vaultId) + .delete(deleteRequest); + + // Print the response to confirm the delete result + console.log('Deletion successful:', response); +} catch(error) { + // Step 4: Handle any exceptions that occur during the delete operation + if (error instanceof SkyflowError) { + console.error('Skyflow Specific Error:', { + code: error.error?.http_code, + message: error.message, + details: error.error?.details, + }); + } else { + console.error('Unexpected Error:', error); + } +} +``` +#### An [example](https://github.com/skyflowapi/skyflow-node/blob/v2/samples/vault-api/delete-records.ts) of delete call + +```typescript +import { + DeleteRequest, + DeleteResponse, + SkyflowError +} from 'skyflow-node'; + +/* +This example demonstrates how to delete records from a Skyflow vault using specified Skyflow IDs. + +1. Initializes the Skyflow client with a given Vault ID. +2. Constructs a delete request by specifying the IDs of the records to delete. +3. Sends the delete request to the Skyflow vault to delete the specified records. +4. Prints the response to confirm the success or failure of the delete operation. +*/ + +try { + // Initialize Skyflow client + // Step 1: Prepare a list of Skyflow IDs for the records to delete + // The list stores the Skyflow IDs of the records that need to be deleted from the vault + const deleteIds: Array = ['9cbf66df-6357-48f3-b77b-0f1acbb69280', 'ea74bef4-f27e-46fe-b6a0-a28e91b4477b', '47700796-6d3b-4b54-9153-3973e281cafb']; // Replace with actual Skyflow IDs + const tableName: string = 'table1'; // Replace with the actual table name from which to delete + + // Step 2: Create a DeleteRequest to define the delete operation + const deleteRequest: DeleteRequest = new DeleteRequest( + tableName, + deleteIds + ); + + // Step 3: Send the delete request to the Skyflow vault + const response: DeleteResponse = await skyflowClient + .vault(primaryVaultConfig.vaultId) + .delete(deleteRequest); + + // Print the response to confirm the delete result + console.log('Deletion successful:', response); +} catch(error) { + // Step 4: Handle any exceptions that occur during the delete operation + if (error instanceof SkyflowError) { + console.error('Skyflow Specific Error:', { + code: error.error?.http_code, + message: error.message, + details: error.error?.details, + }); + } else { + console.error('Unexpected Error:', error); + } +} +``` + +Sample response: + +```typescript +DeleteResponse { + deletedIds: [ + '9cbf66df-6357-48f3-b77b-0f1acbb69280', + 'ea74bef4-f27e-46fe-b6a0-a28e91b4477b', + '47700796-6d3b-4b54-9153-3973e281cafb' + ], + errors: null +} +``` + +### Query + +To retrieve data with SQL queries, use the `query` method. `QueryRequest` is class that takes the `query` parameter as follows: + +#### Construct a query request +Refer to [Query your data](https://docs.skyflow.com/query-data/) and [Execute Query](https://docs.skyflow.com/record/#QueryService_ExecuteQuery) for guidelines and restrictions on supported SQL statements, operators, and keywords. + +```typescript +import { + QueryRequest, + QueryResponse, + SkyflowError, +} from 'skyflow-node'; + +/* +This example demonstrates how to execute a custom SQL query on a Skyflow vault, along with QueryRequest schema. +*/ + +try { + // Initialize Skyflow client + // Step 1: Define the SQL query to execute on the Skyflow vault + // Replace "" with the actual SQL query you want to run + const query: string = ""; // Example: "SELECT * FROM table1 WHERE column1 = 'value'" + + // Step 2: Create a QueryRequest with the specified SQL query + const queryRequest: QueryRequest = new QueryRequest(query); + + // Step 3: Execute the query request on the specified Skyflow vault + const response: QueryResponse = await skyflowClient + .vault('') // Replace with your actual Vault ID + .query(queryRequest); + + // Step 4: Print the response containing the query results + console.log('Query Result:', response); +} catch(error) { + // Step 5: Handle any exceptions that occur during the query execution + if (error instanceof SkyflowError) { + console.error('Skyflow Specific Error:', { + code: error.error?.http_code, + message: error.message, + details: error.error?.details, + }); + } else { + console.error('Unexpected Error:', error); + } +} +``` + +#### An [example](https://github.com/skyflowapi/skyflow-node/blob/v2/samples/vault-api/query-records.ts) of query call + +```typescript +import { + QueryRequest, + QueryResponse, + SkyflowError, +} from 'skyflow-node'; + +/* +This example demonstrates how to execute a SQL query on a Skyflow vault to retrieve data. + +1. Initializes the Skyflow client with the Vault ID. +2. Constructs a query request with a specified SQL query. +3. Executes the query against the Skyflow vault. +4. Prints the response from the query execution. +*/ + +try { + // Initialize Skyflow client + // Step 1: Define the SQL query to execute on the Skyflow vault + // Example query: Retrieve all records from the "cards" table with a specific skyflow_id + const query: string = "SELECT * FROM cards WHERE skyflow_id='3ea3861-x107-40w8-la98-106sp08ea83f'"; + + // Step 2: Create a QueryRequest with the specified SQL query + const queryRequest: QueryRequest = new QueryRequest(query); + + // Step 3: Execute the query request on the specified Skyflow vault + const response: QueryResponse = await skyflowClient + .vault(primaryVaultConfig.vaultId) // Replace with actual Vault ID + .query(queryRequest); + + // Step 4: Print the response containing the query results + console.log('Query Result:', response); +} catch(error) { + // Step 5: Handle any exceptions that occur during the query execution + if (error instanceof SkyflowError) { + console.error('Skyflow Specific Error:', { + code: error.error?.http_code, + message: error.message, + details: error.error?.details, + }); + } else { + console.error('Unexpected Error:', error); + } +} +``` + +Sample Response: + +```typescript +QueryResponse { + fields: [ + { + card_number: 'XXXXXXXXXXXX1112', + name: 'S***ar', + skyflow_id: '3ea3861-x107-40w8-la98-106sp08ea83f', + tokenizedData: {} + } + ], + errors: null, +} +``` + +## Detect +Skyflow Detect enables you to deidentify and reidentify sensitive data in text and files, supporting advanced privacy-preserving workflows. The Detect API supports the following operations: + +### Deidentify Text +To deidentify text, use the `deidentifyText` method. The `DeidentifyTextRequest` class creates a deidentify text request, which includes the text to be deidentified. Additionally, you can provide optional parameters using the `DeidentifyTextOptions` class. + +```typescript +import { + DeidentifyTextRequest, + DeidentifyTextOptions, + SkyflowError, + TokenFormat, + TokenType, + Transformations, + DetectEntities +} from 'skyflow-node'; + +try { + // Step 1: Prepare the text to be deidentified + const deidentifyTextRequest = new DeidentifyTextRequest( + '' + ); + + // Step 2: Configure DeidentifyTextOptions + const options = new DeidentifyTextOptions(); + options.setEntities([DetectEntities.ACCOUNT_NUMBER, DetectEntities.SSN]); // Entities to deidentify + options.setAllowRegexList(['']); // Allowlist regex patterns + options.setRestrictRegexList(['']); // Restrict regex patterns + + const tokenFormat = new TokenFormat(); // Specify the token format for deidentified entities + tokenFormat.setDefault(TokenType.VAULT_TOKEN); + optionsText.setTokenFormat(tokenFormat); + + const transformations = new Transformations(); // Specify custom transformations for entities + transformations.setShiftDays({ + max: 30, // Maximum shift days + min: 30, // Minimum shift days + entities: [DetectEntities.ACCOUNT_NUMBER, DetectEntities.SSN], // Entities to apply the shift + }); + optionsText.setTransformations(transformations); + + // Step 3: Call deidentifyText + const response = await skyflowClient + .detect(primaryVaultConfig.vaultId) + .deidentifyText(deidentifyTextRequest, options); + + console.log('Deidentify Text Response:', response); + +} catch (error) { + if (error instanceof SkyflowError) { + console.error('Skyflow Error:', error.message); + } else { + console.error('Unexpected Error:', JSON.stringify(error)); + } +} +``` + +#### An example of a deidentify text call + +```typescript +import { + SkyflowError, + DeidentifyTextRequest, + DeidentifyTextOptions, + TokenFormat, + TokenType, + Transformations, + DetectEntities, + DeidentifyTextResponse +} from 'skyflow-node'; + +/** + * Skyflow Deidentify Text Example + * + * This example demonstrates how to: + * 1. Configure credentials + * 2. Set up vault configuration + * 3. Create a deidentify text request + * 4. Use all available options for deidentification + * 5. Handle response and errors + */ + +async function performDeidentifyText() { + try { + + // Step 1: Prepare Deidentify Text Request + const textReq = new DeidentifyTextRequest( + 'My SSN is 123-45-6789 and my card is 4111 1111 1111 1111.', // Text to be deidentified + ); + + // Step 2: Configure DeidentifyTextOptions + const optionsText = new DeidentifyTextOptions(); + + // setEntities: Specify which entities to deidentify + optionsText.setEntities([DetectEntities.SSN, DetectEntities.CREDIT_CARD_NUMBER]); + + // setTokenFormat: Specify the token format for deidentified entities + const tokenFormat = new TokenFormat(); + tokenFormat.setDefault(TokenType.VAULT_TOKEN); + optionsText.setTokenFormat(tokenFormat); + + // setTransformations: Specify custom transformations for entities + const transformations = new Transformations(); + transformations.setShiftDays({ + max: 30, // Maximum shift days + min: 30, // Minimum shift days + entities: [DetectEntities.DOB], // Entities to apply the shift + }); + optionsText.setTransformations(transformations); + + // Step 3: Call deidentifyText API + const response: DeidentifyTextResponse = await skyflowClient + .detect(primaryVaultConfig.vaultId) + .deidentifyText(textReq, optionsText); + + // Handle Successful Response + console.log('Deidentify Text Response:', response); + + } catch (error) { + // Comprehensive Error Handling + if (error instanceof SkyflowError) { + console.error('Skyflow Specific Error:', { + code: error.error?.http_code, + message: error.message, + details: error.error?.details, + }); + } else { + console.error('Unexpected Error:', JSON.stringify(error)); + } + } +} + +// Invoke the deidentify text function +performDeidentifyText(); + +``` +Sample Response: + +```typescript +{ + "processedText": "My SSN is [SSN_0ykQWPA] and my card is [CREDIT_CARD_N92QAVa].", + "entities": [ + { + "token": "SSN_0ykQWPA", + "value": "123-45-6789", + "textIndex": { + "start": 10, + "end": 21 + }, + "processedIndex": { + "start": 10, + "end": 23 + }, + "entity": "SSN", + "scores": { + "SSN": 0.9383999705314636 + } + }, + { + "token": "CREDIT_CARD_N92QAVa", + "value": "4111 1111 1111 1111", + "textIndex": { + "start": 37, + "end": 56 + }, + "processedIndex": { + "start": 39, + "end": 60 + }, + "entity": "CREDIT_CARD", + "scores": { + "CREDIT_CARD": 0.9050999879837 + } + } + ], + "wordCount": 9, + "charCount": 57 +} +``` + +### Reidentify Text + +To reidentify text, use the `reidentifyText` method. The `ReidentifyTextRequest` class creates a reidentify text request, which includes the redacted or deidentified text to be reidentified. Additionally, you can provide optional parameters using the `ReidentifyTextOptions` class to control how specific entities are returned (as redacted, masked, or plain text). + + +```typescript +import { + ReidentifyTextRequest, + ReidentifyTextOptions, + SkyflowError, + DetectEntities, + ReidentifyTextResponse +} from 'skyflow-node'; + +try { + // Step 1: Prepare the redacted text to be reidentified + const textReq = new ReidentifyTextRequest( + '' + ); + + // Step 2: Configure ReidentifyTextOptions + const options = new ReidentifyTextOptions(); + options.setRedactedEntities([DetectEntities.SSN]); // Entities to keep redacted + options.setMaskedEntities([DetectEntities.CREDIT_CARD_NUMBER]); // Entities to mask + options.setPlainTextEntities([DetectEntities.NAME]); // Entities to return as plain text + + // Step 3: Call reidentifyText + const response: ReidentifyTextResponse = await skyflowClient + .detect(primaryVaultConfig.vaultId) + .reidentifyText(textReq, options); + + console.log('Reidentify Text Response:', response); + +} catch (error) { + // Comprehensive Error Handling + if (error instanceof SkyflowError) { + console.error('Skyflow Specific Error:', { + code: error.error?.http_code, + message: error.message, + details: error.error?.details, + }); + } else { + console.error('Unexpected Error:', JSON.stringify(error)); + } +} +``` + +#### An example of a reidentify text call + +```typescript +import { + ReidentifyTextRequest, + ReidentifyTextOptions, + DetectEntities, + ReidentifyTextResponse +} from 'skyflow-node'; + +/** + * Skyflow Reidentify Text Example + * + * This example demonstrates how to: + * 1. Configure credentials + * 2. Set up vault configuration + * 3. Create a reidentify text request + * 4. Use all available options for reidentification + * 5. Handle response and errors + */ + +async function performReidentifyText() { + try { + // Step 1: Prepare Reidentify Text Request + const reidentifyTextRequest = new ReidentifyTextRequest( + 'My SSN is [SSN_0ykQWPA] and my card is [CREDIT_CARD_N92QAVa].' // The redacted text to reidentify + ); + + // Step 2: Configure ReidentifyTextOptions + const options = new ReidentifyTextOptions(); + + // Specify which entities to reidentify as redacted, masked, or plain text + options.setPlainTextEntities([DetectEntities.CREDIT_CARD, DetectEntities.SSN]); + + // Step 4: Call reidentifyText + const response: ReidentifyTextResponse = await skyflowClient + .detect(primaryVaultConfig.vaultId) + .reidentifyText(reidentifyTextRequest, options); + + // Step 5: Handle response + console.log('Reidentified Text Response:', response); + + } catch (error) { + // Comprehensive Error Handling + if (error instanceof SkyflowError) { + console.error('Skyflow Specific Error:', { + code: error.error?.http_code, + message: error.message, + details: error.error?.details, + }); + } else { + console.error('Unexpected Error:', JSON.stringify(error)); + } + } +} + +// Invoke the reidentify text function +performReidentifyText(); +``` + +Sample Response: + +```typescript +{ + processedText: 'My SSN is 123-45-6789 and my card is 4111 1111 1111 1111.' +} +``` + +### Deidentify File + +To deidentify files, use the `deidentifyFile` method. The `DeidentifyFileRequest` class creates a deidentify file request, which includes the file to be deidentified (such as images, PDFs, audio, documents, spreadsheets, or presentations). Additionally, you can provide optional parameters using the `DeidentifyFileOptions` class to control how entities are detected and deidentified, as well as how the output is generated for different file types. + +**Note:** File deidentification requires Node.js version 20 or above. + +```typescript +import { + DeidentifyFileRequest, + DeidentifyFileOptions, + DeidentifyFileResponse + SkyflowError, + DetectEntities, + MaskingMethod, + DetectOutputTranscription, + Bleep +} from 'skyflow-node'; + +try { + // Step 1: Prepare the file to be deidentified + const filePath: string = ''; + const buffer = fs.readFileSync(filePath); + const file = new File([buffer], filePath); + + //Step 2: Construct the file input by providing either file or filePath but not both + const fileInput: FileInput = { file: file } + // const fileInput: FileInput = { filePath: filePath } + const fileReq = new DeidentifyFileRequest(fileInput); + + // Step 3: Configure DeidentifyFileOptions + const options = new DeidentifyFileOptions(); + options.setEntities([DetectEntities.SSN, DetectEntities.ACCOUNT_NUMBER]); + options.setAllowRegexList(['']); + options.setRestrictRegexList(['']); + + const tokenFormat = new TokenFormat(); // Token format for deidentified entities + tokenFormat.setDefault(TokenType.ENTITY_ONLY); + options.setTokenFormat(tokenFormat); + + const transformations = new Transformations(); // transformations for entities + transformations.setShiftDays({ + max: 30, + min: 10, + entities: [DetectEntities.SSN], + }); + options.setTransformations(transformations); + + options.setOutputDirectory(''); // Output directory for saving the deidentified file. This is not supported in Cloudflare workers + + options.setWaitTime(15); // Wait time for response (max 64 seconds; throws error if more) + + // ===== Image Options (apply when file is an image) ===== + + // options.setOutputProcessedImage(true); // Include processed image in output + + // options.setOutputOcrText(true); // Include OCR text in response + + // options.setMaskingMethod(MaskingMethod.Blackbox); // Masking method for image entities + + // ===== PDF Options (apply when file is a PDF) ===== + + // options.setPixelDensity(300); // Pixel density for PDF processing + + // options.setMaxResolution(2000); // Max resolution for PDF + + // ===== Audio Options (apply when file is audio) ===== + + // options.setOutputProcessedAudio(true); // Include processed audio in output + + // options.setOutputTranscription(DetectOutputTranscription.PLAINTEXT_TRANSCRIPTION); // Type of transcription + + // const bleep = new Bleep(); // Bleep audio configuration + // bleep.setGain(5); // Relative loudness in dB + // bleep.setFrequency(1000); // Pitch in Hz + // bleep.setStartPadding(0.1); // Padding at start in seconds + // bleep.setStopPadding(0.2); // Padding at end in seconds + // options.setBleep(bleep); + + // Step 4: Call deidentifyFile + const response: DeidentifyFileResponse = await skyflowClient + .detect(primaryVaultConfig.vaultId) + .deidentifyFile(fileReq, options); + + console.log('Deidentify File Response:', response); + +} catch (error) { + // Comprehensive Error Handling + if (error instanceof SkyflowError) { + console.error('Skyflow Specific Error:', { + code: error.error?.http_code, + message: error.message, + details: error.error?.details, + }); + } else { + console.error('Unexpected Error:', JSON.stringify(error)); + } +} +``` + +#### An example of a deidentify file + +```typescript +import { + SkyflowError, + DeidentifyFileRequest, + DeidentifyFileOptions, + DetectEntities, + TokenFormat, + TokenType, + Transformations, + DeidentifyFileResponse +} from 'skyflow-node'; +import fs from 'fs'; + +/** + * Skyflow Deidentify File Example + * + * This sample demonstrates how to use all available options for deidentifying files. + * Supported file types: images (jpg, png, etc.), pdf, audio (mp3, wav), documents, spreadsheets, presentations, structured text. + */ + +async function performDeidentifyFile() { + try { + // Step 1: Prepare Deidentify File Request + // Replace with your file object (e.g., from fs.readFileSync or browser File API) + const filePath: string = '/detect/sample.txt'; + const buffer = fs.readFileSync(filePath); + const file = new File([buffer], filePath); + + + //Step 2: Construct the file input by providing either file or filePath but not both + const fileInput: FileInput = { file: file } + // const fileInput: FileInput = { filePath: filePath } + + const fileReq = new DeidentifyFileRequest(fileInput); + + // Step 3: Configure DeidentifyFileOptions + const options = new DeidentifyFileOptions(); + + // Entities to detect and deidentify + options.setEntities([DetectEntities.SSN, DetectEntities.ACCOUNT_NUMBER]); + + // Token format for deidentified entities + const tokenFormat = new TokenFormat(); + tokenFormat.setDefault(TokenType.ENTITY_ONLY); + options.setTokenFormat(tokenFormat); + + // Custom transformations for entities + const transformations = new Transformations(); + transformations.setShiftDays({ + max: 30, + min: 10, + entities: [DetectEntities.SSN], + }); + options.setTransformations(transformations); + + // Output directory for saving the deidentified file + options.setOutputDirectory('/home/user/output'); // Replace with your desired output directory. This is not supported in Cloudflare workers + + // Wait time for response (max 64 seconds) + options.setWaitTime(15); + + + // Step 4: Call deidentifyFile API + const response: DeidentifyFileResponse = await skyflowClient + .detect(primaryVaultConfig.vaultId) + .deidentifyFile(fileReq, options); + + // Handle Successful Response + console.log('Deidentify File Response:', response); + + } catch (error) { + // Comprehensive Error Handling + if (error instanceof SkyflowError) { + console.error('Skyflow Specific Error:', { + code: error.error?.http_code, + message: error.message, + details: error.error?.details, + }); + } else { + console.error('Unexpected Error:', JSON.stringify(error)); + } + } +} + +// Invoke the deidentify file function +performDeidentifyFile(); + +``` + +Sample Response: + +```typescript +{ + entities: [ + { + file: '0X2xhYmVsIjoiQ1JFRElUX0NB==', + extension: 'json' + } + ], + fileBase64: 'TXkgU1NOIGlzIFtTU0==', + file: File { + size: 15075, + type: '', + name: 'deidentified.jpeg', + lastModified: 1750791985426 + }, + type: 'redacted_file', + extension: 'txt', + wordCount: 12, + charCount: 58, + sizeInKb: 0.06, + durationInSeconds: 0, + pageCount: 0, + slideCount: 0, + runId: undefined, + status: 'SUCCESS' +} +``` + +**Supported file types:** +- Documents: `doc`, `docx`, `pdf` +- PDFs: `pdf` +- Images: `bmp`, `jpeg`, `jpg`, `png`, `tif`, `tiff` +- Structured text: `json`, `xml` +- Spreadsheets: `csv`, `xls`, `xlsx` +- Presentations: `ppt`, `pptx` +- Audio: `mp3`, `wav` + +**Note:** +- Transformations cannot be applied to Documents, Images, or PDFs file formats. + +- The `waitTime` option must be ≤ 64 seconds; otherwise, an error is thrown. + +- If the API takes more than 64 seconds to process the file, it will return only the run ID in the response. + +Sample response (when the API takes more than 64 seconds): +```typescript + +{ + entities: undefined, + file: undefined, + type: undefined, + extension: undefined, + wordCount: undefined, + charCount: undefined, + sizeInKb: undefined, + durationInSeconds: undefined, + pageCount: undefined, + slideCount: undefined, + runId: '1ad6dc12-8405-46cf-1c13-db1123f9f4c5', + status: 'IN_PROGRESS' +} +``` + +### Get run +To retrieve the results of a previously started file deidentification operation, use the `getDetectRun` method. +The `GetDetectRunRequest` class is initialized with the `runId` returned from a prior `deidentifyFile` call. +This method allows you to fetch the final results of the file processing operation once they are available. + + +```typescript +import { + GetDetectRunRequest, + DeidentifyFileResponse, + DeidentifyFileResponse + SkyflowError +} from 'skyflow-node'; + +try { + // Step 1: Prepare the GetDetectRunRequest with the runId from a previous deidentifyFile call + const getDetectRunRequest = new GetDetectRunRequest({ + runId: '', // Replace with the runId you received earlier + }); + + // Step 2: Call getDetectRun + const response: DeidentifyFileResponse = await skyflowClient + .detect(primaryVaultConfig.vaultId) + .getDetectRun(getDetectRunRequest); + + // Step 3: Handle the response + console.log('Get Detect Run Response:', response); + +} catch (error) { + if (error instanceof SkyflowError) { + console.error('Skyflow Error:', error.message); + } else { + console.error('Unexpected Error:', error); + } +} +``` + +#### An example of a get run function + +```typescript +import { + Credentials, + Env, + LogLevel, + Skyflow, + SkyflowConfig, + VaultConfig, + SkyflowError, + GetDetectRunRequest, + DeidentifyFileResponse +} from 'skyflow-node'; + +/** + * Skyflow Get Detect Run Example + * + * This example demonstrates how to: + * 1. Configure credentials + * 2. Set up vault configuration + * 3. Create a get detect run request + * 4. Call getDetectRun to poll for file processing results + * 5. Handle response and errors + */ + +async function performGetDetectRun() { + try { + // Step 1: Configure Credentials + const credentials: Credentials = { + token: '', // Replace with your BEARER token + }; + + // Step 2: Configure Vault + const primaryVaultConfig: VaultConfig = { + vaultId: '', // Unique vault identifier + clusterId: '', // From vault URL + env: Env.PROD, // Deployment environment + credentials: credentials // Authentication method + }; + + // Step 3: Configure Skyflow Client + const skyflowConfig: SkyflowConfig = { + vaultConfigs: [primaryVaultConfig], + logLevel: LogLevel.INFO, // Recommended to use LogLevel.ERROR in production environment. + }; + + // Initialize Skyflow Client + const skyflowClient: Skyflow = new Skyflow(skyflowConfig); + + // Step 4: Prepare GetDetectRunRequest + const getDetectRunRequest = new GetDetectRunRequest({ + runId: '', // Replace with the runId from a previous deidentifyFile call + }); + + // Step 5: Call getDetectRun API + const response: DeidentifyFileResponse = await skyflowClient + .detect(primaryVaultConfig.vaultId) + .getDetectRun(getDetectRunRequest); + + // Handle Successful Response + console.log('Get Detect Run Response:', response); + + } catch (error) { + // Comprehensive Error Handling + if (error instanceof SkyflowError) { + console.error('Skyflow Specific Error:', { + code: error.error?.http_code, + message: error.message, + details: error.error?.details, + }); + } else { + console.error('Unexpected Error:', error); + } + } +} + +// Invoke the get detect run function +performGetDetectRun(); +``` + +Sample Response + +```typescript +{ + entities: [ + { + file: '0X2xhYmVsIjoiQ1JFRElUX0NB==', + extension: 'json' + } + ], + file: 'TXkgU1NOIGlzIFtTU0==', + type: 'redacted_file', + extension: 'txt', + wordCount: 12, + charCount: 58, + sizeInKb: 0.06, + durationInSeconds: 0, + pageCount: 0, + slideCount: 0, + status: 'SUCCESS' +} +``` + +## Connections + +Skyflow Connections is a gateway service that uses tokenization to securely send and receive data between your systems and first- or third-party services. The [connections](https://github.com/skyflowapi/skyflow-node/tree/v2/src/vault/controller/connections) module invokes both inbound and/or outbound connections. +- **Inbound connections**: Act as intermediaries between your client and server, tokenizing sensitive data before it reaches your backend, ensuring downstream services handle only tokenized data. +- **Outbound connections**: Enable secure extraction of data from the vault and transfer it to third-party services via your backend server, such as processing checkout or card issuance flows. + +#### Invoke a connection +To invoke a connection, use the `invoke` method of the Skyflow client. +#### Construct an invoke connection request + +```typescript +import { + InvokeConnectionRequest, + RequestMethod, + ConnectionConfig, + SkyflowError, + InvokeConnectionResponse +} from 'skyflow-node'; + +/* +This example demonstrates how to invoke an external connection using the Skyflow SDK, along with corresponding InvokeConnectionRequest schema. +*/ + +try { + // Initialize Skyflow client + // Step 1: Define the request body parameters + // These are the values you want to send in the request body + const requestBody = { + COLUMN_NAME_1: '', // Replace with actual key-value pairs + COLUMN_NAME_2: '', + }; + + // Step 2: Define the request headers + // Add any required headers that need to be sent with the request + const requestHeaders = { + HEADER_NAME_1: '', + HEADER_NAME_2: '', + }; + + // Step 3: Define the path parameters + // Path parameters are part of the URL and typically used in RESTful APIs + const pathParams = { + YOUR_PATH_PARAM_KEY_1: '', + YOUR_PATH_PARAM_KEY_2: '' + } + + // Step 4: Define the query parameters + // Query parameters are included in the URL after a '?' and are used to filter or modify the response + const queryParams = { + YOUR_QUERY_PARAM_KEY_1: '', + YOUR_QUERY_PARAM_KEY_2: '', + } + + // Step 5: Define the request method + const requestMethod: RequestMethod = RequestMethod.POST; + + // Step 6: Build the InvokeConnectionRequest using the provided parameters + const invokeReq: InvokeConnectionRequest = new InvokeConnectionRequest( + requestMethod, + requestBody, + requestHeaders, + pathParams, + queryParams + ); + + // Step 7: Invoke the connection using the request + const response: InvokeConnectionResponse = await skyflowClient + .connection() + .invoke(invokeReq); + + // Step 8: Print the response from the invoked connection + console.log('Connection invocation successful:', response); +} catch(error) { + // Step 9: Handle any exceptions that occur during the connection invocation + if (error instanceof SkyflowError) { + console.error('Skyflow Specific Error:', { + code: error.error?.http_code, + message: error.message, + details: error.error?.details + }); + } else { + console.error('Unexpected Error:', error); + } +} +``` + +`method` supports the following methods: + +- GET +- POST +- PUT +- PATCH +- DELETE + +**pathParams, queryParams, header, body** are the JSON objects represented as dictionaries that will be sent through the connection integration url. + +#### An [example](https://github.com/skyflowapi/skyflow-node/blob/v2/samples/vault-api/invoke-connection.ts) of Invoke Connection + +```typescript +import { + InvokeConnectionRequest, + RequestMethod, + ConnectionConfig, + SkyflowError, + InvokeConnectionResponse +} from 'skyflow-node'; + +/* +This example demonstrates how to invoke an external connection using the Skyflow SDK. +It configures a connection, sets up the request, and sends a POST request to the external service. + +1. Initialize Skyflow client with connection details. +2. Define the request body, headers, and method. +3. Execute the connection request. +4. Print the response from the invoked connection. +*/ + +try { + // Initialize Skyflow client + // Step 1: Set up credentials and connection configuration + // Load credentials from a JSON file (you need to provide the correct path) + const credentials: Credentials = { + path: '', + }; + + // Define the connection configuration (URL and credentials) + const connectionConfig: ConnectionConfig = { + connectionId: '', // Replace with actual connection ID + connectionUrl: 'https://connection.url.com', // Replace with actual connection URL + credentials: credentials // Set credentials for the connection + }; + + // Step 2: Configure Skyflow Client + const skyflowConfig: SkyflowConfig = { + connectionConfigs: [connectionConfig], // Add connection configuration to client + logLevel: LogLevel.INFO // Recommended to use LogLevel.ERROR in production environment. + }; + + // Step 3: Initialize Skyflow Client + const skyflowClient: Skyflow = new Skyflow(skyflowConfig); + + // Step 4: Define the request body parameters + const requestBody = { + card_number: '4337-1696-5866-0865', // Example card number + ssn: '524-41-4248', // Example SSN + }; + + // Step 5: Define the request headers + // Add any required headers that need to be sent with the request + const requestHeaders = { + 'content-type': 'application/json', // Set content type for the request + }; + + // Step 6: Define the request method + const requestMethod: RequestMethod = RequestMethod.POST; + + // Step 7: Build the InvokeConnectionRequest with required parameters + const invokeReq: InvokeConnectionRequest = new InvokeConnectionRequest( + requestMethod, + requestBody, + requestHeaders + ); + + // Step 8: Invoke the connection using the request + const response: InvokeConnectionResponse = await skyflowClient + .connection() + .invoke(invokeReq); + + // Step 9: Print the response from the invoked connection + console.log('Connection invocation successful:', response); +} catch(error) { + // Step 9: Handle any exceptions that occur during the connection invocation + if (error instanceof SkyflowError) { + console.error('Skyflow Specific Error:', { + code: error.error?.http_code, + message: error.message, + details: error.error?.details + }); + } else { + console.error('Unexpected Error:', error); + } +} +``` + +Sample response: + +```typescript +InvokeConnectionResponse { + data: { + card_number: '4337-1696-5866-0865', + ssn: '524-41-4248', + }, + metadata: { + requestId: '801279ety19289899' + }, + errors: null +} +``` + +## Authenticate with bearer tokens +This section covers methods for generating and managing tokens to authenticate API calls: + +- **Generate a bearer token:** +Enable the creation of bearer tokens using service account credentials. These tokens, valid for 60 minutes, provide secure access to Vault services and management APIs based on the service account's permissions. Use this for general API calls when you only need basic authentication without additional context or role-based restrictions. +- **Generate a bearer token with context:** +Support embedding context values into bearer tokens, enabling dynamic access control and the ability to track end-user identity. These tokens include context claims and allow flexible authorization for Vault services. Use this when policies depend on specific contextual attributes or when tracking end-user identity is required. +- **Generate a scoped bearer token:** +Facilitate the creation of bearer tokens with role-specific access, ensuring permissions are limited to the operations allowed by the designated role. This is particularly useful for service accounts with multiple roles. Use this to enforce fine-grained role-based access control, ensuring tokens only grant permissions for a specific role. +- **Generate signed data tokens:** +Add an extra layer of security by digitally signing data tokens with the service account's private key. These signed tokens can be securely detokenized, provided the necessary bearer token and permissions are available. Use this to add cryptographic protection to sensitive data, enabling secure detokenization with verified integrity and authenticity. + +#### Generate a bearer token +The [Service Account](https://github.com/skyflowapi/skyflow-node/tree/v2/src/service-account) Node package generates service account tokens using a service account credentials file, which is provided when a service account is created. The tokens generated by this module are valid for 60 minutes and can be used to make API calls to the [Data](https://docs.skyflow.com/record/) and [Management](https://docs.skyflow.com/management/) APIs, depending on the permissions assigned to the service account. + +The `generateBearerToken(filepath)` function takes the credentials file path for token generation, alternatively, you can also send the entire credentials as string, by using `generateBearerTokenFromCreds(credentials)` + +#### [Example](http://github.com/skyflowapi/skyflow-node/blob/v2/samples/service-account/token-generation-example.ts): + +```javascript +import { + generateBearerToken, + generateBearerTokenFromCreds, + isExpired, + SkyflowError +} from 'skyflow-node'; + +/* +Example program to generate a Bearer Token using Skyflow's service account utilities. +The token can be generated in two ways: +1. Using the file path to a credentials.json file. +2. Using the JSON content of the credentials file as a string. +*/ + +// Variable to store the generated token +let bearerToken: string = ''; + +// Specify the full file path to the credentials.json file +const filepath = 'CREDENTIALS_FILE_PATH'; + + +// Example 1: Generate Bearer Token using a credentials.json file +function getSkyflowBearerTokenFromFilePath() { + return new Promise((resolve, reject) => { + try { + // Check if the token is already generated and still valid + if (!isExpired(bearerToken)) resolve(bearerToken); + else { + // Generate a new Bearer Token from the credentials file + generateBearerToken(filepath) + .then(response => { + bearerToken = response.accessToken; + // Resolve the generated Bearer Token + resolve(bearerToken); + }) + .catch(error => { + // Handle any errors that occur during the generation process + reject(error); + }); + } + } catch (e) { + // Handle any other unexpected exceptions + reject(e); + } + }); +} + +// Example 2: Generate Bearer Token using the credentials JSON string +function getSkyflowBearerTokenFromCreds() { + + // To generate Bearer Token from credentials string. + const skyflowCredentials = { + clientID: '', + clientName: '', + keyID: '', + tokenURI: '', + privateKey: '', + }; + + // Convert credentials dictionary to JSON string + const credentialsString = JSON.stringify(skyflowCredentials); + + return new Promise((resolve, reject) => { + try { + // Check if the token is already generated and still valid + if (!isExpired(bearerToken)) resolve(bearerToken); + else { + // Generate a new Bearer Token from the credentials string + generateBearerTokenFromCreds(credentialsString) + .then(response => { + bearerToken = response.accessToken; + // Resolve the generated Bearer Token + resolve(bearerToken); + }) + .catch(error => { + // Handle any errors that occur during the generation process + reject(error); + }); + } + } catch (e) { + // Handle any other unexpected exceptions + reject(e); + } + }); +} + +const tokens = async () => { + console.log(await getSkyflowBearerTokenFromFilePath()); + console.log(await getSkyflowBearerTokenFromCreds()); +}; + +tokens(); +``` + +#### Generate bearer tokens with context +**Context-aware authorization** embeds context values into a bearer token during its generation and so you can reference those values in your policies. This enables more flexible access controls, such as helping you track end-user identity when making API calls using service accounts, and facilitates using signed data tokens during detokenization. + +A service account with the context_id identifier generates bearer tokens containing context information, represented as a JWT claim in a Skyflow-generated bearer token. Tokens generated from such service accounts include a context_identifier claim, are valid for 60 minutes, and can be used to make API calls to the Data and Management APIs, depending on the service account's permissions. + +#### [Example](https://github.com/skyflowapi/skyflow-node/blob/v2/samples/service-account/token-generation-with-context-example.ts): + +```javascript +import { + generateBearerToken, + generateBearerTokenFromCreds, + isExpired, + SkyflowError +} from 'skyflow-node'; + +/* +Example program to generate a Bearer Token using Skyflow's BearerToken utility. +The token is generated using two approaches: +1. By providing the credentials.json file path. +2. By providing the contents of credentials.json as a string. +*/ + +// Variable to store the generated token +let bearerToken = ''; + +// Specify the full file path to the credentials.json file +const filepath = 'CREDENTIALS_FILE_PATH'; + +// Approach 1: Generate Bearer Token by specifying the path to the credentials.json file +function getSkyflowBearerTokenWithContextFromFilePath() { + return new Promise((resolve, reject) => { + try { + // Set context string (example: "context_id") + const options = { + ctx: 'context_id', + }; + // Check if the token is already generated and still valid + if (!isExpired(bearerToken)) resolve(bearerToken); + else { + // Generate a new Bearer Token from the credentials file + generateBearerToken(filepath, options) + .then(response => { + bearerToken = response.accessToken; + // Resolve the generated Bearer Token + resolve(bearerToken); + }) + .catch(error => { + // Handle any errors that occur during the generation process + reject(error); + }); + } + } catch (e) { + // Handle any other unexpected exceptions + reject(e); + } + }); +} + +// Approach 2: Generate Bearer Token by specifying the contents of credentials.json as a string +function getSkyflowBearerTokenWithContextFromCreds() { + + // To generate Bearer Token from credentials string. + const skyflowCredentials = { + clientID: '', + clientName: '', + keyID: '', + tokenURI: '', + privateKey: '', + }; + + // Convert credentials dictionary to JSON string + const credentialsString = JSON.stringify(skyflowCredentials); + + return new Promise((resolve, reject) => { + try { + // Set context string (example: "context_id") + const options = { + ctx: 'context_id', + }; + // Check if the token is already generated and still valid + if (!isExpired(bearerToken)) resolve(bearerToken); + else { + // Generate a new Bearer Token from the credentials string and options + generateBearerTokenFromCreds(credentialsString, options) + .then(response => { + bearerToken = response.accessToken; + // Resolve the generated Bearer Token + resolve(bearerToken); + }) + .catch(error => { + // Handle any errors that occur during the generation process + reject(error); + }); + } + } catch (e) { + // Handle any other unexpected exceptions + reject(e); + } + }); +} + +const tokens = async () => { + console.log(await getSkyflowBearerTokenWithContextFromFilePath()); + console.log(await getSkyflowBearerTokenWithContextFromCreds()); +}; + +tokens(); +``` + +#### Generate scoped bearer tokens +A service account with multiple roles can generate bearer tokens with access limited to a specific role by specifying the appropriate roleID. This can be used to limit access to specific roles for services with multiple responsibilities, such as segregating access for billing and analytics. The generated bearer tokens are valid for 60 minutes and can only execute operations permitted by the permissions associated with the designated role. + +#### [Example](https://github.com/skyflowapi/skyflow-node/blob/v2/samples/service-account/scoped-token-generation-example.ts): + +```javascript +import { + generateBearerToken, + generateBearerTokenFromCreds, + isExpired, + SkyflowError +} from 'skyflow-node'; + +/* +Example program to generate a Scoped Token using Skyflow's BearerToken utility. +The token is generated by providing the file path to the credentials.json file +and specifying roles associated with the token. +*/ + +// Variable to store the generated token +let bearerToken = ''; + +// Specify the full file path to the credentials.json file +const filepath = 'CREDENTIALS_FILE_PATH'; + +// Example: Generate Scoped Token by specifying the credentials.json file path +function getScopedBearerTokenFromFilePath() { + return new Promise((resolve, reject) => { + try { + // Set the role ids + const options = { + roleIDs: ['roleID1', 'roleID2'], + }; + // Check if the token is already generated and still valid + if (!isExpired(bearerToken)) resolve(bearerToken); + else { + // Generate a new Bearer Token from the credentials file and associated roles + generateBearerToken(filepath, options) + .then(response => { + bearerToken = response.accessToken; + // Resolve the generated Bearer Token + resolve(bearerToken); + }) + .catch(error => { + // Handle any errors that occur during the generation process + reject(error); + }); + } + } catch (e) { + // Handle any other unexpected exceptions + reject(e); + } + }); +} + +const tokens = async () => { + console.log(await getScopedBearerTokenFromFilePath()); +}; + +tokens(); + +``` + +#### Generate signed data tokens +Skyflow generates data tokens when sensitive data is inserted into the vault. These data tokens can be digitally signed with a service account's private key, adding an extra layer of protection. Signed tokens can only be detokenized by providing the signed data token along with a bearer token generated from the service account's credentials. The service account must have the necessary permissions and context to successfully detokenize the signed data tokens. + +#### [Example](https://github.com/skyflowapi/skyflow-node/blob/v2/samples/service-account/signed-token-generation-example.ts): + +```javascript +import { + generateSignedDataTokens, + generateSignedDataTokensFromCreds + isExpired, + SkyflowError +} from 'skyflow-node'; + +/* +Example program to generate Signed Data Tokens using Skyflow's utilities. +Signed Data Tokens can be generated in two ways: +1. By specifying the file path to the credentials.json file. +2. By providing the credentials as a JSON string. +*/ + +// Specify the full file path to the credentials.json file +const filepath = 'CREDENTIALS_FILE_PATH'; + +// Example 1: Generate Signed Data Tokens using a credentials file +function getSignedTokenFromFilePath() { + return new Promise(async (resolve, reject) => { + try { + // Options for generating signed data tokens + const options = { + ctx: 'ctx', // Set the context value + dataTokens: ['dataToken1', 'dataToken2'], // Set the data tokens to be signed + timeToLive: 90 // Set the token's time-to-live (TTL) in seconds + }; + // Generate and retrieve the signed data tokens + let response = await generateSignedDataTokens(filepath, options); + resolve(response); + } catch (e) { + // Handle any errors that occur during the generation process + reject(e); + } + }); +} + +// Example 2: Generate Signed Data Tokens using credentials as a JSON string +function getSignedTokenFromCreds() { + + // To generate Bearer Token from credentials string. + const skyflowCredentials = { + clientID: '', + clientName: '', + keyID: '', + tokenURI: '', + privateKey: '', + }; + + // Convert credentials dictionary to JSON string + const credentialsString = JSON.stringify(skyflowCredentials); + + return new Promise(async (resolve, reject) => { + try { + // Options for generating signed data tokens + const options = { + ctx: 'ctx', // Set the context value + dataTokens: ['dataToken1', 'dataToken2'], // Set the data tokens to be signed + timeToLive: 90, // Set the token's time-to-live (TTL) in seconds + }; + // Generate and retrieve the signed data tokens + let response = await generateSignedDataTokensFromCreds( + credentialsString, + options + ); + resolve(response); + } catch (e) { + // Handle any errors that occur during the generation process + reject(e); + } + }); +} + +const tokens = async () => { + try { + const tokenResponseFromFilePath = await getSignedTokenFromFilePath(); + tokenResponseFromFilePath.forEach((response) => { + console.log(`Data Token: ${response.token}`); + console.log(`Signed Data Token: ${response.signedToken}`); + }); + + const tokenResponseFromCreds = await getSignedTokenFromCreds(); + tokenResponseFromCreds.forEach((response) => { + console.log(`Data Token: ${response.token}`); + console.log(`Signed Data Token: ${response.signedToken}`); + }); + } catch (error) { + console.log(error); + } +}; + +tokens(); +``` + +Notes: +- The `timeToLive` (TTL) value should be specified in seconds. +- By default, the TTL value is set to 60 seconds. + +#### Bearer token expiry edge case +When you use bearer tokens for authentication and API requests in SDKs, there's the potential for a token to expire after the token is verified as valid but before the actual API call is made, causing the request to fail unexpectedly due to the token's expiration. An error from this edge case would look something like this: + +```txt +message: Authentication failed. Bearer token is expired. Use a valid bearer token. See https://docs.skyflow.com/api-authentication/ +``` + +If you encounter this kind of error, retry the request. During the retry, the SDK detects that the previous bearer token has expired and generates a new one for the current and subsequent requests. + +#### [Example](https://github.com/skyflowapi/skyflow-node/blob/v2/samples/service-account/bearer-token-expiry-example.ts.ts): + +```javascript +import { + Credentials, + DetokenizeOptions, + DetokenizeRequest, + DetokenizeResponse, + DetokenizeData, + Env, + LogLevel, + RedactionType, + Skyflow, + SkyflowError, + VaultConfig, + SkyflowConfig +} from 'skyflow-node'; + +/** +* This example demonstrates how to configure and use the Skyflow SDK +* to detokenize sensitive data stored in a Skyflow vault. +* It includes setting up credentials, configuring the vault, and +* making a detokenization request. The code also implements a retry +* mechanism to handle unauthorized access errors (HTTP 401). +*/ +async function detokenizeData(skyflowClient: Skyflow, vaultId: string) { + try { + // Creating a list of tokens to be detokenized + const detokenizeData: DetokenizeData[] = [ + { + token: "", // Replace with your actual token value + redactionType: RedactionType.MASKED, // Redaction type + }, + { + token: "", // Replace with your actual token value + redactionType: RedactionType.PLAIN_TEXT, // Redaction type + }, + ]; + + // Building a detokenization request + const detokenizeRequest: DetokenizeRequest = new DetokenizeRequest( + detokenizeData + ); + + // Configuring detokenization options + const detokenizeOptions: DetokenizeOptions = new DetokenizeOptions(); + detokenizeOptions.setContinueOnError(false); // Stop on error + detokenizeOptions.setDownloadURL(false); // Disable download URL generation + + // Sending the detokenization request and receiving the response + const response: DetokenizeResponse = await skyflowClient + .vault(vaultId) + .detokenize(detokenizeRequest, detokenizeOptions); + + // Printing the detokenized response + console.log('Detokenization successful:', response); + } catch (err) { + throw err; + } +} + +async function main() { + try { + // Setting up credentials for accessing the Skyflow vault + const credentials: Credentials = { + credentialsString: '', // Credentials string for authentication + }; + + // Configuring the Skyflow vault with necessary details + const primaryVaultConfig: VaultConfig = { + vaultId: '', // Vault ID + clusterId: '', // Cluster ID + env: Env.PROD, // Environment set to PROD + credentials: credentials // Setting credentials + }; + + // Creating a Skyflow client instance with the configured vault + const skyflowConfig: SkyflowConfig = { + vaultConfigs: [primaryVaultConfig], + logLevel: LogLevel.INFO, // Recommended to use LogLevel.ERROR in production environment. + }; + + const skyflowClient: Skyflow = new Skyflow(skyflowConfig); + + // Attempting to detokenize data using the Skyflow client + try { + await detokenizeData(skyflowClient, primaryVaultConfig.vaultId); + } catch (err) { + // Retry detokenization if the error is due to unauthorized access (HTTP 401) + if (err instanceof SkyflowError && err.error?.http_code === 401) { + console.warn('Unauthorized access detected. Retrying...'); + await detokenizeData(skyflowClient, primaryVaultConfig.vaultId); + } else { + // Rethrow the exception for other error codes + throw err; + } + } + } catch (err) { + // Handling any exceptions that occur during the process + console.error('An error occurred:', err); + } +} + +// Invoke the main function +main(); + +``` + +## Logging + +The SDK provides useful logging. By default the logging level of the SDK is set to `LogLevel.ERROR`. This can be changed by setting the `logLevel` in Skyflow Config while creating the Skyflow Client as shown below: + +Currently, the following five log levels are supported: +- `DEBUG`: +When `LogLevel.DEBUG` is passed, logs at all levels will be printed (DEBUG, INFO, WARN, ERROR). +- `INFO`: +When `LogLevel.INFO` is passed, INFO logs for every event that occurs during SDK flow execution will be printed, along with WARN and ERROR logs. +- `WARN`: +When `LogLevel.WARN` is passed, only WARN and ERROR logs will be printed. +- `ERROR`: +When `LogLevel.ERROR` is passed, only ERROR logs will be printed. +- `OFF`: +`LogLevel.OFF` can be used to turn off all logging from the Skyflow Python SDK. + +**Note:** The ranking of logging levels is as follows: `DEBUG` < `INFO` < `WARN` < `ERROR` < `OFF`. + +```typescript +import { + Skyflow, + LogLevel, + SkyflowError +} from 'skyflow-node'; + +/* +This example demonstrates how to configure the Skyflow client with custom log levels and authentication credentials (either token, credentials string, or other methods). It also shows how to configure a vault connection using specific parameters. +1. Set up credentials with a Bearer token or credentials string. +2. Define the Vault configuration. +3. Build the Skyflow client with the chosen configuration and set log level. +4. Example of changing the log level from ERROR (default) to INFO. +*/ + +try { + // Step 1: Set up credentials - either pass token or use credentials string + // In this case, we are using a Bearer token for authentication + const credentials: Credentials = { + token: '', // Replace with actual Bearer token + }; + + // Step 2: Define the Vault configuration + // Configure the vault with necessary details like vault ID, cluster ID, and environment + const vaultConfig: VaultConfig = { + vaultId: '', // Replace with actual Vault ID (primary vault) + clusterId: '', // Replace with actual Cluster ID (from vault URL) + env: Env.PROD, // Set the environment (default is PROD) + credentials: credentials // Set credentials for the vault (either token or credentials) + }; + + // Step 3: Define additional Skyflow credentials (optional, if needed for credentials string) + const skyflowCredentials = { + clientID: '', + clientName: '', + keyID: '', + tokenURI: '', + privateKey: '', + }; + + // Convert the credentials object to a json string format to be used for generating a Bearer Token + const credentialsString = JSON.stringify(skyflowCredentials); + + // Step 9: Build and initialize the Skyflow client after creating Skyflow Config + // Skyflow client is configured with multiple vaults and credentials. + const skyflowConfig: SkyflowConfig = { + vaultConfigs: [vaultConfig], // Add the Vault configuration + skyflowCredentials: skyflowCredentials, // Use Skyflow credentials if no token is passed + logLevel: LogLevel.INFO // Recommended to use LogLevel.ERROR in production environment. + }; + + // Step 10: Initialize Skyflow Client + const skyflowClient: Skyflow = new Skyflow(skyflowConfig); + + // Now, the Skyflow client is ready to use with the specified log level and credentials + console.log('Skyflow client has been successfully configured with log level: INFO.') +} catch(error) { + // Step 11: Handle any exceptions that occur + if (error instanceof SkyflowError) { + console.error('Skyflow Specific Error:', { + code: error.error?.http_code, + message: error.message, + details: error.error?.details + }); + } else { + console.error('Unexpected Error:', error); + } +} +``` + +## Reporting a Vulnerability + +If you discover a potential security issue in this project, please reach out to us at **security@skyflow.com**. Please do not create public GitHub issues or Pull Requests, as malicious actors could potentially view them. \ No newline at end of file diff --git a/docs/skyflow-node/samples/README.md b/docs/skyflow-node/samples/README.md new file mode 100644 index 0000000..967f240 --- /dev/null +++ b/docs/skyflow-node/samples/README.md @@ -0,0 +1,138 @@ +# Node.js SDK samples + +Test the SDK by adding `VAULT-ID`, `VAULT-URL`, and `SERVICE-ACCOUNT` details in the required places for each sample. + +## Prerequisites + +- A Skylow account. If you don't have one, register for one on the [Try Skyflow](https://skyflow.com/try-skyflow) page. +- [Node 7.6.0](https://nodejs.org/en/) and above. + +## Prepare + +- Install the Node SDK: + +```bash +npm install skyflow-node +``` + +### Create the vault + +1. In a browser, navigate to Skyflow Studio. +2. Create a vault by clicking **Create Vault** > **Start With a Template** > **Quickstart vault**. +3. Once the vault is created, click the gear icon and select **Edit Vault Details**. +4. Note your **Vault URL** and **Vault ID** values, then click **Cancel**. You'll need these later. + +### Create a service account + +1. In the side navigation click, **IAM** > **Service Accounts** > **New Service Account**. +2. For **Name**, enter "SDK Samples". For **Roles**, choose **Vault Editor**. +3. Click **Create**. Your browser downloads a **credentials.json** file. Keep this file secure. You'll need it for each of the samples. + +## The samples + +### Detokenize + +Detokenize a data token from the vault. Make sure the specified token is for data that exists in the vault. If you need a valid token, use [Insert.ts](./vault-api/Insert.ts) to insert the data, then use this data's token for detokenization. + +#### Configure + +1. Replace **** with **VAULT ID** +2. Replace **** with **VAULT URL**. +3. Replace **** with **COLUMN NAME**. +4. Replace **** with **Data Token 1**. +5. Replace **** with **Data Token 2**. +6. Replace **** with **Data Token 3**. +7. Replace **** with **Data Token 4**. +8. Replace **** with relative path of **SERVICE ACCOUNT CREDENTIAL FILE**. + +#### Run the sample + +```bash +ts-node Detokenize.ts +``` + +### GetById + +Get data using skyflow id. + +#### Configure + +1. Replace **** with **VAULT ID** +2. Replace **** with **VAULT URL**. +3. Replace **** with **Skyflow Id 1**. +4. Replace **** with **Skyflow Id 2**. +5. Replace **** with **Skyflow Id 3**. +6. Replace **** with relative path of **SERVICE ACCOUNT CREDENTIAL FILE**. +7. Replace **** with **credit_cards**. + +#### Run the sample + +```bash +ts-node GetById.ts +``` + +### Insert + +Insert data in the vault. + +#### Configure + +1. Replace **** with **VAULT ID**. +2. Replace **** with **VAULT URL**. +3. Replace **** with relative path of **SERVICE ACCOUNT CREDENTIAL FILE**. +4. Replace **** with **credit_cards**. +5. Replace **** with **column name**. +6. Replace **** with **valid value corresponding to column name**. + +#### Run the sample + +```bash +ts-node Insert.ts +``` + +### InvokeConnection + +Skyflow Connections is a gateway service that uses Skyflow's underlying tokenization capabilities to securely connect to first-party and third-party services. This way, your infrastructure is never directly exposed to sensitive data, and you offload security and compliance requirements to Skyflow. + +#### Configure + +1. Replace **** with **VAULT ID**. +2. Replace **** with **VAULT URL**. +3. Replace **** with relative path of **SERVICE ACCOUNT CREDENTIAL FILE**. +4. Replace **** with **Connection url**. +5. Give **Authorization** value as the tokens. +6. Replace value of **requestBody** with your's request body content. + +#### Run the sample + +```bash +ts-node InvokeConnection.ts +``` + +### Service account token generation + +Generates a service account Bearer token using the file path of credentials.json. + +#### Configure + +1. Replace **** with relative path of **SERVICE ACCOUNT CREDENTIAL FILE**. + +#### Run the sample + +```bash +ts-node TokenGenerationExample.ts +``` + +### Generate Bearer Token From Credentails + +Generates a service account bearer token using the JSON content of a credentials file. + +#### Configure + +1. Replace **credentials\*** with json data of downloaded credentials file while creation Service account. + +#### Run the sample + +```bash +ts-node TokenGenerationUsingCredContent.ts +``` diff --git a/src/commands/deidentify.ts b/src/commands/deidentify.ts new file mode 100644 index 0000000..7a750af --- /dev/null +++ b/src/commands/deidentify.ts @@ -0,0 +1,220 @@ +import { Command } from 'commander'; +import inquirer from 'inquirer'; +import { + DeidentifyTextRequest, + DeidentifyTextOptions, + DeidentifyTextResponse, + DetectEntities, + TokenFormat, + TokenType, +} from 'skyflow-node'; +import { DeidentifyCommandOptions } from '../types'; +import { + initializeSkyflowClient, + handleSkyflowError, + resolveVaultId, +} from '../utils/skyflow'; +import { logVerbose } from '../utils/logger'; + +// Map of entity aliases to DetectEntities enum values +const ENTITY_MAP: Record = { + SSN: DetectEntities.SSN, + CREDIT_CARD: DetectEntities.CREDIT_CARD, + CREDIT_CARD_NUMBER: DetectEntities.CREDIT_CARD, + EMAIL: DetectEntities.EMAIL_ADDRESS, + EMAIL_ADDRESS: DetectEntities.EMAIL_ADDRESS, + PHONE_NUMBER: DetectEntities.PHONE_NUMBER, + PHONE: DetectEntities.PHONE_NUMBER, + NAME: DetectEntities.NAME, + DOB: DetectEntities.DOB, + DATE_OF_BIRTH: DetectEntities.DOB, + ACCOUNT_NUMBER: DetectEntities.ACCOUNT_NUMBER, + DRIVER_LICENSE: DetectEntities.DRIVER_LICENSE, + PASSPORT_NUMBER: DetectEntities.PASSPORT_NUMBER, + PASSPORT: DetectEntities.PASSPORT_NUMBER, +}; + +const AVAILABLE_ENTITIES = Object.keys(ENTITY_MAP).join(', '); + +export const deidentifyCommand = (program: Command): void => { + program + .command('deidentify') + .description('Detect and redact sensitive data from text using Skyflow Detect API') + .option('--text ', 'Text to deidentify (or pipe from stdin)') + .option( + '--entities ', + `Comma-separated entity types to detect (e.g., SSN,CREDIT_CARD). Available: ${AVAILABLE_ENTITIES}` + ) + .option( + '--token-type ', + 'Token format: vault_token (stored), entity_only (labels), random_token (ephemeral)', + 'vault_token' + ) + .option('--output ', 'Output format: text or json', 'text') + .option('--vault-id ', 'Vault ID (or set SKYFLOW_VAULT_ID)') + .option('--cluster-id ', 'Cluster ID (or set SKYFLOW_VAULT_URL)') + .option('--environment ', 'Environment: PROD, SANDBOX, STAGE, DEV', 'PROD') + .action(async (options: DeidentifyCommandOptions) => { + try { + // Get text from option or stdin + let textInput = options.text; + if (!textInput) { + // Check if stdin is being piped + if (!process.stdin.isTTY) { + logVerbose('Reading text from stdin...'); + textInput = await readStdin(); + } else { + // Prompt for text + const answers = await inquirer.prompt([ + { + type: 'editor', + name: 'text', + message: 'Enter text to deidentify:', + validate: (input: string) => { + if (!input) return 'Text is required'; + return true; + }, + }, + ]); + textInput = answers.text; + } + } + + // Parse entities if provided + let entityList: DetectEntities[] | undefined; + if (options.entities) { + const rawEntities = options.entities.split(',').map((e) => e.trim().toUpperCase()); + entityList = rawEntities.map((entity) => { + const mapped = ENTITY_MAP[entity]; + if (!mapped) { + throw new Error( + `Unknown entity type: ${entity}\nAvailable entities: ${AVAILABLE_ENTITIES}` + ); + } + return mapped; + }); + logVerbose(`Detecting entities: ${entityList.join(', ')}`); + } else { + // Use common entities by default + entityList = [ + DetectEntities.SSN, + DetectEntities.CREDIT_CARD, + DetectEntities.EMAIL_ADDRESS, + DetectEntities.PHONE_NUMBER, + DetectEntities.NAME, + DetectEntities.DOB, + ]; + logVerbose('Using default entity detection (SSN, CREDIT_CARD, EMAIL_ADDRESS, PHONE_NUMBER, NAME, DOB)'); + } + + // Parse token type + let tokenType: TokenType; + switch (options.tokenType?.toLowerCase()) { + case 'vault_token': + tokenType = TokenType.VAULT_TOKEN; + break; + case 'entity_only': + tokenType = TokenType.ENTITY_ONLY; + break; + case 'entity_unique_counter': + tokenType = TokenType.ENTITY_UNIQUE_COUNTER; + break; + default: + tokenType = TokenType.VAULT_TOKEN; + logVerbose(`Unknown token type '${options.tokenType}', using vault_token`); + } + logVerbose(`Using token type: ${options.tokenType || 'vault_token'}`); + + // Resolve vault ID + const vaultId = resolveVaultId(options.vaultId); + logVerbose(`Using vault ID: ${vaultId}`); + + // Initialize Skyflow client + const verbose = program.opts().verbose || false; + const skyflowClient = initializeSkyflowClient( + vaultId, + options.clusterId, + options.environment, + verbose + ); + + // Create deidentify request + const deidentifyRequest = new DeidentifyTextRequest(textInput!); + logVerbose('Created deidentify request'); + + // Configure options + const deidentifyOptions = new DeidentifyTextOptions(); + deidentifyOptions.setEntities(entityList); + + const tokenFormat = new TokenFormat(); + tokenFormat.setDefault(tokenType); + deidentifyOptions.setTokenFormat(tokenFormat); + + // Execute deidentify + logVerbose('Executing deidentify operation...'); + const response: DeidentifyTextResponse = await skyflowClient + .detect(vaultId!) + .deidentifyText(deidentifyRequest, deidentifyOptions); + + // Display results + if (options.output === 'json') { + console.log(JSON.stringify(response, null, 2)); + } else { + console.log('\nDeidentified Text:'); + console.log('─'.repeat(60)); + console.log(response.processedText); + console.log('─'.repeat(60)); + + if (response.entities && response.entities.length > 0) { + console.log(`\nDetected ${response.entities.length} sensitive entities:\n`); + response.entities.forEach((entity, index) => { + console.log(`${index + 1}. ${entity.entity}`); + console.log(` Original: "${entity.value}"`); + console.log(` Token: ${entity.token}`); + if (entity.textIndex) { + console.log(` Position: ${entity.textIndex.start}-${entity.textIndex.end}`); + } + if (entity.scores) { + const confidence = Object.values(entity.scores)[0]; + console.log(` Confidence: ${(confidence * 100).toFixed(1)}%`); + } + console.log(); + }); + } else { + console.log('\nNo sensitive entities detected.'); + } + + if (response.wordCount !== undefined) { + console.log(`Word count: ${response.wordCount}`); + } + if (response.charCount !== undefined) { + console.log(`Character count: ${response.charCount}`); + } + } + } catch (error) { + handleSkyflowError(error); + } + }); +}; + +/** + * Read input from stdin + */ +const readStdin = (): Promise => { + return new Promise((resolve, reject) => { + let data = ''; + process.stdin.setEncoding('utf8'); + + process.stdin.on('data', (chunk) => { + data += chunk; + }); + + process.stdin.on('end', () => { + resolve(data.trim()); + }); + + process.stdin.on('error', (error) => { + reject(error); + }); + }); +}; diff --git a/src/commands/insert.ts b/src/commands/insert.ts new file mode 100644 index 0000000..ab75657 --- /dev/null +++ b/src/commands/insert.ts @@ -0,0 +1,177 @@ +import { Command } from 'commander'; +import inquirer from 'inquirer'; +import { InsertRequest, InsertOptions, InsertResponse } from 'skyflow-node'; +import { InsertCommandOptions, InsertData } from '../types'; +import { + initializeSkyflowClient, + handleSkyflowError, + resolveVaultId, +} from '../utils/skyflow'; +import { logVerbose } from '../utils/logger'; + +export const insertCommand = (program: Command): void => { + program + .command('insert') + .description('Insert sensitive data into a Skyflow vault table') + .option('--table ', 'Table name to insert data into') + .option('--data ', 'JSON data to insert (or pipe from stdin)') + .option('--return-tokens', 'Return tokens for inserted data', false) + .option('--continue-on-error', 'Continue if some records fail', false) + .option('--upsert-column ', 'Column name for upsert operations') + .option('--vault-id ', 'Vault ID (or set SKYFLOW_VAULT_ID)') + .option('--cluster-id ', 'Cluster ID (or set SKYFLOW_VAULT_URL)') + .option('--environment ', 'Environment: PROD, SANDBOX, STAGE, DEV', 'PROD') + .action(async (options: InsertCommandOptions) => { + try { + // Prompt for missing required options + if (!options.table) { + const answers = await inquirer.prompt([ + { + type: 'input', + name: 'table', + message: 'Enter table name:', + validate: (input: string) => { + if (!input) return 'Table name is required'; + return true; + }, + }, + ]); + options.table = answers.table; + } + + // Get data from option or stdin + let dataInput = options.data; + if (!dataInput) { + // Check if stdin is being piped + if (!process.stdin.isTTY) { + logVerbose('Reading data from stdin...'); + dataInput = await readStdin(); + } else { + // Prompt for data + const answers = await inquirer.prompt([ + { + type: 'editor', + name: 'data', + message: 'Enter JSON data to insert:', + validate: (input: string) => { + if (!input) return 'Data is required'; + try { + JSON.parse(input); + return true; + } catch { + return 'Invalid JSON format'; + } + }, + }, + ]); + dataInput = answers.data; + } + } + + // Parse JSON data + let insertData: InsertData[]; + try { + const parsed = JSON.parse(dataInput!); + // Ensure data is an array + insertData = Array.isArray(parsed) ? parsed : [parsed]; + logVerbose(`Parsed ${insertData.length} record(s) to insert`); + } catch (error) { + throw new Error(`Invalid JSON data: ${error instanceof Error ? error.message : String(error)}`); + } + + // Resolve vault ID + const vaultId = resolveVaultId(options.vaultId); + logVerbose(`Using vault ID: ${vaultId}`); + + // Initialize Skyflow client + const verbose = program.opts().verbose || false; + const skyflowClient = initializeSkyflowClient( + vaultId, + options.clusterId, + options.environment, + verbose + ); + + // Create insert request + const insertRequest = new InsertRequest(options.table, insertData); + logVerbose(`Created insert request for table: ${options.table}`); + + // Configure insert options + const insertOptions = new InsertOptions(); + if (options.returnTokens) { + insertOptions.setReturnTokens(true); + logVerbose('Configured to return tokens'); + } + if (options.continueOnError) { + insertOptions.setContinueOnError(true); + logVerbose('Configured to continue on error'); + } + if (options.upsertColumn) { + insertOptions.setUpsertColumn(options.upsertColumn); + logVerbose(`Configured upsert on column: ${options.upsertColumn}`); + } + + // Execute insert + logVerbose('Executing insert operation...'); + const response: InsertResponse = await skyflowClient + .vault(vaultId!) + .insert(insertRequest, insertOptions); + + // Display results + console.log('\nInsert completed successfully!\n'); + + if (response.insertedFields && response.insertedFields.length > 0) { + console.log('Inserted records:'); + response.insertedFields.forEach((record, index) => { + console.log(`\nRecord ${index + 1}:`); + if (record.skyflowId) { + console.log(` Skyflow ID: ${record.skyflowId}`); + } + if (options.returnTokens) { + Object.keys(record).forEach((key) => { + if (key !== 'skyflowId') { + console.log(` ${key}: ${record[key]}`); + } + }); + } + }); + } + + if (response.errors && response.errors.length > 0) { + console.log('\nErrors:'); + response.errors.forEach((error, index) => { + console.log(`\nError ${index + 1}:`); + console.log(` ${JSON.stringify(error, null, 2)}`); + }); + } + + console.log(`\nTotal records processed: ${insertData.length}`); + console.log(`Successful: ${response.insertedFields?.length || 0}`); + console.log(`Failed: ${response.errors?.length || 0}`); + } catch (error) { + handleSkyflowError(error); + } + }); +}; + +/** + * Read input from stdin + */ +const readStdin = (): Promise => { + return new Promise((resolve, reject) => { + let data = ''; + process.stdin.setEncoding('utf8'); + + process.stdin.on('data', (chunk) => { + data += chunk; + }); + + process.stdin.on('end', () => { + resolve(data.trim()); + }); + + process.stdin.on('error', (error) => { + reject(error); + }); + }); +}; diff --git a/src/commands/reidentify.ts b/src/commands/reidentify.ts new file mode 100644 index 0000000..3cc6f31 --- /dev/null +++ b/src/commands/reidentify.ts @@ -0,0 +1,190 @@ +import { Command } from 'commander'; +import inquirer from 'inquirer'; +import { + ReidentifyTextRequest, + ReidentifyTextOptions, + ReidentifyTextResponse, + DetectEntities, +} from 'skyflow-node'; +import { ReidentifyCommandOptions } from '../types'; +import { + initializeSkyflowClient, + handleSkyflowError, + resolveVaultId, +} from '../utils/skyflow'; +import { logVerbose } from '../utils/logger'; + +// Map of entity aliases to DetectEntities enum values +const ENTITY_MAP: Record = { + SSN: DetectEntities.SSN, + CREDIT_CARD: DetectEntities.CREDIT_CARD, + CREDIT_CARD_NUMBER: DetectEntities.CREDIT_CARD, + EMAIL: DetectEntities.EMAIL_ADDRESS, + EMAIL_ADDRESS: DetectEntities.EMAIL_ADDRESS, + PHONE_NUMBER: DetectEntities.PHONE_NUMBER, + PHONE: DetectEntities.PHONE_NUMBER, + NAME: DetectEntities.NAME, + DOB: DetectEntities.DOB, + DATE_OF_BIRTH: DetectEntities.DOB, + ACCOUNT_NUMBER: DetectEntities.ACCOUNT_NUMBER, + DRIVER_LICENSE: DetectEntities.DRIVER_LICENSE, + PASSPORT_NUMBER: DetectEntities.PASSPORT_NUMBER, + PASSPORT: DetectEntities.PASSPORT_NUMBER, +}; + +const AVAILABLE_ENTITIES = Object.keys(ENTITY_MAP).join(', '); + +export const reidentifyCommand = (program: Command): void => { + program + .command('reidentify') + .description('Restore original values from tokenized text using Skyflow Detect API') + .option('--text ', 'Tokenized text to reidentify (or pipe from stdin)') + .option( + '--plain-text ', + `Comma-separated entities to return as plain text (e.g., SSN,CREDIT_CARD). Available: ${AVAILABLE_ENTITIES}` + ) + .option( + '--masked ', + 'Comma-separated entities to return masked' + ) + .option( + '--redacted ', + 'Comma-separated entities to keep redacted' + ) + .option('--output ', 'Output format: text or json', 'text') + .option('--vault-id ', 'Vault ID (or set SKYFLOW_VAULT_ID)') + .option('--cluster-id ', 'Cluster ID (or set SKYFLOW_VAULT_URL)') + .option('--environment ', 'Environment: PROD, SANDBOX, STAGE, DEV', 'PROD') + .action(async (options: ReidentifyCommandOptions) => { + try { + // Get text from option or stdin + let textInput = options.text; + if (!textInput) { + // Check if stdin is being piped + if (!process.stdin.isTTY) { + logVerbose('Reading text from stdin...'); + textInput = await readStdin(); + } else { + // Prompt for text + const answers = await inquirer.prompt([ + { + type: 'editor', + name: 'text', + message: 'Enter tokenized text to reidentify:', + validate: (input: string) => { + if (!input) return 'Text is required'; + return true; + }, + }, + ]); + textInput = answers.text; + } + } + + // Parse entity options + const parseEntities = (entityString?: string): DetectEntities[] | undefined => { + if (!entityString) return undefined; + const rawEntities = entityString.split(',').map((e) => e.trim().toUpperCase()); + return rawEntities.map((entity) => { + const mapped = ENTITY_MAP[entity]; + if (!mapped) { + throw new Error( + `Unknown entity type: ${entity}\nAvailable entities: ${AVAILABLE_ENTITIES}` + ); + } + return mapped; + }); + }; + + const plainTextEntities = parseEntities(options.plainText); + const maskedEntities = parseEntities(options.masked); + const redactedEntities = parseEntities(options.redacted); + + if (plainTextEntities) { + logVerbose(`Plain text entities: ${plainTextEntities.join(', ')}`); + } + if (maskedEntities) { + logVerbose(`Masked entities: ${maskedEntities.join(', ')}`); + } + if (redactedEntities) { + logVerbose(`Redacted entities: ${redactedEntities.join(', ')}`); + } + + // If no options specified, default to plain text for all entities + const hasEntityOptions = plainTextEntities || maskedEntities || redactedEntities; + if (!hasEntityOptions) { + logVerbose('No entity options specified, returning all as plain text'); + } + + // Resolve vault ID + const vaultId = resolveVaultId(options.vaultId); + logVerbose(`Using vault ID: ${vaultId}`); + + // Initialize Skyflow client + const verbose = program.opts().verbose || false; + const skyflowClient = initializeSkyflowClient( + vaultId, + options.clusterId, + options.environment, + verbose + ); + + // Create reidentify request + const reidentifyRequest = new ReidentifyTextRequest(textInput!); + logVerbose('Created reidentify request'); + + // Configure options + const reidentifyOptions = new ReidentifyTextOptions(); + if (plainTextEntities) { + reidentifyOptions.setPlainTextEntities(plainTextEntities); + } + if (maskedEntities) { + reidentifyOptions.setMaskedEntities(maskedEntities); + } + if (redactedEntities) { + reidentifyOptions.setRedactedEntities(redactedEntities); + } + + // Execute reidentify + logVerbose('Executing reidentify operation...'); + const response: ReidentifyTextResponse = await skyflowClient + .detect(vaultId!) + .reidentifyText(reidentifyRequest, reidentifyOptions); + + // Display results + if (options.output === 'json') { + console.log(JSON.stringify(response, null, 2)); + } else { + console.log('\nReidentified Text:'); + console.log('─'.repeat(60)); + console.log(response.processedText); + console.log('─'.repeat(60)); + console.log('\nOriginal sensitive data has been restored.'); + } + } catch (error) { + handleSkyflowError(error); + } + }); +}; + +/** + * Read input from stdin + */ +const readStdin = (): Promise => { + return new Promise((resolve, reject) => { + let data = ''; + process.stdin.setEncoding('utf8'); + + process.stdin.on('data', (chunk) => { + data += chunk; + }); + + process.stdin.on('end', () => { + resolve(data.trim()); + }); + + process.stdin.on('error', (error) => { + reject(error); + }); + }); +}; diff --git a/src/index.ts b/src/index.ts index cb5103b..3a5518c 100644 --- a/src/index.ts +++ b/src/index.ts @@ -3,6 +3,9 @@ import { Command } from 'commander'; import { createVaultCommand } from './commands/createVault'; import { configureCommand } from './commands/configure'; +import { insertCommand } from './commands/insert'; +import { deidentifyCommand } from './commands/deidentify'; +import { reidentifyCommand } from './commands/reidentify'; import { loadConfig } from './utils/config'; import { setVerbose } from './utils/logger'; @@ -24,16 +27,27 @@ program // Register commands configureCommand(program); createVaultCommand(program); +insertCommand(program); +deidentifyCommand(program); +reidentifyCommand(program); // Error handler for authentication program.hook('preAction', async (thisCommand, actionCommand) => { + const commandName = actionCommand.name(); + // Skip authentication for configure command - if (actionCommand.name() === 'configure') { + if (commandName === 'configure') { + return; + } + + // Skip authentication for SDK commands (they handle their own auth) + const sdkCommands = ['insert', 'deidentify', 'reidentify']; + if (sdkCommands.includes(commandName)) { return; } try { - // Load and validate configuration + // Load and validate configuration for API commands loadConfig(); } catch (error) { console.error(`Authentication Error: ${error instanceof Error ? error.message : String(error)}`); diff --git a/src/types.ts b/src/types.ts index 92ae940..627d77b 100644 --- a/src/types.ts +++ b/src/types.ts @@ -39,3 +39,42 @@ export interface CreateVaultResult { serviceAccountID?: string; serviceAccountApiKey?: string; } + +// Insert command types +export interface InsertCommandOptions { + table: string; + data?: string; + returnTokens?: boolean; + continueOnError?: boolean; + upsertColumn?: string; + vaultId?: string; + clusterId?: string; + environment?: string; +} + +export interface InsertData { + [key: string]: unknown; +} + +// Deidentify command types +export interface DeidentifyCommandOptions { + text?: string; + entities?: string; + tokenType?: 'vault_token' | 'entity_only' | 'random_token'; + output?: 'text' | 'json'; + vaultId?: string; + clusterId?: string; + environment?: string; +} + +// Reidentify command types +export interface ReidentifyCommandOptions { + text?: string; + plainText?: string; + masked?: string; + redacted?: string; + output?: 'text' | 'json'; + vaultId?: string; + clusterId?: string; + environment?: string; +} diff --git a/src/utils/logger.ts b/src/utils/logger.ts index 1da4348..1e1f6d9 100644 --- a/src/utils/logger.ts +++ b/src/utils/logger.ts @@ -46,3 +46,13 @@ export const errorLog = (message: string, details?: any): void => { console.error(`ERROR: ${message}`); } }; + +/** + * Alias for verboseLog for consistency + */ +export const logVerbose = verboseLog; + +/** + * Alias for errorLog for consistency + */ +export const logError = errorLog; diff --git a/src/utils/skyflow.ts b/src/utils/skyflow.ts new file mode 100644 index 0000000..ca43083 --- /dev/null +++ b/src/utils/skyflow.ts @@ -0,0 +1,194 @@ +import { + Skyflow, + Credentials, + VaultConfig, + SkyflowConfig, + Env, + LogLevel, + SkyflowError, +} from 'skyflow-node'; +import { loadConfig } from './config'; +import { logError, logVerbose } from './logger'; + +/** + * Load Skyflow credentials from configuration or environment + */ +export const loadSkyflowCredentials = (): Credentials => { + // Check for API key in environment first + const apiKey = process.env.SKYFLOW_API_KEY; + if (apiKey) { + return { apiKey }; + } + + // Check for credentials file path in environment + const credentialsPath = process.env.SKYFLOW_CREDENTIALS_PATH; + if (credentialsPath) { + return { path: credentialsPath }; + } + + // Check for credentials string in environment + const credentialsString = process.env.SKYFLOW_CREDENTIALS; + if (credentialsString) { + return { credentialsString }; + } + + // Fall back to bearer token from config + const config = loadConfig(); + if (config.bearerToken) { + return { token: config.bearerToken }; + } + + throw new Error( + 'No Skyflow credentials found. Please set one of:\n' + + ' - SKYFLOW_API_KEY environment variable\n' + + ' - SKYFLOW_CREDENTIALS_PATH environment variable\n' + + ' - SKYFLOW_CREDENTIALS environment variable\n' + + ' - Or run: sky configure' + ); +}; + +/** + * Extract cluster ID from vault URL + * Example: https://ebfc9bee4242.vault.skyflowapis.com -> ebfc9bee4242 + */ +export const extractClusterIdFromUrl = (vaultUrl: string): string => { + const match = vaultUrl.match(/https?:\/\/([^.]+)\.vault\.skyflowapis/); + if (!match || !match[1]) { + throw new Error(`Invalid vault URL format: ${vaultUrl}`); + } + return match[1]; +}; + +/** + * Parse environment string to Env enum + */ +export const parseEnvironment = (env?: string): Env => { + const envUpper = (env || 'PROD').toUpperCase(); + switch (envUpper) { + case 'PROD': + case 'PRODUCTION': + return Env.PROD; + case 'SANDBOX': + return Env.SANDBOX; + case 'STAGE': + case 'STAGING': + return Env.STAGE; + case 'DEV': + case 'DEVELOPMENT': + return Env.DEV; + default: + logVerbose(`Unknown environment '${env}', defaulting to PROD`); + return Env.PROD; + } +}; + +/** + * Initialize Skyflow client with vault configuration + */ +export const initializeSkyflowClient = ( + vaultId: string, + clusterId?: string, + environment?: string, + verbose = false +): Skyflow => { + logVerbose('Initializing Skyflow client...'); + + // Load credentials + const credentials = loadSkyflowCredentials(); + logVerbose('Credentials loaded successfully'); + + // If no cluster ID provided, try to get from config or environment + let resolvedClusterId = clusterId; + if (!resolvedClusterId) { + const vaultUrl = process.env.SKYFLOW_VAULT_URL; + if (vaultUrl) { + resolvedClusterId = extractClusterIdFromUrl(vaultUrl); + logVerbose(`Cluster ID extracted from vault URL: ${resolvedClusterId}`); + } else { + throw new Error( + 'Cluster ID not provided. Please provide --cluster-id or set SKYFLOW_VAULT_URL environment variable' + ); + } + } + + // Parse environment + const env = parseEnvironment(environment); + logVerbose(`Using environment: ${environment || 'PROD'}`); + + // Create vault configuration + const vaultConfig: VaultConfig = { + vaultId, + clusterId: resolvedClusterId, + env, + credentials, + }; + + // Create Skyflow configuration + const skyflowConfig: SkyflowConfig = { + vaultConfigs: [vaultConfig], + skyflowCredentials: credentials, + logLevel: verbose ? LogLevel.INFO : LogLevel.ERROR, + }; + + // Initialize and return client + const client = new Skyflow(skyflowConfig); + logVerbose('Skyflow client initialized successfully'); + + return client; +}; + +/** + * Handle Skyflow-specific errors with user-friendly messages + */ +export const handleSkyflowError = (error: unknown): never => { + if (error instanceof SkyflowError) { + const errorInfo = error.error; + const message = error.message; + + // Build detailed error message + const errorParts: string[] = ['Skyflow API Error:']; + + if (errorInfo?.http_code) { + errorParts.push(` HTTP Code: ${errorInfo.http_code}`); + } + + if (message) { + errorParts.push(` Message: ${message}`); + } + + if (errorInfo?.details && Array.isArray(errorInfo.details)) { + errorParts.push(` Details: ${errorInfo.details.join(', ')}`); + } + + if (errorInfo?.request_ID) { + errorParts.push(` Request ID: ${errorInfo.request_ID}`); + } + + logError(errorParts.join('\n')); + process.exit(1); + } else if (error instanceof Error) { + logError(`Error: ${error.message}`); + process.exit(1); + } else { + logError(`Unknown error: ${String(error)}`); + process.exit(1); + } +}; + +/** + * Get vault ID from options or environment + */ +export const resolveVaultId = (vaultId?: string): string => { + if (vaultId) { + return vaultId; + } + + const envVaultId = process.env.SKYFLOW_VAULT_ID; + if (envVaultId) { + return envVaultId; + } + + throw new Error( + 'Vault ID not provided. Please provide --vault-id or set SKYFLOW_VAULT_ID environment variable' + ); +}; From c01fabff0cfed8dfbc7535b9acf191dbc6000319 Mon Sep 17 00:00:00 2001 From: Joseph McCarron Date: Tue, 7 Oct 2025 15:03:45 -0500 Subject: [PATCH 04/17] docs revamp part 1 --- .gitignore | 3 +- README.md | 270 +++++++++++++++++- .../API_DOCUMENTATION.md | 0 3 files changed, 271 insertions(+), 2 deletions(-) rename API_DOCUMENTATION.md => docs/API_DOCUMENTATION.md (100%) diff --git a/.gitignore b/.gitignore index bb38019..3b0787e 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,8 @@ node_modules node_modules/ dist +dist/ dist/* .clinerules .DS_Store -package-lock.json \ No newline at end of file +package-lock.json diff --git a/README.md b/README.md index 047fa92..d07e601 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,9 @@ A command-line interface for interacting with the Skyflow data privacy platform. - Create and manage vaults - Create and manage service accounts - Assign roles to service accounts +- Insert sensitive data into vault tables +- Detect and redact sensitive data (PII/PHI) using Skyflow Detect API +- Restore original values from tokenized text - Interactive prompts for missing options ## Installation @@ -34,7 +37,11 @@ npm i -g . ## Configuration -Before using the CLI, you need to configure it with your Skyflow credentials: +Before using the CLI, you need to configure it with your Skyflow credentials. + +### For Vault Management Commands + +For commands like `create-vault`, run: ```bash sky configure @@ -54,6 +61,43 @@ export SKYFLOW_ACCOUNT_ID="your-account-id" export SKYFLOW_WORKSPACE_ID="your-workspace-id" ``` +### For Data Operations Commands (Insert, Deidentify, Reidentify) + +These commands use the Skyflow Node.js SDK and support multiple authentication methods (in priority order): + +1. **API Key** (Recommended): + + ```bash + export SKYFLOW_API_KEY="your-api-key" + ``` + +2. **Service Account Credentials File**: + + ```bash + export SKYFLOW_CREDENTIALS_PATH="/path/to/credentials.json" + ``` + +3. **Service Account Credentials JSON String**: + + ```bash + export SKYFLOW_CREDENTIALS='{"clientID":"...","clientName":"...","tokenURI":"...","keyID":"...","privateKey":"..."}' + ``` + +4. **Bearer Token** (from `sky configure`): + + ```bash + # Uses the bearer token stored in ~/.skyflow/config.json + ``` + +Additionally, set these environment variables: + +```bash +export SKYFLOW_VAULT_ID="your-vault-id" +export SKYFLOW_VAULT_URL="https://your-cluster.vault.skyflowapis.com" +``` + +Or provide them as command-line options (`--vault-id`, `--cluster-id`). + ## Usage ### Creating a Vault @@ -107,8 +151,232 @@ sky create-vault ![prompts](assets/prompts.png) +### Inserting Data into a Vault + +Insert sensitive data into a vault table and optionally receive tokens: + +```bash +sky insert --table credit_cards --data '{"card_number":"4111111111111111","cvv":"123"}' --return-tokens +``` + +#### Insert Command Options + +- `--table `: Table name to insert data into (required) +- `--data `: JSON data to insert (can also pipe from stdin) +- `--return-tokens`: Return tokens for inserted data (default: false) +- `--continue-on-error`: Continue if some records fail (default: false) +- `--upsert-column `: Column name for upsert operations +- `--vault-id `: Vault ID (or set SKYFLOW_VAULT_ID) +- `--cluster-id `: Cluster ID (or set SKYFLOW_VAULT_URL) +- `--environment `: Environment: PROD, SANDBOX, STAGE, DEV (default: PROD) +- `--verbose`: Enable detailed logging + +#### Insert Command Examples + +Insert a single record: + +```bash +sky insert --table users --data '{"email":"user@example.com","ssn":"123-45-6789"}' +``` + +Insert multiple records: + +```bash +sky insert --table users --data '[{"email":"user1@example.com"},{"email":"user2@example.com"}]' +``` + +Insert from stdin: + +```bash +echo '{"card_number":"4111111111111111"}' | sky insert --table credit_cards --return-tokens +``` + +Upsert based on email column: + +```bash +sky insert --table users --data '{"email":"user@example.com","name":"John Doe"}' --upsert-column email +``` + +### Deidentifying Sensitive Data + +Detect and redact sensitive information (PII/PHI) from text using Skyflow Detect API: + +```bash +sky deidentify --text "My SSN is 123-45-6789 and my email is user@example.com" +``` + +#### Deidentify Command Options + +- `--text `: Text to deidentify (can also pipe from stdin) +- `--entities `: Comma-separated entity types to detect (e.g., SSN,CREDIT_CARD,EMAIL) +- `--token-type `: Token format - vault_token (stored), entity_only (labels), entity_unique_counter (default: vault_token) +- `--output `: Output format - text or json (default: text) +- `--vault-id `: Vault ID (or set SKYFLOW_VAULT_ID) +- `--cluster-id `: Cluster ID (or set SKYFLOW_VAULT_URL) +- `--environment `: Environment: PROD, SANDBOX, STAGE, DEV (default: PROD) +- `--verbose`: Enable detailed logging + +#### Supported Entity Types + +SSN, CREDIT_CARD, EMAIL, PHONE_NUMBER, NAME, DOB, ACCOUNT_NUMBER, DRIVER_LICENSE, PASSPORT_NUMBER, and many more. + +#### Deidentify Command Examples + +Detect all default entities (SSN, CREDIT_CARD, EMAIL, PHONE_NUMBER, NAME, DOB): + +```bash +sky deidentify --text "My SSN is 123-45-6789 and card is 4111-1111-1111-1111" +``` + +Detect specific entities only: + +```bash +sky deidentify --text "Contact me at user@example.com or 555-1234" --entities EMAIL,PHONE_NUMBER +``` + +Use entity-only tokens (no vault storage): + +```bash +sky deidentify --text "My SSN is 123-45-6789" --token-type entity_only +``` + +Pipe from file or command: + +```bash +cat sensitive_data.txt | sky deidentify --entities SSN,EMAIL,CREDIT_CARD +``` + +JSON output for programmatic use: + +```bash +sky deidentify --text "SSN: 123-45-6789" --output json +``` + +### Reidentifying Tokenized Data + +Restore original values from tokenized text: + +```bash +sky reidentify --text "My SSN is [SSN_abc123] and card is [CREDIT_CARD_xyz789]" +``` + +#### Reidentify Command Options + +- `--text `: Tokenized text to reidentify (can also pipe from stdin) +- `--plain-text `: Comma-separated entities to return as plain text +- `--masked `: Comma-separated entities to return masked (e.g., XXX-XX-1234) +- `--redacted `: Comma-separated entities to keep redacted +- `--output `: Output format - text or json (default: text) +- `--vault-id `: Vault ID (or set SKYFLOW_VAULT_ID) +- `--cluster-id `: Cluster ID (or set SKYFLOW_VAULT_URL) +- `--environment `: Environment: PROD, SANDBOX, STAGE, DEV (default: PROD) +- `--verbose`: Enable detailed logging + +#### Reidentify Command Examples + +Reidentify all tokens as plain text: + +```bash +sky reidentify --text "SSN: [SSN_abc123], Card: [CREDIT_CARD_xyz789]" +``` + +Return SSN as plain text, credit card as masked: + +```bash +sky reidentify --text "SSN: [SSN_abc123], Card: [CREDIT_CARD_xyz789]" --plain-text SSN --masked CREDIT_CARD +``` + +Pipe from file: + +```bash +cat tokenized_data.txt | sky reidentify --plain-text SSN,EMAIL +``` + +JSON output: + +```bash +sky reidentify --text "[SSN_token] data" --output json +``` + ## Output +### Insert Command Output + +The insert command displays: + +- Number of records successfully inserted +- Skyflow IDs for inserted records +- Tokens for each field (if `--return-tokens` is used) +- Any errors that occurred during insertion + +Example output: + +```text +Insert completed successfully! + +Inserted records: + +Record 1: + Skyflow ID: a8f3ed5d-55eb-4f32-bf7e-2dbf4b9d9097 + card_number: 5484-7829-1702-9110 + cardholder_name: b2308e2a-c1f5-469b-97b7-1f193159399b + +Total records processed: 1 +Successful: 1 +Failed: 0 +``` + +### Deidentify Command Output + +The deidentify command displays: + +- Processed text with sensitive data replaced by tokens +- Details about each detected entity (type, original value, token, position, confidence) +- Word and character counts + +Example output: + +```text +Deidentified Text: +──────────────────────────────────────────────────────────── +My SSN is [SSN_0ykQWPA] and my card is [CREDIT_CARD_N92QAVa]. +──────────────────────────────────────────────────────────── + +Detected 2 sensitive entities: + +1. SSN + Original: "123-45-6789" + Token: SSN_0ykQWPA + Position: 10-21 + Confidence: 93.8% + +2. CREDIT_CARD + Original: "4111 1111 1111 1111" + Token: CREDIT_CARD_N92QAVa + Position: 37-56 + Confidence: 90.5% + +Word count: 9 +Character count: 57 +``` + +### Reidentify Command Output + +The reidentify command displays the text with tokens replaced by original values (or masked/redacted based on options): + +Example output: + +```text +Reidentified Text: +──────────────────────────────────────────────────────────── +My SSN is 123-45-6789 and my card is 4111 1111 1111 1111. +──────────────────────────────────────────────────────────── + +Original sensitive data has been restored. +``` + +### Vault Creation Output + Upon successful vault creation, the CLI will output: - Vault details (name, description, ID, etc.) diff --git a/API_DOCUMENTATION.md b/docs/API_DOCUMENTATION.md similarity index 100% rename from API_DOCUMENTATION.md rename to docs/API_DOCUMENTATION.md From 5fbad5c30b566c81c87201d9c46afe5795935142 Mon Sep 17 00:00:00 2001 From: "claude[bot]" <209825114+claude[bot]@users.noreply.github.com> Date: Tue, 7 Oct 2025 20:06:56 +0000 Subject: [PATCH 05/17] refactor: extract duplicated code to shared utilities and fix syntax error MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fix missing closing brace in ReidentifyCommandOptions interface (src/types.ts:80) - Extract duplicated readStdin() function to src/utils/input.ts - Extract duplicated ENTITY_MAP constants to src/utils/entities.ts - Update all command files to use shared utilities - Remove 55+ lines of code duplication across commands 🤖 Generated with [Claude Code](https://claude.ai/code) Co-authored-by: Joseph McCarron --- src/commands/deidentify.ts | 42 ++------------------------------------ src/commands/insert.ts | 22 +------------------- src/commands/reidentify.ts | 42 ++------------------------------------ src/types.ts | 2 ++ src/utils/entities.ts | 30 +++++++++++++++++++++++++++ src/utils/input.ts | 26 +++++++++++++++++++++++ 6 files changed, 63 insertions(+), 101 deletions(-) create mode 100644 src/utils/entities.ts create mode 100644 src/utils/input.ts diff --git a/src/commands/deidentify.ts b/src/commands/deidentify.ts index 7a750af..f2466ab 100644 --- a/src/commands/deidentify.ts +++ b/src/commands/deidentify.ts @@ -15,26 +15,9 @@ import { resolveVaultId, } from '../utils/skyflow'; import { logVerbose } from '../utils/logger'; +import { readStdin } from '../utils/input'; +import { ENTITY_MAP, AVAILABLE_ENTITIES } from '../utils/entities'; -// Map of entity aliases to DetectEntities enum values -const ENTITY_MAP: Record = { - SSN: DetectEntities.SSN, - CREDIT_CARD: DetectEntities.CREDIT_CARD, - CREDIT_CARD_NUMBER: DetectEntities.CREDIT_CARD, - EMAIL: DetectEntities.EMAIL_ADDRESS, - EMAIL_ADDRESS: DetectEntities.EMAIL_ADDRESS, - PHONE_NUMBER: DetectEntities.PHONE_NUMBER, - PHONE: DetectEntities.PHONE_NUMBER, - NAME: DetectEntities.NAME, - DOB: DetectEntities.DOB, - DATE_OF_BIRTH: DetectEntities.DOB, - ACCOUNT_NUMBER: DetectEntities.ACCOUNT_NUMBER, - DRIVER_LICENSE: DetectEntities.DRIVER_LICENSE, - PASSPORT_NUMBER: DetectEntities.PASSPORT_NUMBER, - PASSPORT: DetectEntities.PASSPORT_NUMBER, -}; - -const AVAILABLE_ENTITIES = Object.keys(ENTITY_MAP).join(', '); export const deidentifyCommand = (program: Command): void => { program @@ -197,24 +180,3 @@ export const deidentifyCommand = (program: Command): void => { }); }; -/** - * Read input from stdin - */ -const readStdin = (): Promise => { - return new Promise((resolve, reject) => { - let data = ''; - process.stdin.setEncoding('utf8'); - - process.stdin.on('data', (chunk) => { - data += chunk; - }); - - process.stdin.on('end', () => { - resolve(data.trim()); - }); - - process.stdin.on('error', (error) => { - reject(error); - }); - }); -}; diff --git a/src/commands/insert.ts b/src/commands/insert.ts index ab75657..8b4e4d2 100644 --- a/src/commands/insert.ts +++ b/src/commands/insert.ts @@ -8,6 +8,7 @@ import { resolveVaultId, } from '../utils/skyflow'; import { logVerbose } from '../utils/logger'; +import { readStdin } from '../utils/input'; export const insertCommand = (program: Command): void => { program @@ -154,24 +155,3 @@ export const insertCommand = (program: Command): void => { }); }; -/** - * Read input from stdin - */ -const readStdin = (): Promise => { - return new Promise((resolve, reject) => { - let data = ''; - process.stdin.setEncoding('utf8'); - - process.stdin.on('data', (chunk) => { - data += chunk; - }); - - process.stdin.on('end', () => { - resolve(data.trim()); - }); - - process.stdin.on('error', (error) => { - reject(error); - }); - }); -}; diff --git a/src/commands/reidentify.ts b/src/commands/reidentify.ts index 3cc6f31..cb8d389 100644 --- a/src/commands/reidentify.ts +++ b/src/commands/reidentify.ts @@ -13,26 +13,9 @@ import { resolveVaultId, } from '../utils/skyflow'; import { logVerbose } from '../utils/logger'; +import { readStdin } from '../utils/input'; +import { ENTITY_MAP, AVAILABLE_ENTITIES } from '../utils/entities'; -// Map of entity aliases to DetectEntities enum values -const ENTITY_MAP: Record = { - SSN: DetectEntities.SSN, - CREDIT_CARD: DetectEntities.CREDIT_CARD, - CREDIT_CARD_NUMBER: DetectEntities.CREDIT_CARD, - EMAIL: DetectEntities.EMAIL_ADDRESS, - EMAIL_ADDRESS: DetectEntities.EMAIL_ADDRESS, - PHONE_NUMBER: DetectEntities.PHONE_NUMBER, - PHONE: DetectEntities.PHONE_NUMBER, - NAME: DetectEntities.NAME, - DOB: DetectEntities.DOB, - DATE_OF_BIRTH: DetectEntities.DOB, - ACCOUNT_NUMBER: DetectEntities.ACCOUNT_NUMBER, - DRIVER_LICENSE: DetectEntities.DRIVER_LICENSE, - PASSPORT_NUMBER: DetectEntities.PASSPORT_NUMBER, - PASSPORT: DetectEntities.PASSPORT_NUMBER, -}; - -const AVAILABLE_ENTITIES = Object.keys(ENTITY_MAP).join(', '); export const reidentifyCommand = (program: Command): void => { program @@ -167,24 +150,3 @@ export const reidentifyCommand = (program: Command): void => { }); }; -/** - * Read input from stdin - */ -const readStdin = (): Promise => { - return new Promise((resolve, reject) => { - let data = ''; - process.stdin.setEncoding('utf8'); - - process.stdin.on('data', (chunk) => { - data += chunk; - }); - - process.stdin.on('end', () => { - resolve(data.trim()); - }); - - process.stdin.on('error', (error) => { - reject(error); - }); - }); -}; diff --git a/src/types.ts b/src/types.ts index 6231290..a9eba9c 100644 --- a/src/types.ts +++ b/src/types.ts @@ -77,6 +77,8 @@ export interface ReidentifyCommandOptions { vaultId?: string; clusterId?: string; environment?: string; +} + export interface FunctionInfo { deploymentID: string; method: string; diff --git a/src/utils/entities.ts b/src/utils/entities.ts new file mode 100644 index 0000000..b1fe81d --- /dev/null +++ b/src/utils/entities.ts @@ -0,0 +1,30 @@ +/** + * Shared entity mapping constants for deidentify and reidentify commands + */ + +import { DetectEntities } from 'skyflow-node'; + +/** + * Map of entity aliases to DetectEntities enum values + */ +export const ENTITY_MAP: Record = { + SSN: DetectEntities.SSN, + CREDIT_CARD: DetectEntities.CREDIT_CARD, + CREDIT_CARD_NUMBER: DetectEntities.CREDIT_CARD, + EMAIL: DetectEntities.EMAIL_ADDRESS, + EMAIL_ADDRESS: DetectEntities.EMAIL_ADDRESS, + PHONE_NUMBER: DetectEntities.PHONE_NUMBER, + PHONE: DetectEntities.PHONE_NUMBER, + NAME: DetectEntities.NAME, + DOB: DetectEntities.DOB, + DATE_OF_BIRTH: DetectEntities.DOB, + ACCOUNT_NUMBER: DetectEntities.ACCOUNT_NUMBER, + DRIVER_LICENSE: DetectEntities.DRIVER_LICENSE, + PASSPORT_NUMBER: DetectEntities.PASSPORT_NUMBER, + PASSPORT: DetectEntities.PASSPORT_NUMBER, +}; + +/** + * Comma-separated list of available entity types + */ +export const AVAILABLE_ENTITIES = Object.keys(ENTITY_MAP).join(', '); \ No newline at end of file diff --git a/src/utils/input.ts b/src/utils/input.ts new file mode 100644 index 0000000..76ce218 --- /dev/null +++ b/src/utils/input.ts @@ -0,0 +1,26 @@ +/** + * Utilities for handling user input from various sources + */ + +/** + * Read input from stdin + * @returns Promise that resolves with the stdin content as a string + */ +export const readStdin = (): Promise => { + return new Promise((resolve, reject) => { + let data = ''; + process.stdin.setEncoding('utf8'); + + process.stdin.on('data', (chunk) => { + data += chunk; + }); + + process.stdin.on('end', () => { + resolve(data.trim()); + }); + + process.stdin.on('error', (error) => { + reject(error); + }); + }); +}; \ No newline at end of file From 2af68971efe46071ef2d0f8aea901b8ffb532750 Mon Sep 17 00:00:00 2001 From: Joseph McCarron Date: Tue, 7 Oct 2025 15:30:54 -0500 Subject: [PATCH 06/17] claude settings update, readme update --- .claude/settings.json | 10 +- docs/unit-testing-plan.md | 581 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 590 insertions(+), 1 deletion(-) create mode 100644 docs/unit-testing-plan.md diff --git a/.claude/settings.json b/.claude/settings.json index e8f289d..a28c0cf 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -1,3 +1,11 @@ { - "enableAllProjectMcpServers": false + "enableAllProjectMcpServers": false, + "permissions": { + "allow": [ + "Bash(npm run lint)", + "Bash(npm run build)", + "Bash(npm run test)", + "Bash(npm run test:*)" + ] + } } \ No newline at end of file diff --git a/docs/unit-testing-plan.md b/docs/unit-testing-plan.md new file mode 100644 index 0000000..16752da --- /dev/null +++ b/docs/unit-testing-plan.md @@ -0,0 +1,581 @@ +# Unit Testing Implementation Plan + +## Overview + +This document outlines a comprehensive unit testing strategy for the Skyflow CLI project. The project currently has no existing tests, so we'll establish a testing framework from scratch. + +## Testing Stack Recommendation + +### Core Testing Tools + +1. **Jest** - Testing framework + - Industry standard for TypeScript/Node.js projects + - Built-in mocking, assertions, coverage reporting + - Great TypeScript support + +2. **ts-jest** - TypeScript preprocessor for Jest + - Allows running tests directly on TypeScript files + - Integrates seamlessly with Jest + +3. **@types/jest** - TypeScript type definitions for Jest + +4. **nock** - HTTP mocking library + - Mock external API calls (Skyflow APIs) + - No actual network requests during tests + +### Installation Commands + +```bash +npm install --save-dev jest ts-jest @types/jest nock @types/nock +``` + +## Project Structure + +``` +sky_cli/ +├── src/ +│ ├── commands/ +│ ├── utils/ +│ └── types.ts +├── tests/ # NEW +│ ├── unit/ +│ │ ├── commands/ +│ │ │ ├── configure.test.ts +│ │ │ ├── createVault.test.ts +│ │ │ ├── createConnection.test.ts +│ │ │ ├── insert.test.ts +│ │ │ ├── deidentify.test.ts +│ │ │ └── reidentify.test.ts +│ │ └── utils/ +│ │ ├── api.test.ts +│ │ ├── config.test.ts +│ │ ├── skyflow.test.ts +│ │ ├── entities.test.ts +│ │ ├── input.test.ts +│ │ ├── logger.test.ts +│ │ └── prompts.test.ts +│ ├── integration/ # FUTURE +│ │ └── e2e.test.ts +│ └── fixtures/ +│ ├── connection-config.json +│ ├── vault-schema.json +│ └── sample-data.json +├── jest.config.js # NEW +└── package.json +``` + +## Phase 1: Setup & Configuration + +### 1.1 Jest Configuration + +Create `jest.config.js`: + +```javascript +module.exports = { + preset: 'ts-jest', + testEnvironment: 'node', + roots: ['/tests'], + testMatch: ['**/*.test.ts'], + collectCoverageFrom: [ + 'src/**/*.ts', + '!src/index.ts', + '!src/types.ts', + ], + coverageDirectory: 'coverage', + coverageReporters: ['text', 'lcov', 'html'], + coverageThreshold: { + global: { + branches: 70, + functions: 70, + lines: 70, + statements: 70, + }, + }, + moduleNameMapper: { + '^@/(.*)$': '/src/$1', + }, + setupFilesAfterEnv: ['/tests/setup.ts'], + verbose: true, +}; +``` + +### 1.2 Test Setup File + +Create `tests/setup.ts`: + +```typescript +// Global test setup +beforeAll(() => { + // Suppress console logs during tests + global.console = { + ...console, + log: jest.fn(), + error: jest.fn(), + warn: jest.fn(), + }; +}); + +afterAll(() => { + jest.restoreAllMocks(); +}); +``` + +### 1.3 Update package.json Scripts + +Add to `scripts` section: + +```json +{ + "test": "jest", + "test:watch": "jest --watch", + "test:coverage": "jest --coverage", + "test:unit": "jest tests/unit", + "test:integration": "jest tests/integration" +} +``` + +## Phase 2: Utility Tests (Foundation) + +Start with utility functions as they're the foundation for command tests. + +### 2.1 Logger Tests (`tests/unit/utils/logger.test.ts`) + +**Priority: HIGH** + +Test cases: +- `setVerbose()` correctly sets verbose mode +- `isVerboseMode()` returns correct state +- `verboseLog()` logs when verbose is true +- `verboseLog()` doesn't log when verbose is false +- `errorLog()` always logs errors +- `logVerbose` and `logError` aliases work correctly + +### 2.2 Config Tests (`tests/unit/utils/config.test.ts`) + +**Priority: HIGH** + +Test cases: +- `loadConfig()` reads from environment variables first +- `loadConfig()` reads from config file if no env vars +- `loadConfig()` throws error if no config found +- `loadConfig()` throws error if config is invalid +- `saveConfig()` creates config directory if missing +- `saveConfig()` writes config file correctly +- `saveConfig()` calls configure() with credentials + +Mock requirements: +- File system operations (`fs.existsSync`, `fs.readFileSync`, `fs.writeFileSync`) +- Environment variables + +### 2.3 Entities Tests (`tests/unit/utils/entities.test.ts`) + +**Priority: MEDIUM** + +Test cases: +- `ENTITY_MAP` contains all expected entity types +- `ENTITY_MAP` maps aliases correctly (CREDIT_CARD → DetectEntities.CREDIT_CARD) +- `AVAILABLE_ENTITIES` contains comma-separated string of all keys + +### 2.4 Input Tests (`tests/unit/utils/input.test.ts`) + +**Priority: MEDIUM** + +Test cases: +- `readStdin()` reads data from stdin +- `readStdin()` trims whitespace +- `readStdin()` handles errors correctly +- `readStdin()` resolves with complete data + +Mock requirements: +- `process.stdin` events (data, end, error) + +### 2.5 Skyflow SDK Utilities Tests (`tests/unit/utils/skyflow.test.ts`) + +**Priority: HIGH** + +Test cases: +- `loadSkyflowCredentials()` returns API key from env +- `loadSkyflowCredentials()` returns credentials path from env +- `loadSkyflowCredentials()` returns credentials string from env +- `loadSkyflowCredentials()` falls back to bearer token +- `loadSkyflowCredentials()` throws error if no credentials +- `extractClusterIdFromUrl()` extracts cluster ID correctly +- `extractClusterIdFromUrl()` throws error for invalid URL +- `parseEnvironment()` maps PROD/PRODUCTION to Env.PROD +- `parseEnvironment()` maps SANDBOX to Env.SANDBOX +- `parseEnvironment()` defaults to PROD for unknown values +- `initializeSkyflowClient()` creates client with correct config +- `handleSkyflowError()` formats SkyflowError correctly +- `handleSkyflowError()` handles generic Error +- `resolveVaultId()` returns provided vault ID +- `resolveVaultId()` returns env var if no param +- `resolveVaultId()` throws error if none provided + +Mock requirements: +- Environment variables +- `skyflow-node` module +- `loadConfig()` function + +### 2.6 API Tests (`tests/unit/utils/api.test.ts`) + +**Priority: HIGH** + +Test cases: +- `configure()` sets axios defaults correctly +- `createVault()` makes POST request with correct payload +- `createServiceAccount()` makes POST request +- `assignRole()` makes POST request with role assignment +- `createConnection()` makes POST request with connection data +- API functions handle errors correctly +- API functions include verbose logging + +Mock requirements: +- `axios` HTTP client (use `nock` or `jest.mock('axios')`) + +### 2.7 Prompts Tests (`tests/unit/utils/prompts.test.ts`) + +**Priority: LOW** + +Test cases: +- `promptForVaultId()` prompts user for vault ID +- `promptForVaultId()` validates non-empty input +- Other prompt functions work correctly + +Mock requirements: +- `inquirer` module + +## Phase 3: Command Tests + +Test each command's business logic independently of Commander.js. + +### 3.1 Configure Command Tests (`tests/unit/commands/configure.test.ts`) + +**Priority: HIGH** + +Test cases: +- Prompts user for bearer token, account ID, workspace ID +- Validates required fields +- Saves configuration successfully +- Displays success message +- Handles errors gracefully + +Mock requirements: +- `inquirer.prompt()` +- `saveConfig()` + +### 3.2 Create Vault Command Tests (`tests/unit/commands/createVault.test.ts`) + +**Priority: HIGH** + +Test cases: +- Creates vault with provided options +- Prompts for missing options +- Creates service account when flag is true +- Assigns VAULT_OWNER role to service account +- Displays output correctly +- Handles vault creation errors +- Handles service account creation errors +- Validates schema file exists before reading +- Parses schema file correctly + +Mock requirements: +- `loadConfig()` +- `createVault()` +- `createServiceAccount()` +- `assignRole()` +- File system operations + +### 3.3 Create Connection Command Tests (`tests/unit/commands/createConnection.test.ts`) + +**Priority: HIGH** + +Test cases: +- Validates file path is provided +- Validates file exists +- Parses connection config file (array format) +- Parses connection config file (object with connections property) +- Validates required connection fields (name, vaultID, routes) +- Creates multiple connections from config +- Continues on error when one connection fails +- Displays summary with success/failure counts +- Exits with error code if any connection fails +- Uses vault ID from option/env/prompt + +Mock requirements: +- File system operations +- `createConnection()` +- `promptForVaultId()` + +### 3.4 Insert Command Tests (`tests/unit/commands/insert.test.ts`) + +**Priority: HIGH** + +Test cases: +- Prompts for table name if not provided +- Parses JSON data from `--data` option +- Reads data from stdin when piped +- Converts single object to array +- Validates JSON format +- Initializes Skyflow client correctly +- Creates InsertRequest with correct parameters +- Configures InsertOptions based on flags (returnTokens, continueOnError, upsertColumn) +- Executes insert operation +- Displays results (records, tokens, errors) +- Handles insert errors gracefully + +Mock requirements: +- `inquirer.prompt()` +- `initializeSkyflowClient()` +- `resolveVaultId()` +- `readStdin()` +- Skyflow SDK (InsertRequest, InsertOptions) +- `process.stdin.isTTY` + +### 3.5 Deidentify Command Tests (`tests/unit/commands/deidentify.test.ts`) + +**Priority: HIGH** + +Test cases: +- Reads text from `--text` option +- Reads text from stdin when piped +- Prompts for text if not provided +- Parses entity list from `--entities` option +- Uses default entities when not specified +- Maps entity aliases correctly (EMAIL → EMAIL_ADDRESS) +- Throws error for unknown entity types +- Parses token type correctly (vault_token, entity_only, entity_unique_counter) +- Defaults to vault_token +- Initializes Skyflow client correctly +- Creates DeidentifyTextRequest +- Configures DeidentifyTextOptions with entities and token format +- Executes deidentify operation +- Displays text output with entity details +- Displays JSON output when `--output json` +- Handles deidentify errors gracefully + +Mock requirements: +- `inquirer.prompt()` +- `initializeSkyflowClient()` +- `resolveVaultId()` +- `readStdin()` +- Skyflow SDK (DeidentifyTextRequest, DeidentifyTextOptions) +- `process.stdin.isTTY` + +### 3.6 Reidentify Command Tests (`tests/unit/commands/reidentify.test.ts`) + +**Priority: HIGH** + +Test cases: +- Reads text from `--text` option +- Reads text from stdin when piped +- Prompts for text if not provided +- Parses plain-text entities from option +- Parses masked entities from option +- Parses redacted entities from option +- Maps entity aliases correctly +- Throws error for unknown entity types +- Defaults to plain text for all when no options specified +- Initializes Skyflow client correctly +- Creates ReidentifyTextRequest +- Configures ReidentifyTextOptions with entity display options +- Executes reidentify operation +- Displays text output +- Displays JSON output when `--output json` +- Handles reidentify errors gracefully + +Mock requirements: +- `inquirer.prompt()` +- `initializeSkyflowClient()` +- `resolveVaultId()` +- `readStdin()` +- Skyflow SDK (ReidentifyTextRequest, ReidentifyTextOptions) +- `process.stdin.isTTY` + +## Phase 4: Integration Tests (Future) + +### 4.1 End-to-End Command Tests + +Test commands with real Commander.js integration: +- Parse command-line arguments correctly +- Execute commands end-to-end +- Handle pre-action hooks (authentication) +- Handle unknown commands + +### 4.2 Full Workflow Tests + +Test complete workflows: +- Configure → Create Vault → Insert Data +- Deidentify → Reidentify round-trip +- Create Connection from config file + +## Testing Best Practices + +### 1. Test Isolation + +- Each test should be independent +- Use `beforeEach` and `afterEach` to reset state +- Mock external dependencies + +### 2. Mock Strategy + +```typescript +// Example: Mock Skyflow SDK +jest.mock('skyflow-node', () => ({ + Skyflow: jest.fn().mockImplementation(() => ({ + vault: jest.fn().mockReturnValue({ + insert: jest.fn().mockResolvedValue({ insertedFields: [] }), + }), + detect: jest.fn().mockReturnValue({ + deidentifyText: jest.fn().mockResolvedValue({ processedText: 'redacted' }), + reidentifyText: jest.fn().mockResolvedValue({ processedText: 'original' }), + }), + })), + Env: { PROD: 'PROD', SANDBOX: 'SANDBOX' }, + LogLevel: { ERROR: 'ERROR', INFO: 'INFO' }, + DetectEntities: { SSN: 'SSN', CREDIT_CARD: 'CREDIT_CARD' }, + TokenType: { VAULT_TOKEN: 'VAULT_TOKEN', ENTITY_ONLY: 'ENTITY_ONLY' }, +})); +``` + +### 3. Test Data + +- Store test fixtures in `tests/fixtures/` +- Use realistic but fake data (no real credentials) +- Examples: + - `connection-config.json` - Sample connection configurations + - `vault-schema.json` - Sample vault schema + - `sample-data.json` - Sample insert data + +### 4. Coverage Goals + +- **Utilities**: 90%+ coverage (critical infrastructure) +- **Commands**: 80%+ coverage (business logic) +- **Types**: No coverage needed (type definitions) + +### 5. Naming Conventions + +- Test files: `*.test.ts` +- Test suites: `describe('ComponentName', () => {})` +- Test cases: `it('should do something when condition', () => {})` +- Mock setup: `beforeEach(() => { /* setup mocks */ })` + +## Implementation Timeline + +### Week 1: Foundation (16-20 hours) +- ✅ Setup Jest and configuration files +- ✅ Write utility tests (logger, config, entities, input) +- ✅ Establish testing patterns and examples + +### Week 2: Core Commands (20-24 hours) +- ✅ Write tests for configure, createVault, createConnection commands +- ✅ Achieve 70%+ coverage on utils and core commands + +### Week 3: Skyflow SDK Commands (16-20 hours) +- ✅ Write tests for insert, deidentify, reidentify commands +- ✅ Write tests for skyflow.ts utilities +- ✅ Achieve 80%+ overall coverage + +### Week 4: Polish & Documentation (8-12 hours) +- ✅ Fix any failing tests +- ✅ Improve coverage for edge cases +- ✅ Document testing patterns for future contributors +- ✅ Add CI/CD integration (GitHub Actions) + +## CI/CD Integration + +### GitHub Actions Workflow + +Create `.github/workflows/test.yml`: + +```yaml +name: Tests + +on: + push: + branches: [main, develop] + pull_request: + branches: [main, develop] + +jobs: + test: + runs-on: ubuntu-latest + + strategy: + matrix: + node-version: [18.x, 20.x] + + steps: + - uses: actions/checkout@v3 + + - name: Use Node.js ${{ matrix.node-version }} + uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + + - name: Install dependencies + run: npm ci + + - name: Run tests + run: npm test -- --coverage + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + with: + files: ./coverage/lcov.info + flags: unittests +``` + +## Example Test File Template + +```typescript +import { functionToTest } from '@/utils/module'; + +describe('FunctionName', () => { + beforeEach(() => { + // Setup mocks + jest.clearAllMocks(); + }); + + afterEach(() => { + // Cleanup + jest.restoreAllMocks(); + }); + + describe('when condition A', () => { + it('should do X', () => { + // Arrange + const input = 'test'; + + // Act + const result = functionToTest(input); + + // Assert + expect(result).toBe('expected'); + }); + + it('should handle errors', () => { + // Arrange & Act & Assert + expect(() => functionToTest('')).toThrow('error message'); + }); + }); + + describe('when condition B', () => { + it('should do Y', async () => { + // Test async functions + const result = await asyncFunctionToTest(); + expect(result).toEqual({ key: 'value' }); + }); + }); +}); +``` + +## Summary + +This comprehensive testing plan provides: +- **Complete coverage** of all commands and utilities +- **Phased approach** from utilities → commands → integration +- **Best practices** for testing Node.js CLI applications +- **CI/CD integration** for automated testing +- **Clear timeline** for implementation (8-12 weeks part-time) + +The testing infrastructure will ensure code quality, prevent regressions, and make future development more confident and efficient. From 98766b2b0ce32c33956f118b99d66acf959b82c7 Mon Sep 17 00:00:00 2001 From: Joseph McCarron Date: Tue, 7 Oct 2025 15:38:59 -0500 Subject: [PATCH 07/17] now with unit testing foundations --- dist/commands/configure.js | 64 ---------- dist/commands/createVault.js | 183 --------------------------- dist/commands/deidentify.js | 189 ---------------------------- dist/commands/insert.js | 164 ------------------------- dist/commands/reidentify.js | 158 ------------------------ dist/index.js | 63 ---------- dist/types.js | 2 - dist/utils/api.js | 198 ------------------------------ dist/utils/config.js | 66 ---------- dist/utils/logger.js | 59 --------- dist/utils/prompts.js | 193 ----------------------------- dist/utils/skyflow.js | 162 ------------------------ jest.config.js | 26 ++++ package.json | 11 +- tests/setup.ts | 21 ++++ tests/unit/utils/entities.test.ts | 111 +++++++++++++++++ tests/unit/utils/input.test.ts | 120 ++++++++++++++++++ tests/unit/utils/logger.test.ts | 149 ++++++++++++++++++++++ 18 files changed, 437 insertions(+), 1502 deletions(-) delete mode 100644 dist/commands/configure.js delete mode 100644 dist/commands/createVault.js delete mode 100644 dist/commands/deidentify.js delete mode 100644 dist/commands/insert.js delete mode 100644 dist/commands/reidentify.js delete mode 100755 dist/index.js delete mode 100644 dist/types.js delete mode 100644 dist/utils/api.js delete mode 100644 dist/utils/config.js delete mode 100644 dist/utils/logger.js delete mode 100644 dist/utils/prompts.js delete mode 100644 dist/utils/skyflow.js create mode 100644 jest.config.js create mode 100644 tests/setup.ts create mode 100644 tests/unit/utils/entities.test.ts create mode 100644 tests/unit/utils/input.test.ts create mode 100644 tests/unit/utils/logger.test.ts diff --git a/dist/commands/configure.js b/dist/commands/configure.js deleted file mode 100644 index d5b3356..0000000 --- a/dist/commands/configure.js +++ /dev/null @@ -1,64 +0,0 @@ -"use strict"; -var __importDefault = (this && this.__importDefault) || function (mod) { - return (mod && mod.__esModule) ? mod : { "default": mod }; -}; -Object.defineProperty(exports, "__esModule", { value: true }); -exports.configureCommand = void 0; -const inquirer_1 = __importDefault(require("inquirer")); -const config_1 = require("../utils/config"); -const configureCommand = (program) => { - program - .command('configure') - .description('Configure Skyflow CLI authentication') - .action(async () => { - try { - console.log('Please provide your Skyflow API credentials:'); - const answers = await inquirer_1.default.prompt([ - { - type: 'password', - name: 'bearerToken', - message: 'Enter your Skyflow Bearer Token:', - validate: (input) => { - if (!input) - return 'Bearer token is required'; - return true; - } - }, - { - type: 'input', - name: 'accountId', - message: 'Enter your Skyflow Account ID:', - validate: (input) => { - if (!input) - return 'Account ID is required'; - return true; - } - }, - { - type: 'input', - name: 'workspaceID', - message: 'Enter your Skyflow Workspace ID:', - validate: (input) => { - if (!input) - return 'Workspace ID is required'; - return true; - } - } - ]); - // Save the configuration - (0, config_1.saveConfig)({ - bearerToken: answers.bearerToken, - accountId: answers.accountId, - workspaceID: answers.workspaceID - }); - console.log('\nConfiguration saved successfully!'); - console.log('\nYou can now use the Skyflow CLI. Try running:'); - console.log(' sky create-vault --help'); - } - catch (error) { - console.error(`Error: ${error instanceof Error ? error.message : String(error)}`); - process.exit(1); - } - }); -}; -exports.configureCommand = configureCommand; diff --git a/dist/commands/createVault.js b/dist/commands/createVault.js deleted file mode 100644 index 41573ce..0000000 --- a/dist/commands/createVault.js +++ /dev/null @@ -1,183 +0,0 @@ -"use strict"; -var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) { - if (k2 === undefined) k2 = k; - var desc = Object.getOwnPropertyDescriptor(m, k); - if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) { - desc = { enumerable: true, get: function() { return m[k]; } }; - } - Object.defineProperty(o, k2, desc); -}) : (function(o, m, k, k2) { - if (k2 === undefined) k2 = k; - o[k2] = m[k]; -})); -var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) { - Object.defineProperty(o, "default", { enumerable: true, value: v }); -}) : function(o, v) { - o["default"] = v; -}); -var __importStar = (this && this.__importStar) || (function () { - var ownKeys = function(o) { - ownKeys = Object.getOwnPropertyNames || function (o) { - var ar = []; - for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k; - return ar; - }; - return ownKeys(o); - }; - return function (mod) { - if (mod && mod.__esModule) return mod; - var result = {}; - if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]); - __setModuleDefault(result, mod); - return result; - }; -})(); -var __importDefault = (this && this.__importDefault) || function (mod) { - return (mod && mod.__esModule) ? mod : { "default": mod }; -}; -Object.defineProperty(exports, "__esModule", { value: true }); -exports.createVaultCommand = void 0; -const api = __importStar(require("../utils/api")); -const prompts = __importStar(require("../utils/prompts")); -const config_1 = require("../utils/config"); -const logger_1 = require("../utils/logger"); -const inquirer_1 = __importDefault(require("inquirer")); -const fs_1 = __importDefault(require("fs")); -// Add the create-vault command to the CLI -const createVaultCommand = (program) => { - program - .command('create-vault') - .description('Create a new Skyflow vault') - .option('--name ', 'Name for the vault (lowercase, no special characters)') - .option('--template