From 8117f1d9a296b8459f22af7e365a8931b0371a96 Mon Sep 17 00:00:00 2001 From: Bryan Larsen Date: Mon, 27 Jun 2016 17:13:19 -0400 Subject: [PATCH 1/5] Use server-sent-events to display progress bar --- lib/cli.js | 9 +++++++- lib/method.js | 50 +++++++++++++++++++++++++++++++++++++---- lib/resources/scenes.js | 6 +++-- package.json | 1 + 4 files changed, 59 insertions(+), 7 deletions(-) diff --git a/lib/cli.js b/lib/cli.js index a37a9e5..4b3c6e1 100644 --- a/lib/cli.js +++ b/lib/cli.js @@ -4,6 +4,7 @@ var mapIndexed = R.addIndex(R.map); var fs = require('fs'); var jsonQuery = require('json-query'); var log = require('npmlog'); +var Progress = require('progress'); var config = require('../lib/config'); var conf = config(); @@ -66,7 +67,7 @@ function buildCommand(info, key, section) { R.forEach(R.curry(addOption)(query), R.keys(query)); if (info.body) addOption(info, 'body'); - if (info.async) { + if (info.async || info.sse) { cmd.option('--url', 'Return the URL of the result'); } @@ -103,6 +104,12 @@ function buildCommand(info, key, section) { var resource = resources[section][key]; + var progress = new Progress(':status [:bar] :message', { total : 80, width: 40 }); + + opts.progressCallback = function(job) { + if (job.progress) progress.update(job.progress, job); + }; + claraApi[section][key](Object.assign({}, args, queryArgs, {url: cmd.url}), opts, function(err, result) { if (err) return fail(err, result); diff --git a/lib/method.js b/lib/method.js index d44682b..56e8584 100644 --- a/lib/method.js +++ b/lib/method.js @@ -76,7 +76,7 @@ module.exports = function(data, key) { queryArgs[key] = JSON.stringify(parseJSON(queryArgs[key])); }, data.jsonQueryKeys); var qs = R.keys(queryArgs).length ? '?'+queryString.stringify(queryArgs) : ''; - var onlyLocation = data.async && queryOptions.url; + var onlyLocation = (data.async || data.sse) && queryOptions.url; var opts; @@ -91,7 +91,6 @@ module.exports = function(data, key) { }, data.jsonKeys); } - var errors = []; function checkRequired(fromObj, obj, key) { if (fromObj[key].required && obj[key] === undefined) { @@ -164,7 +163,8 @@ module.exports = function(data, key) { } var req = superagent[data.method](url); - if (data.output === 'json') req.set('Accept', 'application/json') + if (data.output === 'json') req.set('Accept', 'application/json'); + if (data.sse) req.set('Accept', 'text/event-stream'); if (conf.get('apiToken')) req.auth(conf.get('username'), conf.get('apiToken')); R.forEach(function(fileKey) { @@ -179,7 +179,49 @@ module.exports = function(data, key) { if (data.method !== 'get') { log.debug('sending: ', JSON.stringify(opts, null, ' ')); - req.send(opts) + req.send(opts); + } + + + if (data.sse) { + var stream = require('stream').Writable(); + var buf = ''; + var job = { status: 'starting', message: '', progress: 0}; + stream._write = function(chunk, enc, next) { + buf = buf + chunk.toString('utf8'); + var lines = buf.split('\n'); + buf = lines[lines.length-1]; + for (var i = 0; i < lines.length-1; i++) { + var line = lines[i]; + if (line === '') continue; + var match = line.match(/^data: (.*)$/); + if (!match) { + log.debug('unmatched line', line); + continue; + } + job = R.merge(job, JSON.parse(match[1])); + if (opts.progressCallback) opts.progressCallback(job); + } + + next(); + }; + + stream.on('error', function(err) { + done(err); + }); + + stream.on('finish', function() { + if (!job.files || !job.files.length) return done(null); + if (onlyLocation) { + callback(null, job.files[0].url); + return resolve(job.files[0].url); + } + var req2 = superagent.get(job.files[0].url); + req2.end(done); + }); + + req.pipe(stream); + return; } req.end(function(err, res) { diff --git a/lib/resources/scenes.js b/lib/resources/scenes.js index 2bbfafe..0235de2 100644 --- a/lib/resources/scenes.js +++ b/lib/resources/scenes.js @@ -114,7 +114,8 @@ module.exports = { description: 'Render an image', method: 'post', path: '/scenes/{sceneId}/render', - async: true, + async: false, + sse: true, query: { time: {type: Number, description: 'Frame number to render'}, width: {type: Number, description: 'Width in pixels of the desired image'}, @@ -129,7 +130,8 @@ module.exports = { }, options: { setupCommand: {type: String, as: 'string', description: 'Command to be executed before render'}, - data: {type: 'json', as: 'filename', description: 'Optional data for setupCommand'} + data: {type: 'json', as: 'filename', description: 'Optional data for setupCommand'}, + progressCallback: {type: Function, description: "Called regularly with job information"} }, output: 'binary' }, diff --git a/package.json b/package.json index b591ec1..1b80926 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,7 @@ "md5-file": "^2.0.4", "npmlog": "^2.0.0", "osenv": "^0.1.3", + "progress": "git+https://github.com/visionmedia/node-progress.git#d47913502ba5b551fcaad9e94fe7b2f5876a7939", "query-string": "^3.0.0", "ramda": "^0.18.0", "superagent": "^1.5.0", From ffc0d05a1802815e995070a9b1c9b719f62b8e8b Mon Sep 17 00:00:00 2001 From: Bryan Larsen Date: Tue, 28 Jun 2016 10:59:56 -0400 Subject: [PATCH 2/5] add event-stream based progress bar to import and export too --- lib/cli.js | 2 +- lib/method.js | 8 ++++++-- lib/resources/scenes.js | 8 +++++++- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/lib/cli.js b/lib/cli.js index 4b3c6e1..2ccbbf6 100644 --- a/lib/cli.js +++ b/lib/cli.js @@ -130,7 +130,7 @@ function buildCommand(info, key, section) { if (program.output) { fs.writeFileSync(program.output, output, resource.isBinary ? 'binary' : 'utf8'); - } else { + } else if (output) { process.stdout.write(output); } }).catch(fail); diff --git a/lib/method.js b/lib/method.js index 56e8584..7d19b5c 100644 --- a/lib/method.js +++ b/lib/method.js @@ -112,7 +112,10 @@ module.exports = function(data, key) { function done(err, res) { var result; - if (!data.isBinary) { + if (!res) { + callback(null, res); + resolve(res); + } else if (!data.isBinary) { result = data.isJSON ? res.body : res.text; callback(null, result); resolve(result); @@ -200,6 +203,7 @@ module.exports = function(data, key) { continue; } job = R.merge(job, JSON.parse(match[1])); + if (job.message === null) job.message = ''; if (opts.progressCallback) opts.progressCallback(job); } @@ -211,7 +215,7 @@ module.exports = function(data, key) { }); stream.on('finish', function() { - if (!job.files || !job.files.length) return done(null); + if (!job.files || !job.files.length || !job.files[0].url) return done(null); if (onlyLocation) { callback(null, job.files[0].url); return resolve(job.files[0].url); diff --git a/lib/resources/scenes.js b/lib/resources/scenes.js index 0235de2..beae0e8 100644 --- a/lib/resources/scenes.js +++ b/lib/resources/scenes.js @@ -70,12 +70,14 @@ module.exports = { description: 'Import files into the scene', method: 'post', path: '/scenes/{sceneId}/import', + sse: true, query: { async: {type: Boolean}, data: {type: 'json', as: 'filename', description: 'Optional data for import'}, fileIds: {type: Array, description: 'Array of existing file ids'} }, options: { + progressCallback: {type: Function, description: "Called regularly with job information"}, file: {type: 'File', description: 'A File', required: false}, files: {type: 'Files', description: 'An array of files ', required: false} } @@ -107,7 +109,11 @@ module.exports = { method: 'post', path: '/scenes/{sceneId}/export/{extension}', output: 'binary', - async: true + async: false, + sse: true, + options: { + progressCallback: {type: Function, description: "Called regularly with job information"} + } }, render: { From 49161b3ca770524f04c7a7f0c43de7882c22ff78 Mon Sep 17 00:00:00 2001 From: Bryan Larsen Date: Tue, 28 Jun 2016 11:00:17 -0400 Subject: [PATCH 3/5] docs of random stuff --- README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README.md b/README.md index 27af251..81ab5a1 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,7 @@ $ clara --apiToken --username scenes:get * scenes:delete Delete a scene * scenes:clone Clone a scene * scenes:import [options] Import a file into the scene + * scenes:importOptimized [options] Import a file into the scene, taking care to not reupload shared assets * scenes:export Export a scene * scenes:render [options] Render an image * scenes:command [options] Run a command @@ -130,7 +131,17 @@ $ clara set apiToken your-api-token $ clara set username your-username $ clara scenes:get scene-uuid ``` +### Configuration Variables +Available configuration and their defaults: + +``` + apiToken: null, + dryRun: false, + host: 'https://clara.io', + logLevel: 'info', // silly, verbose, info, warn, error, silent + username: null +``` ## Development From 1e992d58fee1c71043d92aa551bbc2eb9d2e7069 Mon Sep 17 00:00:00 2001 From: Bryan Larsen Date: Tue, 28 Jun 2016 12:39:00 -0400 Subject: [PATCH 4/5] bump version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 1b80926..9b19bb4 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "clara", - "version": "0.4.4", + "version": "0.4.5", "description": "Clara.io API wrapper", "main": "lib/index.js", "bin": "bin/clara.js", From ee9290ffaa7bc251cc6648815e6be0e62110f6b0 Mon Sep 17 00:00:00 2001 From: Bryan Larsen Date: Tue, 28 Jun 2016 13:23:26 -0400 Subject: [PATCH 5/5] error handling --- lib/method.js | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/lib/method.js b/lib/method.js index 7d19b5c..ea6db24 100644 --- a/lib/method.js +++ b/lib/method.js @@ -189,6 +189,7 @@ module.exports = function(data, key) { if (data.sse) { var stream = require('stream').Writable(); var buf = ''; + var failed = false; var job = { status: 'starting', message: '', progress: 0}; stream._write = function(chunk, enc, next) { buf = buf + chunk.toString('utf8'); @@ -211,10 +212,13 @@ module.exports = function(data, key) { }; stream.on('error', function(err) { + if (failed) return; done(err); + failed = true; }); stream.on('finish', function() { + if (failed) return; if (!job.files || !job.files.length || !job.files[0].url) return done(null); if (onlyLocation) { callback(null, job.files[0].url); @@ -224,6 +228,14 @@ module.exports = function(data, key) { req2.end(done); }); + // https://github.com/visionmedia/superagent/issues/565 + req.on('end', function(err, res) { + if (err || this.res.statusCode >= 400) { + fail(err || this.res.statusCode); + failed = true; + } + }); + req.pipe(stream); return; }