From 2bb22ee1d5a6e83ccc81bcea3742128c1853630f Mon Sep 17 00:00:00 2001 From: Joe Mancewicz Date: Mon, 24 Aug 2015 18:35:46 -0700 Subject: [PATCH] Adding Job Management UI --- .gitignore | 3 + digits/dataset/tasks/analyze_db.py | 11 +- digits/dataset/tasks/create_db.py | 11 +- digits/dataset/tasks/parse_folder.py | 11 +- digits/job.py | 40 ++++ digits/model/tasks/train.py | 26 +-- digits/scheduler.py | 54 ++++++ digits/static/css/style.css | 11 +- digits/static/js/jquery.sparkline.min.js | 5 + digits/static/js/jquery.time_filters.js | 32 ++++ digits/static/js/jquery.timer.min.js | 29 +++ digits/status.py | 2 + digits/task.py | 21 ++ digits/templates/home.html | 74 ++----- digits/templates/job_item.html | 9 + digits/templates/job_management.html | 233 +++++++++++++++++++++++ digits/templates/job_row.html | 59 ++++++ digits/templates/socketio.html | 2 +- digits/test_scheduler.py | 10 +- digits/views.py | 15 ++ 20 files changed, 550 insertions(+), 108 deletions(-) create mode 100644 digits/static/js/jquery.sparkline.min.js create mode 100644 digits/static/js/jquery.time_filters.js create mode 100644 digits/static/js/jquery.timer.min.js create mode 100644 digits/templates/job_item.html create mode 100644 digits/templates/job_management.html create mode 100644 digits/templates/job_row.html diff --git a/.gitignore b/.gitignore index 1d80a33af..01164b992 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,9 @@ *.pyc *.swp +# Ignore emacs backup files +*~ + # Ignore these files .coverage diff --git a/digits/dataset/tasks/analyze_db.py b/digits/dataset/tasks/analyze_db.py index ef1a90290..663f5f67c 100644 --- a/digits/dataset/tasks/analyze_db.py +++ b/digits/dataset/tasks/analyze_db.py @@ -107,16 +107,7 @@ def process_output(self, line): match = re.match(r'Progress: (\d+)\/(\d+)', message) if match: self.progress = float(match.group(1))/float(match.group(2)) - socketio.emit('task update', - { - 'task': self.html_id(), - 'update': 'progress', - 'percentage': int(round(100*self.progress)), - 'eta': utils.time_filters.print_time_diff(self.est_done()), - }, - namespace='/jobs', - room=self.job_id, - ) + self.emit_progress_update() return True # total count diff --git a/digits/dataset/tasks/create_db.py b/digits/dataset/tasks/create_db.py index 7a4e33da8..5b7d094c1 100644 --- a/digits/dataset/tasks/create_db.py +++ b/digits/dataset/tasks/create_db.py @@ -182,16 +182,7 @@ def process_output(self, line): match = re.match(r'Processed (\d+)\/(\d+)', message) if match: self.progress = float(match.group(1))/int(match.group(2)) - socketio.emit('task update', - { - 'task': self.html_id(), - 'update': 'progress', - 'percentage': int(round(100*self.progress)), - 'eta': utils.time_filters.print_time_diff(self.est_done()), - }, - namespace='/jobs', - room=self.job_id, - ) + self.emit_progress_update() return True # distribution diff --git a/digits/dataset/tasks/parse_folder.py b/digits/dataset/tasks/parse_folder.py index f11b7d609..78bf88389 100644 --- a/digits/dataset/tasks/parse_folder.py +++ b/digits/dataset/tasks/parse_folder.py @@ -148,16 +148,7 @@ def process_output(self, line): match = re.match(r'Progress: ([-+]?[0-9]*\.?[0-9]+(e[-+]?[0-9]+)?)', message) if match: self.progress = float(match.group(1)) - socketio.emit('task update', - { - 'task': self.html_id(), - 'update': 'progress', - 'percentage': int(round(100*self.progress)), - 'eta': utils.time_filters.print_time_diff(self.est_done()), - }, - namespace='/jobs', - room=self.job_id, - ) + self.emit_progress_update() return True # totals diff --git a/digits/job.py b/digits/job.py index fa7f9fdbe..1a6fb021b 100644 --- a/digits/job.py +++ b/digits/job.py @@ -8,6 +8,7 @@ import flask +from digits import utils from digits.config import config_value from digits.utils import sizeof_fmt, filesystem as fs from status import Status, StatusCls @@ -158,6 +159,7 @@ def on_status_update(self): 'status': self.status.name, 'css': self.status.css, 'running': self.status.is_running(), + 'job_id': self.id(), } with app.app_context(): message['html'] = flask.render_template('status_updates.html', updates=self.status_history) @@ -168,6 +170,13 @@ def on_status_update(self): room=self.id(), ) + # send message to job_management room as well + socketio.emit('job update', + message, + namespace='/jobs', + room='job_management', + ) + def abort(self): """ Abort a job and stop all running tasks @@ -202,3 +211,34 @@ def disk_size_fmt(self): size = fs.get_tree_size(self._dir) return sizeof_fmt(size) + def get_progress(self): + """ + Return job progress computed from task progress + """ + if len(self.tasks) == 0: + return 0.0 + + progress = 0.0 + + for task in self.tasks: + progress += task.progress + + progress /= len(self.tasks) + return progress + + def emit_progress_update(self): + """ + Call socketio.emit for task job update, by considering task progress. + """ + progress = self.get_progress() + + from digits.webapp import socketio + socketio.emit('job update', + { + 'job_id': self.id(), + 'update': 'progress', + 'percentage': int(round(100*progress)), + }, + namespace='/jobs', + room='job_management' + ) diff --git a/digits/model/tasks/train.py b/digits/model/tasks/train.py index 4ee29374b..3d5f64e80 100644 --- a/digits/model/tasks/train.py +++ b/digits/model/tasks/train.py @@ -207,17 +207,7 @@ def send_progress_update(self, epoch): self.current_epoch = epoch self.progress = epoch/self.train_epochs - - socketio.emit('task update', - { - 'task': self.html_id(), - 'update': 'progress', - 'percentage': int(round(100*self.progress)), - 'eta': utils.time_filters.print_time_diff(self.est_done()), - }, - namespace='/jobs', - room=self.job_id, - ) + self.emit_progress_update() def save_train_output(self, *args): """ @@ -247,6 +237,20 @@ def save_train_output(self, *args): room=self.job_id, ) + if data['columns']: + # isolate the Loss column data for the sparkline + graph_data = data['columns'][0][1:] + socketio.emit('task update', + { + 'task': self.html_id(), + 'job_id': self.job_id, + 'update': 'combined_graph', + 'data': graph_data, + }, + namespace='/jobs', + room='job_management', + ) + # lr graph data data = self.lr_graph_data() if data: diff --git a/digits/scheduler.py b/digits/scheduler.py index 85d7306a8..275dc569b 100644 --- a/digits/scheduler.py +++ b/digits/scheduler.py @@ -175,6 +175,35 @@ def add_job(self, job): return False else: self.jobs.append(job) + + # Need to fix this properly + # if True or flask._app_ctx_stack.top is not None: + from digits.webapp import app + with app.app_context(): + # send message to job_management room that the job is added + import flask + html = flask.render_template('job_row.html', job = job) + + # Convert the html into a list for the jQuery + # DataTable.row.add() method. This regex removes the + # and tags, and splits the string into one element + # for each cell. + import re + html = re.sub('[\s\n\r]*[\s\n\r]*', '', html) + html = re.sub('[\s\n\r]*[\s\n\r]*', '', html) + html = re.split('[\s\n\r]*', html) + + from digits.webapp import socketio + socketio.emit('job update', + { + 'update': 'added', + 'job_id': job.id(), + 'html': html + }, + namespace='/jobs', + room='job_management', + ) + if 'DIGITS_MODE_TEST' not in os.environ: # Let the scheduler do a little work before returning time.sleep(utils.wait_time()) @@ -238,6 +267,15 @@ def delete_job(self, job): if os.path.exists(job.dir()): shutil.rmtree(job.dir()) logger.info('Job deleted.', job_id=job_id) + from digits.webapp import socketio + socketio.emit('job update', + { + 'update': 'deleted', + 'job_id': job.id() + }, + namespace='/jobs', + room='job_management', + ) return True # see if the folder exists on disk @@ -417,6 +455,7 @@ def reserve_resources(self, task, resources): for resource in self.resources[resource_type]: if resource.identifier == identifier: resource.allocate(task, value) + self.emit_gpus_available() found = True break if not found: @@ -439,6 +478,7 @@ def release_resources(self, task, resources): for resource in self.resources[resource_type]: if resource.identifier == identifier: resource.deallocate(task) + self.emit_gpus_available() task.current_resources = None def run_task(self, task, resources): @@ -457,3 +497,17 @@ def run_task(self, task, resources): finally: self.release_resources(task, resources) + def emit_gpus_available(self): + """ + Call socketio.emit gpu availablity + """ + from digits.webapp import scheduler, socketio + socketio.emit('server update', + { + 'update': 'gpus_available', + 'total_gpu_count': len(self.resources['gpus']), + 'remaining_gpu_count': sum(r.remaining() for r in scheduler.resources['gpus']), + }, + namespace='/jobs', + room='job_management' + ) diff --git a/digits/static/css/style.css b/digits/static/css/style.css index b173474f4..d6957cf87 100644 --- a/digits/static/css/style.css +++ b/digits/static/css/style.css @@ -1,3 +1,5 @@ +/* Copyright (c) 2014-2015, NVIDIA CORPORATION. All rights reserved. */ + body { padding-top: 50px; padding-bottom: 50px; @@ -27,6 +29,13 @@ path.c3-line { .autocomplete-group { padding: 2px 5px; } .autocomplete-group strong { display: block; border-bottom: 1px solid #000; } +/* To fix the tooltip in the sparkline */ +.jqstooltip { + -webkit-box-sizing: content-box; + -moz-box-sizing: content-box; + box-sizing: content-box; +} + .btn-file { position: relative; overflow: hidden; @@ -66,5 +75,3 @@ a.active { ul.inline li { display: inline-block; } - - diff --git a/digits/static/js/jquery.sparkline.min.js b/digits/static/js/jquery.sparkline.min.js new file mode 100644 index 000000000..fa616bf96 --- /dev/null +++ b/digits/static/js/jquery.sparkline.min.js @@ -0,0 +1,5 @@ +/* jquery.sparkline 2.1.2 - http://omnipotent.net/jquery.sparkline/ +** Licensed under the New BSD License - see above site for details */ + +(function(a,b,c){(function(a){typeof define=="function"&&define.amd?define(["jquery"],a):jQuery&&!jQuery.fn.sparkline&&a(jQuery)})(function(d){"use strict";var e={},f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w,x,y,z,A,B,C,D,E,F,G,H,I,J,K,L=0;f=function(){return{common:{type:"line",lineColor:"#00f",fillColor:"#cdf",defaultPixelsPerValue:3,width:"auto",height:"auto",composite:!1,tagValuesAttribute:"values",tagOptionsPrefix:"spark",enableTagOptions:!1,enableHighlight:!0,highlightLighten:1.4,tooltipSkipNull:!0,tooltipPrefix:"",tooltipSuffix:"",disableHiddenCheck:!1,numberFormatter:!1,numberDigitGroupCount:3,numberDigitGroupSep:",",numberDecimalMark:".",disableTooltips:!1,disableInteraction:!1},line:{spotColor:"#f80",highlightSpotColor:"#5f5",highlightLineColor:"#f22",spotRadius:1.5,minSpotColor:"#f80",maxSpotColor:"#f80",lineWidth:1,normalRangeMin:c,normalRangeMax:c,normalRangeColor:"#ccc",drawNormalOnTop:!1,chartRangeMin:c,chartRangeMax:c,chartRangeMinX:c,chartRangeMaxX:c,tooltipFormat:new h('● {{prefix}}{{y}}{{suffix}}')},bar:{barColor:"#3366cc",negBarColor:"#f44",stackedBarColor:["#3366cc","#dc3912","#ff9900","#109618","#66aa00","#dd4477","#0099c6","#990099"],zeroColor:c,nullColor:c,zeroAxis:!0,barWidth:4,barSpacing:1,chartRangeMax:c,chartRangeMin:c,chartRangeClip:!1,colorMap:c,tooltipFormat:new h('● {{prefix}}{{value}}{{suffix}}')},tristate:{barWidth:4,barSpacing:1,posBarColor:"#6f6",negBarColor:"#f44",zeroBarColor:"#999",colorMap:{},tooltipFormat:new h('● {{value:map}}'),tooltipValueLookups:{map:{"-1":"Loss",0:"Draw",1:"Win"}}},discrete:{lineHeight:"auto",thresholdColor:c,thresholdValue:0,chartRangeMax:c,chartRangeMin:c,chartRangeClip:!1,tooltipFormat:new h("{{prefix}}{{value}}{{suffix}}")},bullet:{targetColor:"#f33",targetWidth:3,performanceColor:"#33f",rangeColors:["#d3dafe","#a8b6ff","#7f94ff"],base:c,tooltipFormat:new h("{{fieldkey:fields}} - {{value}}"),tooltipValueLookups:{fields:{r:"Range",p:"Performance",t:"Target"}}},pie:{offset:0,sliceColors:["#3366cc","#dc3912","#ff9900","#109618","#66aa00","#dd4477","#0099c6","#990099"],borderWidth:0,borderColor:"#000",tooltipFormat:new h('● {{value}} ({{percent.1}}%)')},box:{raw:!1,boxLineColor:"#000",boxFillColor:"#cdf",whiskerColor:"#000",outlierLineColor:"#333",outlierFillColor:"#fff",medianColor:"#f00",showOutliers:!0,outlierIQR:1.5,spotRadius:1.5,target:c,targetColor:"#4a2",chartRangeMax:c,chartRangeMin:c,tooltipFormat:new h("{{field:fields}}: {{value}}"),tooltipFormatFieldlistKey:"field",tooltipValueLookups:{fields:{lq:"Lower Quartile",med:"Median",uq:"Upper Quartile",lo:"Left Outlier",ro:"Right Outlier",lw:"Left Whisker",rw:"Right Whisker"}}}}},E='.jqstooltip { position: absolute;left: 0px;top: 0px;visibility: hidden;background: rgb(0, 0, 0) transparent;background-color: rgba(0,0,0,0.6);filter:progid:DXImageTransform.Microsoft.gradient(startColorstr=#99000000, endColorstr=#99000000);-ms-filter: "progid:DXImageTransform.Microsoft.gradient(startColorstr=#99000000, endColorstr=#99000000)";color: white;font: 10px arial, san serif;text-align: left;white-space: nowrap;padding: 5px;border: 1px solid white;z-index: 10000;}.jqsfield { color: white;font: 10px arial, san serif;text-align: left;}',g=function(){var a,b;return a=function(){this.init.apply(this,arguments)},arguments.length>1?(arguments[0]?(a.prototype=d.extend(new arguments[0],arguments[arguments.length-1]),a._super=arguments[0].prototype):a.prototype=arguments[arguments.length-1],arguments.length>2&&(b=Array.prototype.slice.call(arguments,1,-1),b.unshift(a.prototype),d.extend.apply(d,b))):a.prototype=arguments[0],a.prototype.cls=a,a},d.SPFormatClass=h=g({fre:/\{\{([\w.]+?)(:(.+?))?\}\}/g,precre:/(\w+)\.(\d+)/,init:function(a,b){this.format=a,this.fclass=b},render:function(a,b,d){var e=this,f=a,g,h,i,j,k;return this.format.replace(this.fre,function(){var a;return h=arguments[1],i=arguments[3],g=e.precre.exec(h),g?(k=g[2],h=g[1]):k=!1,j=f[h],j===c?"":i&&b&&b[i]?(a=b[i],a.get?b[i].get(j)||j:b[i][j]||j):(n(j)&&(d.get("numberFormatter")?j=d.get("numberFormatter")(j):j=s(j,k,d.get("numberDigitGroupCount"),d.get("numberDigitGroupSep"),d.get("numberDecimalMark"))),j)})}}),d.spformat=function(a,b){return new h(a,b)},i=function(a,b,c){return ac?c:a},j=function(a,c){var d;return c===2?(d=b.floor(a.length/2),a.length%2?a[d]:(a[d-1]+a[d])/2):a.length%2?(d=(a.length*c+c)/4,d%1?(a[b.floor(d)]+a[b.floor(d)-1])/2:a[d-1]):(d=(a.length*c+2)/4,d%1?(a[b.floor(d)]+a[b.floor(d)-1])/2:a[d-1])},k=function(a){var b;switch(a){case"undefined":a=c;break;case"null":a=null;break;case"true":a=!0;break;case"false":a=!1;break;default:b=parseFloat(a),a==b&&(a=b)}return a},l=function(a){var b,c=[];for(b=a.length;b--;)c[b]=k(a[b]);return c},m=function(a,b){var c,d,e=[];for(c=0,d=a.length;c0;h-=c)a.splice(h,0,e);return a.join("")},o=function(a,b,c){var d;for(d=b.length;d--;){if(c&&b[d]===null)continue;if(b[d]!==a)return!1}return!0},p=function(a){var b=0,c;for(c=a.length;c--;)b+=typeof a[c]=="number"?a[c]:0;return b},r=function(a){return d.isArray(a)?a:[a]},q=function(b){var c;a.createStyleSheet?a.createStyleSheet().cssText=b:(c=a.createElement("style"),c.type="text/css",a.getElementsByTagName("head")[0].appendChild(c),c[typeof a.body.style.WebkitAppearance=="string"?"innerText":"innerHTML"]=b)},d.fn.simpledraw=function(b,e,f,g){var h,i;if(f&&(h=this.data("_jqs_vcanvas")))return h;if(d.fn.sparkline.canvas===!1)return!1;if(d.fn.sparkline.canvas===c){var j=a.createElement("canvas");if(!j.getContext||!j.getContext("2d")){if(!a.namespaces||!!a.namespaces.v)return d.fn.sparkline.canvas=!1,!1;a.namespaces.add("v","urn:schemas-microsoft-com:vml","#default#VML"),d.fn.sparkline.canvas=function(a,b,c,d){return new J(a,b,c)}}else d.fn.sparkline.canvas=function(a,b,c,d){return new I(a,b,c,d)}}return b===c&&(b=d(this).innerWidth()),e===c&&(e=d(this).innerHeight()),h=d.fn.sparkline.canvas(b,e,this,g),i=d(this).data("_jqs_mhandler"),i&&i.registerCanvas(h),h},d.fn.cleardraw=function(){var a=this.data("_jqs_vcanvas");a&&a.reset()},d.RangeMapClass=t=g({init:function(a){var b,c,d=[];for(b in a)a.hasOwnProperty(b)&&typeof b=="string"&&b.indexOf(":")>-1&&(c=b.split(":"),c[0]=c[0].length===0?-Infinity:parseFloat(c[0]),c[1]=c[1].length===0?Infinity:parseFloat(c[1]),c[2]=a[b],d.push(c));this.map=a,this.rangelist=d||!1},get:function(a){var b=this.rangelist,d,e,f;if((f=this.map[a])!==c)return f;if(b)for(d=b.length;d--;){e=b[d];if(e[0]<=a&&e[1]>=a)return e[2]}return c}}),d.range_map=function(a){return new t(a)},u=g({init:function(a,b){var c=d(a);this.$el=c,this.options=b,this.currentPageX=0,this.currentPageY=0,this.el=a,this.splist=[],this.tooltip=null,this.over=!1,this.displayTooltips=!b.get("disableTooltips"),this.highlightEnabled=!b.get("disableHighlight")},registerSparkline:function(a){this.splist.push(a),this.over&&this.updateDisplay()},registerCanvas:function(a){var b=d(a.canvas);this.canvas=a,this.$canvas=b,b.mouseenter(d.proxy(this.mouseenter,this)),b.mouseleave(d.proxy(this.mouseleave,this)),b.click(d.proxy(this.mouseclick,this))},reset:function(a){this.splist=[],this.tooltip&&a&&(this.tooltip.remove(),this.tooltip=c)},mouseclick:function(a){var b=d.Event("sparklineClick");b.originalEvent=a,b.sparklines=this.splist,this.$el.trigger(b)},mouseenter:function(b){d(a.body).unbind("mousemove.jqs"),d(a.body).bind("mousemove.jqs",d.proxy(this.mousemove,this)),this.over=!0,this.currentPageX=b.pageX,this.currentPageY=b.pageY,this.currentEl=b.target,!this.tooltip&&this.displayTooltips&&(this.tooltip=new v(this.options),this.tooltip.updatePosition(b.pageX,b.pageY)),this.updateDisplay()},mouseleave:function(){d(a.body).unbind("mousemove.jqs");var b=this.splist,c=b.length,e=!1,f,g;this.over=!1,this.currentEl=null,this.tooltip&&(this.tooltip.remove(),this.tooltip=null);for(g=0;g