diff --git a/README.md b/README.md index 1b9c349..03d8f8f 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,96 @@ [![Package quality](http://packagequality.com/badge/loadtest.png)](http://packagequality.com/#?package=loadtest) +# Gwloadtest + +Gwloadtest modified loadtest to creat load tests that imitate client-behavior more closely by using randomly distributed time intervals between requests and allowing for more random url access behavior. + +## How to Use + + 1. clone this repository, and open the newly cloned directory: + +``` + git clone https://github.com/gwcloudlab/gwloadtest.git +``` + + 2. Install Node.js and npm by by running the following commands: + +``` + ./install_node.sh +``` + + 3. Modify sample/request-generator.js with desired options + 4. Run the following command to run the test: + +``` + node sample/request-generator.js +``` +## Modfifications + +### Poisson Interarrival time + +Given a requests per second rate and a time inerval for the load test, gwloadtests send requests at randomly distributed time intervals. + +### Logs and Summaries + +Gwloadtest prints a summary text file along with a log file that includes the index, timestamp, latency, status code, and url of every request. + +## Additions + +### Client Modes + +In an attempt to resmeble client behavior, gwloadtest carries two client behavior policies. + +#### Closed Loop + +The client thread sends a request and waits for it to return an html, which it then parses for hyperlinks and randomly selects a link so send the next request to. If no links are found, it sends the next request to the options url. + +usage example: +```javascript + const options = { + url: 'http://127.0.0.1:5000', + statusCallback: statusCallback, + requestsPerSecond: 10, + rpsInterval: 10, + clientMode: 'closed' + }; +``` + +#### Open Loop + +The client thread sends requests to multiple links according to the weights given in a file. A file must be passed in along with selecting the open clientMode. + +usage example: +```javascript + const options = { + url: 'http://127.0.0.1:5000', + statusCallback: statusCallback, + requestsPerSecond: 10, + rpsInterval: 10, + urlList: 'sample/url_list.txt', //include title of file here + clientMode: 'open' + }; +``` +sample/url_list.txt: +``` + url, weight + http://127.0.0.1:5000, 0.80 + http://127.0.0.1:5000/test1, 0.20 +``` + +### graphing scripts + +Gwloadtest carries 4 types of graphing scripts in the scripts file, which can be run on the Gwloadtest generated log files. + 1. 99th percentile latency vs. requests per second rate + 2. number of occurences vs latency histogram + 3. latency over time line graph + 4. cdf vs response time + + +### Automation Script + +For research purposes, automation/run.py can run multiple tests given a list of requests per second rates. + # loadtest Runs a load test on the selected HTTP or WebSockets URL. The API allows for easy integration in your own tests. diff --git a/automation/function.py b/automation/function.py new file mode 100755 index 0000000..d03f88b --- /dev/null +++ b/automation/function.py @@ -0,0 +1,16 @@ +import os +import time +from flask import Flask +fibapp = Flask(__name__) +@fibapp.route('/') +def index(): + # temp() + return "Hello World!" + +def temp(): + time.sleep(0.1) + print("waited") + return + +if __name__ == '__main__': + fibapp.run() \ No newline at end of file diff --git a/automation/run.py b/automation/run.py new file mode 100644 index 0000000..deb918c --- /dev/null +++ b/automation/run.py @@ -0,0 +1,23 @@ +import os, json +import subprocess +from tracemalloc import start +from Naked.toolshed.shell import execute_js + + + +def generateLoad(req, duration): + command = "sample/request-generator.js --rps=" + str(req) + " --interval=" + str(duration) + execute_js(command) #run the script with the arguments + +def main(): + TEST_LIST = [1, 5, 10, 50, 100, 500, 1000] + duration = 10 + # start_function() + for test in TEST_LIST: + generateLoad(test, duration) + + + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/install_node.sh b/install_node.sh new file mode 100644 index 0000000..e36787a --- /dev/null +++ b/install_node.sh @@ -0,0 +1,4 @@ +sudo apt update +sudo apt-get install -y nodejs +cd ~/gwloadtest +npm install \ No newline at end of file diff --git a/lib/httpClient.js b/lib/httpClient.js index 1bd4d90..83ddfc2 100644 --- a/lib/httpClient.js +++ b/lib/httpClient.js @@ -16,10 +16,18 @@ const websocket = require('websocket'); const Log = require('log'); const {HighResolutionTimer} = require('./hrtimer.js'); const headers = require('./headers.js'); +const fs = require('fs'); +const { syncBuiltinESMExports } = require('module'); // globals const log = new Log('info'); - +const timeStamps = []; // added for gwloadtest timestamps start time of each request +const req_url = []; // added for gwloadtest recording the chosen url for each request +var url_list = null; //added for gwloadtest client behaviour used to store the file of the urls +var total; //added for gwloadtest client behaviour, stores the total number of requests sent +var prev; //added for gwloadtest client behaviour stores the html of the previous request +var next = []; //added for gwloadtest client behaviour, sstores a lidt of possible next urls to follow +var cumulative_time =0; //added to keep track of cumalative time for closed loop request /** * Create a new HTTP client. @@ -106,17 +114,154 @@ class HttpClient { log.debug('Options: %j', this.options); } + /** + * gwloadtest: returns a randomly distributed exponential number + * based on the given rps + */ + randomExp(rps){ + let v = Number(rps); + v = -1/v; + let u = Math.random(); + u = Math.log(u); + let x = u*v; + return x; + } + /** * Start the HTTP client. */ start() { - if (!this.params.requestsPerSecond) { - return this.makeRequest(); + if (this.params.clientMode == 'open'){ //if this is an open loop client request model + //gwloadtest modification: read url list to make requested according to weights + if (this.params.urlList){ + try { + total = this.params.requestsPerSecond * this.params.rpsInterval; + const data = fs.readFileSync(this.params.urlList, 'utf8'); + var arr = this.fileToArray(data, ', ') + console.log(arr); + url_list = arr; + for (var i=0; i this.makeRequest()); + if (this.params.clientMode == 'closed'){ // /if this is a closed loop request model, which means it waits for prev request + //check if there is an interval + total = 0; + this.parseHtml(prev); + var interval = 1000 * this.randomExp(rps); + cumulative_time += interval; + timeStamps[total] = cumulative_time; + total++; + setTimeout(() => this.makeRequest(), cumulative_time); // sets a timer for the first request only + } + else{ // if this is an open loop model, make the requests + if (this.params.rpsInterval) { // if a time interval is given, use exponentially distributed numbers + var rps = this.params.requestsPerSecond; + if (this.params.agentKeepAlive) { + this.options.agent.maxSockets = 10 + rps; + } + + // stop the old requesttimer + if (this.requestTimer !== undefined) { + this.requestTimer.stop(); + } + + var total_time = 0; + var intr = this.params.rpsInterval *1000; + var num_rquests =0; + while (total_time< intr) { + var interval = 1000 * this.randomExp(rps); + total_time += interval; + timeStamps[num_rquests] =total_time; + num_rquests++; + setTimeout(() => this.makeRequest(), total_time); // sets a timer for all the requests + } + } + else{ // if no time interval given, use constant time intervals (original loadtest method) + const interval = 1000 / this.params.requestsPerSecond; + // start new request timer + this.requestTimer = new HighResolutionTimer(interval, () => this.makeRequest()); + } + } + + //gwloadtest: create log files + var intr = this.params.rpsInterval *1000; + var rps = this.params.requestsPerSecond; + let fileTitle = "sample/log-rps-"+rps+"-iv-"+intr/1000 + ".csv"; + let m = 'Index, timestamp, latency, status code, url\n'; + fs.writeFile(fileTitle, m, { flag: 'w' }, err => {}); + } + + /** + * gwloadtest: takes in a text/csv file and returns it as an array + */ + fileToArray(str, delimiter = ",") { + // slice from start of text to the first \n index + // use split to create an array from string by delimiter + const headers = str.slice(0, str.indexOf("\n")).split(delimiter); + + // slice from \n index + 1 to the end of the text + const rows = str.slice(str.indexOf("\n") + 1).split("\n"); + + // Map the rows + const arr = rows.map(function (row) { + const values = row.split(delimiter); + const el = headers.reduce(function (object, header, index) { + object[header] = values[index]; + return object; + }, {}); + return el; + }); + + // return the array + return arr; } + + /** + * gwloadtest: takes in a text html and returns an array of all links found as + * hyperlinks + */ + parseHtml( prev ){ + var table = []; + var links = []; + var temp = String(prev); + var num = 0; + var count = 0; + + for (var i = 0; i', i)){ // parse hyperlinks + var arr = [ temp.indexOf('', i)]; + table[num] = arr; + + if ( temp.indexOf('href="', i)>0){ + var start = temp.indexOf('href="', i); + var end = temp.indexOf('"', start+6); + links[count] = temp.substring(start+6, end); + count++; + } + i = temp.indexOf('/a>', i); + num++; + } + else if( temp.indexOf('0){ //if a url has not handled its weight break, else choose a differnt url + url_list[num].weight--; + temp=url_list[num].url; + req_url[this.operation.requests-1] = temp; + break; + } + } + + } + + else if (this.params.clientMode == 'closed'){ + + if (next.length>0){ // check if prev request returned more links + + var num = Math.floor(Math.random() * next.length); + temp=next[num]; + req_url[this.operation.requests-1] = temp; + } + else { // if not, go to first link + temp = this.options; + req_url[this.operation.requests-1] = this.params.url; + } + + + } + + else { + temp = this.options; + req_url[this.operation.requests-1] = this.params.url; + } + + request = lib.request(temp, this.getConnect(id, requestFinished, this.params.contentInspector)); } if (this.params.timeout) { const timeout = parseInt(this.params.timeout); @@ -196,7 +380,7 @@ class HttpClient { /** * Get a function that finishes one request and goes for the next. */ - getRequestFinisher(id) { + getRequestFinisher(id, timestamp) { return (error, result) => { let errorCode = null; if (error) { @@ -214,16 +398,35 @@ class HttpClient { } const elapsed = this.operation.latency.end(id, errorCode); + if (elapsed < 0) { // not found or not running return; } - const index = this.operation.latency.getRequestIndex(id); + const index = this.operation.latency.getRequestIndex(id); + if (result) { result.requestElapsed = elapsed; result.requestIndex = index; result.instanceIndex = this.operation.instanceIndex; + result.startTime = timeStamps[index]; // stores the timestamps of the request added for gwloadtest + prev = result.body; + result.url = req_url[index]; + this.parseHtml(prev); + var intr = this.params.rpsInterval *1000; + + // if the loop is closed then parse previous html for urls, then set timer for next request + if (this.params.clientMode == 'closed' && cumulative_time this.makeRequest(), interval); // sets a timer for all the requests + } + } + let callback; if (!this.params.requestsPerSecond) { callback = this.makeRequest.bind(this); diff --git a/sample/request-generator.js b/sample/request-generator.js index 765fcdf..02fa776 100644 --- a/sample/request-generator.js +++ b/sample/request-generator.js @@ -1,37 +1,125 @@ -'use strict'; +// requires +const loadtest = require('../lib/loadtest.js'); +const testserver = require('../lib/testserver.js'); +const args = require('minimist')(process.argv.slice(2)); -/** - * Sample request generator usage. - * Contributed by jjohnsonvng: - * https://github.com/alexfernandez/loadtest/issues/86#issuecomment-211579639 - */ -const loadtest = require('../lib/loadtest.js'); +const fs = require('fs'); +//uncomment to use automation script +//rps = args.rps; +//rpsInter= args.interval; +const dta = []; +var errs= 0; +var slowest; +var fastest; +var max; +var avg =0; const options = { - url: 'http://yourHost', - concurrency: 5, - method: 'POST', - body:'', - requestsPerSecond:5, - maxSeconds:30, - requestGenerator: (params, options, client, callback) => { - const message = '{"hi": "ho"}'; - options.headers['Content-Length'] = message.length; - options.headers['Content-Type'] = 'application/json'; - options.body = 'YourPostData'; - options.path = 'YourURLPath'; - const request = client(options, callback); - request.write(message); - return request; - } + url: 'http://127.0.0.1:5000', + statusCallback: statusCallback, + requestsPerSecond: 10, // replace number with rps for automation script + rpsInterval: 10, //replace number with rpsInter for automation script + urlList: 'sample/url_list.txt',// an example of passing in a list of urls with weights by passing in the file title + clientMode: 'open' //passed in to activate gwloadtest modifications 'closed' for closed loop requests and 'open' for open loop requests + }; -loadtest.loadTest(options, (error, results) => { - if (error) { - return console.error('Got an error: %s', error); - } - console.log(results); - console.log('Tests run successfully'); +let fileTitle = "sample/log-rps-"+options.requestsPerSecond+"-iv-"+options.rpsInterval+ ".csv"; +let fileTitle2 = "sample/sum-rps-"+options.requestsPerSecond+"-iv-"+options.rpsInterval + ".txt"; + +//the function called after every request is finished +function statusCallback(error, result, latency) { + console.log('----'); + console.log('Timestamp: ', result.startTime.toFixed(2)); + console.log('Request index: ', result.requestIndex); + console.log('Request elapsed milliseconds: ', result.requestElapsed); + console.log('Code: ', result.statusCode); + console.log('URL: ', result.url); + + let n = result.requestElapsed.toFixed(); + let s = result.requestIndex + ", " +result.startTime.toFixed(2) + ", " + n.toString() + ", " + result.statusCode + ", " + String(result.url) +"\n"; + + fs.writeFileSync(fileTitle, s, { flag: 'a+' }, err => {}); //write to the log file + + dta[result.requestIndex]= result.requestElapsed; // store elpased for percentile calculation + + avg+=parseFloat(result.requestElapsed); // add to sum + + if (result.requestIndex==0){ + slowest = result.requestElapsed; + fastest = result.requestElapsed; + + } + else{ + if (result.requestElapsedslowest){ + slowest = result.requestElapsed; + } + } + + if ( result.statusCode>299){ + errs++; + } + + if ( result.startTime>options.rpsInterval*1000){ + max = result.requestIndex +1; + + // print summary after last request + let f = "Summary: " + '\n'; + f = f + 'total time: '; + m = (result.startTime/1000).toFixed(2) + ' s\n'; + f = f + m + 'total requests: ' + max + '\n'; + + var num = result.startTime/1000; + f = f + 'throughput: ' + (max/num).toFixed(2) + ' req/s\n'; + fs.writeFileSync(fileTitle2, f, { flag: 'w' }, err => {}); + + percentile(); + process.exit(0); + } +} + +function percentile(){ + let f = 'errors: ' + errs + '\n' + 'average: ' + (avg/max).toFixed(2) +' ms\n' + 'fastest: ' + fastest.toFixed(2) + ' ms\n'+ 'slowest: ' + slowest.toFixed(2) + ' ms\n' ; + + dta.sort(function(a, b){return a - b}); + + var p25 =( 0.25*(max-1)).toFixed(); + f = f + '25%ile Latency: ' + dta[p25].toFixed(2) + ' ms\n'; + + var p50 =( 0.50*(max-1)).toFixed(); + f = f + '50%ile Latency: ' + dta[p50].toFixed(2) + ' ms\n'; + + var p75 =( 0.75*(max-1)).toFixed(); + f = f + '75%ile Latency: ' + dta[p75].toFixed(2) + ' ms\n'; + + var p99 =( 0.99*(max-1)).toFixed(); + f = f +'99%ile Latency: ' + dta[p99] + ' ms\n'; + + + var p999 =( 0.999*(max-1)).toFixed(); + f = f + '99.9%ile Latency: ' + dta[p999] + ' ms\n'; + + fs.writeFileSync(fileTitle2, f, { flag: 'a' }, err => {}); +} + + + + +loadtest.loadTest(options, function(error) { + if (error) { + return console.error('Got an error: %s', error); + + } + + console.log('Tests run successfully'); }); +// exports +exports.loadTest = loadtest.loadTest; +exports.startServer = testserver.startServer; + diff --git a/sample/request-generator.ts b/sample/request-generator.ts deleted file mode 100644 index d3e026c..0000000 --- a/sample/request-generator.ts +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Sample request generator usage. - * Contributed by jjohnsonvng: - * https://github.com/alexfernandez/loadtest/issues/86#issuecomment-211579639 - */ - -import loadtest from "loadtest" - -const options: loadtest.LoadTestOptions = { - url: "http://yourHost", - concurrency: 5, - method: "POST", - body: "", - requestsPerSecond: 5, - maxSeconds: 30, - requestGenerator: (params, options, client, callback) => { - const message = '{"hi": "ho"}' - options.headers["Content-Length"] = message.length - options.headers["Content-Type"] = "application/json" - options.body = "YourPostData" - options.path = "YourURLPath" - const request = client(options, callback) - request.write(message) - return request - }, -} - -loadtest.loadTest(options, (error, results) => { - if (error) { - return console.error(`Got an error: ${error}`) - } - console.log("Tests run successfully", {results}) -}) diff --git a/scripts/99ile.py b/scripts/99ile.py new file mode 100644 index 0000000..bf5bf82 --- /dev/null +++ b/scripts/99ile.py @@ -0,0 +1,22 @@ +import csv +import sys +import matplotlib.pyplot as plt +import pandas as pd + +df = pd.read_csv(sys.argv[0]) +print(df) +data = [] +num = [] +f, ax = plt.subplots(1) +for index, row in df.iterrows(): + num.append(row['rps']) + #put header name of data you want in the hard brackets + data.append(row[' 99ile']) + +plt.title('99ile latency vs Rps', fontsize = 18) +plt.xlabel('rps rate', fontsize = 12) +plt.ylabel('99ile latency', fontsize = 12) +ax.plot(num, data) +ax.set_xlim(left=0) +plt.savefig('99ile_plot.png') +plt.show() \ No newline at end of file diff --git a/scripts/cdf.py b/scripts/cdf.py new file mode 100644 index 0000000..f172bd5 --- /dev/null +++ b/scripts/cdf.py @@ -0,0 +1,52 @@ +import pandas as pd +import csv +import sys +import matplotlib.pyplot as plt +import numpy as np +import warnings + +#Put file you want to read in input folder + +#********************************************************************** +#List to store the data (ADD AS MUCH AS NECESSARY) +#********************************************************************** + +xlist = [] +ylist = [] + +#********************************************************************** +#File Read +#fname : put the name of the file you are reading +#********************************************************************** + + +#Change path +df = pd.read_csv(sys.argv[0]) + +#********************************************************************** +#df.iterrows returns series for each row it does not preserve data types across the rows +#iterate through the rows and take the data with it and append it to lists to use +#--------------------------------------------------------------------------------- +for index, row in df.iterrows(): + xlist.append(index) + #put header name of data you want in the hard brackets + ylist.append(row[' latency']) + + +#--------------------------------------------------------------------------------- +#********************************************************************** + +#-------------------------- +#CDF plot +#-------------------------- +sorted_data = np.sort(ylist) +yvals=np.arange(len(sorted_data))/float(len(sorted_data)-1) +plt.style.use('seaborn-whitegrid') # nice and clean grid +plt.title('Response time cdf', fontsize = 21) +plt.xlabel('response time (ms)', fontsize = 15) +plt.ylabel('cdf', fontsize = 15) +plt.ylim(0, 1) +plt.plot(sorted_data, yvals) +plt.savefig('cdf_plot.png') +plt.show() +#********************************************************************** \ No newline at end of file diff --git a/scripts/histogram.py b/scripts/histogram.py new file mode 100644 index 0000000..ba3eb4f --- /dev/null +++ b/scripts/histogram.py @@ -0,0 +1,46 @@ +import pandas as pd +import csv +import sys +import matplotlib.pyplot as plt +import numpy as np + +#Put file you want to read in input folder +#********************************************************************** +#List to store the data (ADD AS MUCH AS NECESSARY) +#********************************************************************** + +xlist = [] +ylist = [] + +#********************************************************************** +#File Read +#fname : put the name of the file you are reading +#********************************************************************** + +#Change path +df = pd.read_csv(sys.argv[0]) + +#********************************************************************** +#df.iterrows returns series for each row it does not preserve data types across the rows +#iterate through the rows and take the data with it and append it to lists +#--------------------------------------------------------------------------------- +for index, row in df.iterrows(): + # xlist.append(index) + #put header name of data you want in the hard brackets + xlist.append(float(row[' latency'])) + +#--------------------------------------------------------------------------------- +#********************************************************************** + +#-------------------------- +#Histogram plot +#-------------------------- +#plt.figure(figsize=(10,5)) # Make it 14x7 inch +plt.style.use('seaborn-whitegrid') # nice and clean grid +plt.title('Response time', fontsize = 21) +plt.xlabel('response time (ms)', fontsize = 15) +plt.ylabel('number of occurences', fontsize = 15) +plt.hist(xlist, linewidth=0.5, alpha=0.7) +plt.savefig('hist_plot.png') +plt.show() +#-------------------------- \ No newline at end of file diff --git a/scripts/linegraph.py b/scripts/linegraph.py new file mode 100644 index 0000000..0279f37 --- /dev/null +++ b/scripts/linegraph.py @@ -0,0 +1,38 @@ +import csv +import matplotlib.pyplot as plt +import os + +plt.figure() + +all_files = os.listdir(os.getcwd()) +file_names = list(filter(lambda f: f.endswith('.csv'), all_files)) +subplotnum = (len(file_names)*100)+11 #should be 311 for 3 files, 211 for 2 files, etc + +def create_subplot (data): + file_name = data + with open (data) as csv_file: + next(csv_file) + csv_reader = csv.reader(csv_file, delimiter=',') + i = 0 + data = [] + num = [] + for row in csv_reader: + data.append((float)(row[0])) + num.append(i) + i+=1 + ax = plt.subplot(subplotnum) + ax.set_title(file_name) + ax.plot(num, data) + +plt.suptitle("Latency over time(ms)", fontsize=16) + +for file in file_names: + create_subplot(file) + subplotnum += 1 + +plt.subplots_adjust(hspace=.7) +plt.show() +plt.savefig('line_plot.pdf', format='pdf') + + +