From dac510f5f4fbe4e57097ea647a4f6a89d31f2b04 Mon Sep 17 00:00:00 2001 From: David Fischer Date: Fri, 23 May 2014 22:31:32 +0200 Subject: [PATCH 01/24] Replace submodule by requirements file --- .gitignore | 4 +--- .gitmodules | 3 --- SocksiPy | 1 - requirements.txt | 1 + 4 files changed, 2 insertions(+), 7 deletions(-) delete mode 100644 .gitmodules delete mode 160000 SocksiPy create mode 100644 requirements.txt diff --git a/.gitignore b/.gitignore index 2bc1b44..8a2b429 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,2 @@ -old/ -*~ -~ +src/ *.pyc diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 091804b..0000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "SocksiPy"] - path = SocksiPy - url = git://github.com/Janhouse/SocksiPy.git diff --git a/SocksiPy b/SocksiPy deleted file mode 160000 index 842d496..0000000 --- a/SocksiPy +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 842d4962cbce16ce4b232d1b7402d0375f9a0c1b diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..3f105e7 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +-e git://github.com/Janhouse/SocksiPy@842d4962cbce16ce4b232d1b7402d0375f9a0c1b#egg=SocksiPy From a5767608d5072723e97c044c649742ebf917bfb8 Mon Sep 17 00:00:00 2001 From: David Fischer Date: Fri, 23 May 2014 22:53:32 +0200 Subject: [PATCH 02/24] Fix most of the PEP-8 warnings --- tespeed.py | 541 ++++++++++++++++++++++++++--------------------------- 1 file changed, 262 insertions(+), 279 deletions(-) diff --git a/tespeed.py b/tespeed.py index bc3d33a..1fa820a 100755 --- a/tespeed.py +++ b/tespeed.py @@ -1,32 +1,31 @@ +# -*- coding: utf-8 -*- + #!/usr/bin/env python2 # -# Copyright 2012-2013 Janis Jansons (janis.jansons@janhouse.lv) -# - -import argparse +# Copyright: +# 2012-2013 Janis Jansons (janis.jansons@janhouse.lv) +# 2014 David Fischer (david.fischer.ch@gmail.com) +import argparse, socket from SocksiPy import socks -import socket + # Magic! def getaddrinfo(*args): return [(socket.AF_INET, socket.SOCK_STREAM, 6, '', (args[0], args[1]))] socket.getaddrinfo = getaddrinfo -import urllib -import urllib2 -import gzip -import sys -from multiprocessing import Process, Pipe, Manager +import gzip, urllib, urllib2, sys, time from lxml import etree -import time from math import radians, cos, sin, asin, sqrt - +from multiprocessing import Process, Pipe, Manager from StringIO import StringIO + # Using StringIO with callback to measure upload progress -class CallbackStringIO(StringIO): - def __init__(self, num, th, d, buf = ''): +class CallbackStringIO(StringIO): + + def __init__(self, num, th, d, buf=''): # Force self.buf to be a string or unicode if not isinstance(buf, basestring): buf = str(buf) @@ -36,82 +35,82 @@ def __init__(self, num, th, d, buf = ''): self.pos = 0 self.closed = False self.softspace = 0 - self.th=th - self.num=num - self.d=d - self.total=self.len*self.th - + self.th = th + self.num = num + self.d = d + self.total = self.len*self.th + def read(self, n=10240): next = StringIO.read(self, n) #if 'done' in self.d: # return - - self.d[self.num]=self.pos - down=0 + + self.d[self.num] = self.pos + down = 0 for i in range(self.th): - down=down+self.d.get(i, 0) - if self.num==0: + down = down + self.d.get(i, 0) + if self.num == 0: percent = float(down) / (self.total) percent = round(percent*100, 2) print_debug("Uploaded %d of %d bytes (%0.2f%%) in %d threads\r" % - (down, self.total, percent, self.th)) + (down, self.total, percent, self.th)) #if down >= self.total: # print_debug('\n') # self.d['done']=1 return next - + def __len__(self): return self.len class TeSpeed: - def __init__(self, server = "", numTop = 0, servercount = 3, store = False, suppress = False, unit = False, chunksize=10240): + def __init__(self, server="", numTop=0, servercount=3, store=False, suppress=False, unit=False, chunksize=10240): self.headers = { - 'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', - 'User-Agent' : 'Mozilla/5.0 (X11; Linux x86_64; rv:11.0) Gecko/20100101 Firefox/11.0', - 'Accept-Language' : 'en-us,en;q=0.5', - 'Connection' : 'keep-alive', - 'Accept-Encoding' : 'gzip, deflate', + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', + 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:11.0) Gecko/20100101 Firefox/11.0', + 'Accept-Language': 'en-us,en;q=0.5', + 'Connection': 'keep-alive', + 'Accept-Encoding': 'gzip, deflate', #'Referer' : 'http://c.speedtest.net/flash/speedtest.swf?v=301256', } - self.num_servers=servercount; - self.servers=[] + self.num_servers = servercount + self.servers = [] if server != "": - self.servers=[server] - - self.server=server - self.down_speed=-1 - self.up_speed=-1 - self.latencycount=10 - self.bestServers=5 - - self.units="Mbit" - self.unit=0 - - self.chunksize=chunksize - + self.servers = [server] + + self.server = server + self.down_speed = -1 + self.up_speed = -1 + self.latencycount = 10 + self.bestServers = 5 + + self.units = "Mbit" + self.unit = 0 + + self.chunksize = chunksize + if unit: - self.units="MiB" - self.unit=1 + self.units = "MiB" + self.unit = 1 - self.store=store - self.suppress=suppress + self.store = store + self.suppress = suppress if store: print_debug("Printing CSV formated results to STDOUT.\n") - self.numTop=int(numTop) + self.numTop = int(numTop) #~ self.downList=['350x350', '500x500', '750x750', '1000x1000', #~ '1500x1500', '2000x2000', '2000x2000', '2500x2500', '3000x3000', #~ '3500x3500', '4000x4000', '4000x4000', '4000x4000', '4000x4000'] - #~ self.upSizes=[1024*256, 1024*256, 1024*512, 1024*512, - #~ 1024*1024, 1024*1024, 1024*1024*2, 1024*1024*2, + #~ self.upSizes=[1024*256, 1024*256, 1024*512, 1024*512, + #~ 1024*1024, 1024*1024, 1024*1024*2, 1024*1024*2, #~ 1024*1024*2, 1024*1024*2] - self.downList=[ + self.downList = [ '350x350', '350x350', '500x500', '500x500', '750x750', '750x750', '1000x1000', '1500x1500', '2000x2000', '2500x2500', '3000x3000','3500x3500','4000x4000','1000x1000','1000x1000','1000x1000','1000x1000','1000x1000','1000x1000','1000x1000', @@ -127,7 +126,7 @@ def __init__(self, server = "", numTop = 0, servercount = 3, store = False, supp # '1500x1500', '2000x2000', '2000x2000', '2500x2500', '3000x3000', # '3500x3500', '4000x4000', '4000x4000', '4000x4000', '4000x4000' #] - self.upSizes=[ + self.upSizes = [ 1024*256, 1024*256, 1024*512, 1024*512, 1024*1024, 1024*1024, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*512, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, @@ -140,65 +139,62 @@ def __init__(self, server = "", numTop = 0, servercount = 3, store = False, supp 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, -# 1024*1024, 1024*1024, 1024*1024*2, 1024*1024*2, +# 1024*1024, 1024*1024, 1024*1024*2, 1024*1024*2, # 1024*1024*2, 1024*1024*2] ] - self.postData="" + self.postData = "" self.TestSpeed() - def Distance(self, one, two): - #Calculate the great circle distance between two points + #Calculate the great circle distance between two points #on the earth specified in decimal degrees (haversine formula) #(http://stackoverflow.com/posts/4913653/revisions) - # convert decimal degrees to radians + # convert decimal degrees to radians lon1, lat1, lon2, lat2 = map(radians, [one[0], one[1], two[0], two[1]]) - # haversine formula - dlon = lon2 - lon1 - dlat = lat2 - lat1 + # haversine formula + dlon = lon2 - lon1 + dlat = lat2 - lat1 a = sin(dlat/2)**2 + cos(lat1) * cos(lat2) * sin(dlon/2)**2 - c = 2 * asin(sqrt(a)) + c = 2 * asin(sqrt(a)) km = 6367 * c - return km - + return km def Closest(self, center, points, num=5): # Returns object that is closest to center - closest={} + closest = {} for p in range(len(points)): now = self.Distance(center, [points[p]['lat'], points[p]['lon']]) - points[p]['distance']=now + points[p]['distance'] = now while True: if now in closest: - now=now+00.1 + now = now + 00.1 else: break - closest[now]=points[p] - n=0 - ret=[] + closest[now] = points[p] + n = 0 + ret = [] for key in sorted(closest): ret.append(closest[key]) - n+=1 - if n >= num and num!=0: + n += 1 + if n >= num and num != 0: break return ret - def TestLatency(self, servers): # Finding servers with lowest latency print_debug("Testing latency...\n") po = [] for server in servers: - now=self.TestSingleLatency(server['url']+"latency.txt?x=" + str( time.time() ))*1000 - now=now/2 # Evil hack or just pure stupidity? Nobody knows... + now = self.TestSingleLatency(server['url'] + "latency.txt?x=" + str(time.time()))*1000 + now = now / 2 # Evil hack or just pure stupidity? Nobody knows... if now == -1 or now == 0: continue - print_debug("%0.0f ms latency for %s (%s, %s, %s) [%0.2f km]\n" % - (now, server['url'], server['sponsor'], server['name'], server['country'], server['distance'])) + print_debug("%0.0f ms latency for %s (%s, %s, %s) [%0.2f km]\n" % + (now, server['url'], server['sponsor'], server['name'], server['country'], server['distance'])) - server['latency']=now + server['latency'] = now # Pick specified ammount of servers with best latency for testing if int(len(po)) < int(self.num_servers): @@ -215,91 +211,86 @@ def TestLatency(self, servers): #if cur['latency'] if largest >= 0: - po[largest]=server + po[largest] = server return po - def TestSingleLatency(self, dest_addr): # Checking latency for single server # Does that by loading latency.txt (empty page) - request=self.GetRequest(dest_addr) - - averagetime=0 - total=0 + request = self.GetRequest(dest_addr) + + averagetime = 0 + total = 0 for i in range(self.latencycount): - error=0 + error = 0 startTime = time.time() try: - response = urllib2.urlopen(request, timeout = 5) - except urllib2.URLError, e: - error=1 + urllib2.urlopen(request, timeout=5) + except urllib2.URLError: + error = 1 - if error==0: + if error == 0: averagetime = averagetime + (time.time() - startTime) - total=total+1 - - if total==0: - return False + total = total + 1 - return averagetime/total + if total == 0: + return False + return averagetime / total def GetRequest(self, uri): # Generates a GET request to be used with urlopen - req = urllib2.Request(uri, headers = self.headers) + req = urllib2.Request(uri, headers=self.headers) return req - def PostRequest(self, uri, stream): # Generate a POST request to be used with urlopen - req = urllib2.Request(uri, stream, headers = self.headers) + req = urllib2.Request(uri, stream, headers=self.headers) return req - def ChunkReport(self, bytes_so_far, chunk_size, total_size, num, th, d, w): # Receiving status update from download thread - - if w==1: + + if w == 1: return - d[num]=bytes_so_far - down=0 + d[num] = bytes_so_far + down = 0 for i in range(th): - down=down+d.get(i, 0) + down = down + d.get(i, 0) - if num==0 or down >= total_size*th: + if num == 0 or down >= total_size * th: - percent = float(down) / (total_size*th) - percent = round(percent*100, 2) + percent = float(down) / (total_size * th) + percent = round(percent * 100, 2) print_debug("Downloaded %d of %d bytes (%0.2f%%) in %d threads\r" % - (down, total_size*th, percent, th)) + (down, total_size*th, percent, th)) #if down >= total_size*th: # print_debug('\n') - def ChunkRead(self, response, num, th, d, w=0, chunk_size=False, report_hook=None): #print_debug("Thread num %d %d %d starting to report\n" % (th, num, d)) if not chunk_size: - chunk_size=self.chunksize + chunk_size = self.chunksize - if(w==1): - return [0,0,0] + if w == 1: + return [0, 0, 0] total_size = response.info().getheader('Content-Length').strip() total_size = int(total_size) bytes_so_far = 0 - start=0 + start = 0 while 1: - chunk=0 + chunk = 0 if start == 0: #print_debug("Started receiving data\n") chunk = response.read(1) start = time.time() - + else: chunk = response.read(chunk_size) if not chunk: @@ -309,20 +300,19 @@ def ChunkRead(self, response, num, th, d, w=0, chunk_size=False, report_hook=Non report_hook(bytes_so_far, chunk_size, total_size, num, th, d, w) end = time.time() - return [ bytes_so_far, start, end ] - + return [bytes_so_far, start, end] def AsyncGet(self, conn, uri, num, th, d): - request=self.GetRequest(uri) - - start=0 - end=0 - size=0 - + request = self.GetRequest(uri) + + start = 0 + end = 0 + size = 0 + try: - response = urllib2.urlopen(request, timeout = 30); - size, start, end=self.ChunkRead(response, num, th, d, report_hook=self.ChunkReport) + response = urllib2.urlopen(request, timeout=30) + size, start, end = self.ChunkRead(response, num, th, d, report_hook=self.ChunkReport) #except urllib2.URLError, e: # print_debug("Failed downloading.\n") except: @@ -335,19 +325,18 @@ def AsyncGet(self, conn, uri, num, th, d): conn.send([size, start, end]) conn.close() - def AsyncPost(self, conn, uri, num, th, d): - postlen=len(self.postData) + postlen = len(self.postData) stream = CallbackStringIO(num, th, d, self.postData) - request=self.PostRequest(uri, stream) + request = self.PostRequest(uri, stream) - start=0 - end=0 + start = 0 + end = 0 try: - response = urllib2.urlopen(request, timeout = 30); - size, start, end=self.ChunkRead(response, num, th, d, 1, report_hook=self.ChunkReport) - #except urllib2.URLError, e: + response = urllib2.urlopen(request, timeout=30) + size, start, end = self.ChunkRead(response, num, th, d, 1, report_hook=self.ChunkReport) + #except urllib2.URLError: # print_debug("Failed uploading.\n") except: print_debug(' \r') @@ -359,78 +348,74 @@ def AsyncPost(self, conn, uri, num, th, d): conn.send([postlen, start, end]) conn.close() - def LoadConfig(self): # Load the configuration file print_debug("Loading speedtest configuration...\n") - uri = "http://speedtest.net/speedtest-config.php?x=" + str( time.time() ) - request=self.GetRequest(uri) + uri = "http://speedtest.net/speedtest-config.php?x=" + str(time.time()) + request = self.GetRequest(uri) response = urllib2.urlopen(request) - # Load etree from XML data config = etree.fromstring(self.DecompressResponse(response)) - - ip=config.find("client").attrib['ip'] - isp=config.find("client").attrib['isp'] - lat=float(config.find("client").attrib['lat']) - lon=float(config.find("client").attrib['lon']) - + + ip = config.find("client").attrib['ip'] + isp = config.find("client").attrib['isp'] + lat = float(config.find("client").attrib['lat']) + lon = float(config.find("client").attrib['lon']) + print_debug("IP: %s; Lat: %f; Lon: %f; ISP: %s\n" % (ip, lat, lon, isp)) - - return { 'ip': ip, 'lat': lat, 'lon': lon, 'isp': isp } - + + return {'ip': ip, 'lat': lat, 'lon': lon, 'isp': isp} def LoadServers(self): # Load server list print_debug("Loading server list...\n") - uri = "http://speedtest.net/speedtest-servers.php?x=" + str( time.time() ) - request=self.GetRequest(uri) - response = urllib2.urlopen(request); + uri = "http://speedtest.net/speedtest-servers.php?x=" + str(time.time()) + request = self.GetRequest(uri) + response = urllib2.urlopen(request) # Load etree from XML data servers_xml = etree.fromstring(self.DecompressResponse(response)) - servers=servers_xml.find("servers").findall("server") - server_list=[] + servers = servers_xml.find("servers").findall("server") + server_list = [] for server in servers: server_list.append({ - 'lat': float(server.attrib['lat']), - 'lon': float(server.attrib['lon']), - 'url': server.attrib['url'].rsplit('/', 1)[0] + '/', - #'url2': server.attrib['url2'].rsplit('/', 1)[0] + '/', - 'name': server.attrib['name'], - 'country': server.attrib['country'], - 'sponsor': server.attrib['sponsor'], - 'id': server.attrib['id'], + 'lat': float(server.attrib['lat']), + 'lon': float(server.attrib['lon']), + 'url': server.attrib['url'].rsplit('/', 1)[0] + '/', + #'url2': server.attrib['url2'].rsplit('/', 1)[0] + '/', + 'name': server.attrib['name'], + 'country': server.attrib['country'], + 'sponsor': server.attrib['sponsor'], + 'id': server.attrib['id'], }) return server_list - def DecompressResponse(sefl, response): # Decompress gzipped response data = StringIO(response.read()) gzipper = gzip.GzipFile(fileobj=data) return gzipper.read() - def FindBestServer(self): print_debug("Looking for closest and best server...\n") - best=self.TestLatency(self.Closest([self.config['lat'], self.config['lon']], self.server_list, self.bestServers)) + best = self.TestLatency(self.Closest([self.config['lat'], self.config['lon']], + self.server_list, self.bestServers)) for server in best: self.servers.append(server['url']) def AsyncRequest(self, url, num, upload=0): - connections=[] - d=Manager().dict() - start=time.time() - for i in range(num): - full_url=self.servers[i % len(self.servers)]+url + connections = [] + d = Manager().dict() + start = time.time() + for i in xrange(num): + full_url = self.servers[i % len(self.servers)] + url #print full_url - connection={} - connection['parent'], connection['child']= Pipe() - if upload==1: + connection = {} + connection['parent'], connection['child'] = Pipe() + if upload == 1: connection['connection'] = Process(target=self.AsyncPost, args=(connection['child'], full_url, i, num, d)) else: connection['connection'] = Process(target=self.AsyncGet, args=(connection['child'], full_url, i, num, d)) @@ -438,133 +423,128 @@ def AsyncRequest(self, url, num, upload=0): connections.append(connection) for c in range(num): - connections[c]['size'], connections[c]['start'], connections[c]['end']=connections[c]['parent'].recv() + connections[c]['size'], connections[c]['start'], connections[c]['end'] = connections[c]['parent'].recv() connections[c]['connection'].join() - end=time.time() - + end = time.time() + print_debug(' \r') - sizes=0 + sizes = 0 #tspeed=0 for c in range(num): if connections[c]['end'] is not False: #tspeed=tspeed+(connections[c]['size']/(connections[c]['end']-connections[c]['start'])) - sizes=sizes+connections[c]['size'] - + sizes = sizes + connections[c]['size'] + # Using more precise times for downloads - if upload==0: - if c==0: - start=connections[c]['start'] - end=connections[c]['end'] + if upload == 0: + if c == 0: + start = connections[c]['start'] + end = connections[c]['end'] else: if connections[c]['start'] < start: - start=connections[c]['start'] + start = connections[c]['start'] if connections[c]['end'] > end: - end=connections[c]['end'] + end = connections[c]['end'] - took=end-start + took = end - start return [sizes, took] def TestUpload(self): # Testing upload speed - url="upload.php?x=" + str( time.time() ) + url = "upload.php?x=" + str(time.time()) - sizes, took=[0,0] - data="" + sizes, took = [0, 0] + data = "" for i in range(0, len(self.upSizes)): if len(data) == 0 or self.upSizes[i] != self.upSizes[i-1]: #print_debug("Generating new string to upload. Length: %d\n" % (self.upSizes[i])) - data=''.join("1" for x in xrange(self.upSizes[i])) - self.postData=urllib.urlencode({'upload6': data }) - - if i<2: - thrds=1 - elif i<5: - thrds=2 - elif i<7: - thrds=2 - elif i<10: - thrds=3 - elif i<25: - thrds=6 - elif i<45: - thrds=4 - elif i<65: - thrds=3 + data = ''.join("1" for x in xrange(self.upSizes[i])) + self.postData = urllib.urlencode({'upload6': data}) + + if i < 2: + thrds = 1 + elif i < 5: + thrds = 2 + elif i < 7: + thrds = 2 + elif i < 10: + thrds = 3 + elif i < 25: + thrds = 6 + elif i < 45: + thrds = 4 + elif i < 65: + thrds = 3 else: - thrds=2 - - sizes, took=self.AsyncRequest(url, thrds, 1) + thrds = 2 + + sizes, took = self.AsyncRequest(url, thrds, 1) #sizes, took=self.AsyncRequest(url, (i<4 and 1 or (i<6 and 2 or (i<6 and 4 or 8))), 1) - if sizes==0: + if sizes == 0: continue - size=self.SpeedConversion(sizes) - speed=size/took - print_debug("Upload size: %0.2f MiB; Uploaded in %0.2f s\n" % - (size, took)) - print_debug("\033[92mUpload speed: %0.2f %s/s\033[0m\n" % - (speed, self.units)) - - if self.up_speed5: + size = self.SpeedConversion(sizes) + speed = size / took + print_debug("Upload size: %0.2f MiB; Uploaded in %0.2f s\n" % (size, took)) + print_debug("\033[92mUpload speed: %0.2f %s/s\033[0m\n" % (speed, self.units)) + + if self.up_speed < speed: + self.up_speed = speed + + if took > 5: break - + #print_debug("Upload size: %0.2f MiB; Uploaded in %0.2f s\n" % (self.SpeedConversion(sizes), took)) #print_debug("Upload speed: %0.2f MiB/s\n" % (self.SpeedConversion(sizes)/took)) def SpeedConversion(self, data): - if self.unit==1: - result=(float(data)/1024/1024) + if self.unit == 1: + result = (float(data) / 1024 / 1024) else: - result=(float(data)/1024/1024)*1.048576*8 + result = (float(data) / 1024 / 1024) * 1.048576 * 8 return result def TestDownload(self): # Testing download speed - sizes, took=[0,0] + sizes, took = [0, 0] for i in range(0, len(self.downList)): - url="random"+self.downList[i]+".jpg?x=" + str( time.time() ) + "&y=3" - - - if i<2: - thrds=1 - elif i<5: - thrds=2 - elif i<11: - thrds=2 - elif i<13: - thrds=4 - elif i<25: - thrds=2 - elif i<45: - thrds=3 - elif i<65: - thrds=2 + url = "random" + self.downList[i] + ".jpg?x=" + str(time.time()) + "&y=3" + + if i < 2: + thrds = 1 + elif i < 5: + thrds = 2 + elif i < 11: + thrds = 2 + elif i < 13: + thrds = 4 + elif i < 25: + thrds = 2 + elif i < 45: + thrds = 3 + elif i < 65: + thrds = 2 else: - thrds=2 - - sizes, took=self.AsyncRequest(url, thrds ) + thrds = 2 + + sizes, took = self.AsyncRequest(url, thrds) #sizes, took=self.AsyncRequest(url, (i<1 and 2 or (i<6 and 4 or (i<10 and 6 or 8))) ) - if sizes==0: + if sizes == 0: continue - size=self.SpeedConversion(sizes) - speed=size/took - print_debug("Download size: %0.2f MiB; Downloaded in %0.2f s\n" % - (size, took)) - print_debug("\033[91mDownload speed: %0.2f %s/s\033[0m\n" % - (speed, self.units)) + size = self.SpeedConversion(sizes) + speed = size / took + print_debug("Download size: %0.2f MiB; Downloaded in %0.2f s\n" % (size, took)) + print_debug("\033[91mDownload speed: %0.2f %s/s\033[0m\n" % (speed, self.units)) - if self.down_speed5: + if took > 5: break #print_debug("Download size: %0.2f MiB; Downloaded in %0.2f s\n" % (self.SpeedConversion(sizes), took)) @@ -572,15 +552,15 @@ def TestDownload(self): def TestSpeed(self): - if self.server=='list-servers': - self.config=self.LoadConfig() - self.server_list=self.LoadServers() + if self.server == 'list-servers': + self.config = self.LoadConfig() + self.server_list = self.LoadServers() self.ListServers(self.numTop) return - + if self.server == '': - self.config=self.LoadConfig() - self.server_list=self.LoadServers() + self.config = self.LoadConfig() + self.server_list = self.LoadServers() self.FindBestServer() self.TestDownload() @@ -589,50 +569,53 @@ def TestSpeed(self): print_result("%0.2f,%0.2f,\"%s\",\"%s\"\n" % (self.down_speed, self.up_speed, self.units, self.servers)) def ListServers(self, num=0): - - allSorted=self.Closest([self.config['lat'], self.config['lon']], self.server_list, num) + + allSorted = self.Closest([self.config['lat'], self.config['lon']], self.server_list, num) for i in range(0, len(allSorted)): - print_result("%s. %s (%s, %s, %s) [%0.2f km]\n" % - (i+1, allSorted[i]['url'], allSorted[i]['sponsor'], allSorted[i]['name'], allSorted[i]['country'], allSorted[i]['distance'])) + print_result("%s. %s (%s, %s, %s) [%0.2f km]\n" % + (i + 1, allSorted[i]['url'], allSorted[i]['sponsor'], allSorted[i]['name'], + allSorted[i]['country'], allSorted[i]['distance'])) + def print_debug(string): - if args.suppress!=True: + if not args.suppress: sys.stderr.write(string.encode('utf8')) - #return + def print_result(string): - if args.store==True: + if args.store: sys.stdout.write(string.encode('utf8')) - #return -# Thx to Ryan Sears for http://bit.ly/17HhSli + +# Thanks to Ryan Sears for http://bit.ly/17HhSli def set_proxy(typ=socks.PROXY_TYPE_SOCKS4, host="127.0.0.1", port=9050): socks.setdefaultproxy(typ, host, port) socket.socket = socks.socksocket + def main(args): if args.use_proxy: - if args.use_proxy==5: + if args.use_proxy == 5: set_proxy(typ=socks.PROXY_TYPE_SOCKS5, host=args.proxy_host, port=args.proxy_port) else: set_proxy(typ=socks.PROXY_TYPE_SOCKS4, host=args.proxy_host, port=args.proxy_port) if args.listservers: - args.store=True + args.store = True - if args.listservers!=True and args.server=='' and args.store!=True: + if not args.listservers and args.server == '' and not args.store: print_debug("Getting ready. Use parameter -h or --help to see available features.\n") else: print_debug("Getting ready\n") try: - t=TeSpeed( - args.listservers and 'list-servers' or args.server, - args.listservers, args.servercount, - args.store and True or False, - args.suppress and True or False, - args.unit and True or False, + TeSpeed( + args.listservers and 'list-servers' or args.server, + args.listservers, args.servercount, + args.store and True or False, + args.suppress and True or False, + args.unit and True or False, chunksize=args.chunksize ) except (KeyboardInterrupt, SystemExit): From fb0113d897fb163e16b87d63141709da3b7c2dcd Mon Sep 17 00:00:00 2001 From: David Fischer Date: Fri, 23 May 2014 22:58:09 +0200 Subject: [PATCH 03/24] Convert to new style class --- tespeed.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tespeed.py b/tespeed.py index 1fa820a..277f4f0 100755 --- a/tespeed.py +++ b/tespeed.py @@ -65,7 +65,7 @@ def __len__(self): return self.len -class TeSpeed: +class TeSpeed(object): def __init__(self, server="", numTop=0, servercount=3, store=False, suppress=False, unit=False, chunksize=10240): From fed5a1f312f4ab078360ddce40d4aa75990f0d87 Mon Sep 17 00:00:00 2001 From: David Fischer Date: Fri, 23 May 2014 23:09:48 +0200 Subject: [PATCH 04/24] Rename methods to follow recommendations of PEP-8 --- tespeed.py | 106 ++++++++++++++++++++++++++--------------------------- 1 file changed, 53 insertions(+), 53 deletions(-) diff --git a/tespeed.py b/tespeed.py index 277f4f0..36e85a5 100755 --- a/tespeed.py +++ b/tespeed.py @@ -144,9 +144,9 @@ def __init__(self, server="", numTop=0, servercount=3, store=False, suppress=Fal ] self.postData = "" - self.TestSpeed() + self.test_speed() - def Distance(self, one, two): + def distance(self, one, two): #Calculate the great circle distance between two points #on the earth specified in decimal degrees (haversine formula) #(http://stackoverflow.com/posts/4913653/revisions) @@ -161,11 +161,11 @@ def Distance(self, one, two): km = 6367 * c return km - def Closest(self, center, points, num=5): + def closest(self, center, points, num=5): # Returns object that is closest to center closest = {} for p in range(len(points)): - now = self.Distance(center, [points[p]['lat'], points[p]['lon']]) + now = self.distance(center, [points[p]['lat'], points[p]['lon']]) points[p]['distance'] = now while True: if now in closest: @@ -182,12 +182,12 @@ def Closest(self, center, points, num=5): break return ret - def TestLatency(self, servers): + def test_latency(self, servers): # Finding servers with lowest latency print_debug("Testing latency...\n") po = [] for server in servers: - now = self.TestSingleLatency(server['url'] + "latency.txt?x=" + str(time.time()))*1000 + now = self.test_single_latency(server['url'] + "latency.txt?x=" + str(time.time()))*1000 now = now / 2 # Evil hack or just pure stupidity? Nobody knows... if now == -1 or now == 0: continue @@ -215,7 +215,7 @@ def TestLatency(self, servers): return po - def TestSingleLatency(self, dest_addr): + def test_single_latency(self, dest_addr): # Checking latency for single server # Does that by loading latency.txt (empty page) request = self.GetRequest(dest_addr) @@ -239,17 +239,17 @@ def TestSingleLatency(self, dest_addr): return averagetime / total - def GetRequest(self, uri): + def get_request(self, uri): # Generates a GET request to be used with urlopen req = urllib2.Request(uri, headers=self.headers) return req - def PostRequest(self, uri, stream): + def post_request(self, uri, stream): # Generate a POST request to be used with urlopen req = urllib2.Request(uri, stream, headers=self.headers) return req - def ChunkReport(self, bytes_so_far, chunk_size, total_size, num, th, d, w): + def chunk_report(self, bytes_so_far, chunk_size, total_size, num, th, d, w): # Receiving status update from download thread if w == 1: @@ -270,7 +270,7 @@ def ChunkReport(self, bytes_so_far, chunk_size, total_size, num, th, d, w): #if down >= total_size*th: # print_debug('\n') - def ChunkRead(self, response, num, th, d, w=0, chunk_size=False, report_hook=None): + def chunk_read(self, response, num, th, d, w=0, chunk_size=False, report_hook=None): #print_debug("Thread num %d %d %d starting to report\n" % (th, num, d)) if not chunk_size: @@ -302,9 +302,9 @@ def ChunkRead(self, response, num, th, d, w=0, chunk_size=False, report_hook=Non return [bytes_so_far, start, end] - def AsyncGet(self, conn, uri, num, th, d): + def async_get(self, conn, uri, num, th, d): - request = self.GetRequest(uri) + request = self.get_request(uri) start = 0 end = 0 @@ -312,7 +312,7 @@ def AsyncGet(self, conn, uri, num, th, d): try: response = urllib2.urlopen(request, timeout=30) - size, start, end = self.ChunkRead(response, num, th, d, report_hook=self.ChunkReport) + size, start, end = self.chunk_read(response, num, th, d, report_hook=self.chunk_report) #except urllib2.URLError, e: # print_debug("Failed downloading.\n") except: @@ -325,17 +325,17 @@ def AsyncGet(self, conn, uri, num, th, d): conn.send([size, start, end]) conn.close() - def AsyncPost(self, conn, uri, num, th, d): + def async_post(self, conn, uri, num, th, d): postlen = len(self.postData) stream = CallbackStringIO(num, th, d, self.postData) - request = self.PostRequest(uri, stream) + request = self.post_request(uri, stream) start = 0 end = 0 try: response = urllib2.urlopen(request, timeout=30) - size, start, end = self.ChunkRead(response, num, th, d, 1, report_hook=self.ChunkReport) + size, start, end = self.chunk_read(response, num, th, d, 1, report_hook=self.chunk_report) #except urllib2.URLError: # print_debug("Failed uploading.\n") except: @@ -348,15 +348,15 @@ def AsyncPost(self, conn, uri, num, th, d): conn.send([postlen, start, end]) conn.close() - def LoadConfig(self): + def load_config(self): # Load the configuration file print_debug("Loading speedtest configuration...\n") uri = "http://speedtest.net/speedtest-config.php?x=" + str(time.time()) - request = self.GetRequest(uri) + request = self.get_request(uri) response = urllib2.urlopen(request) # Load etree from XML data - config = etree.fromstring(self.DecompressResponse(response)) + config = etree.fromstring(self.decompress_response(response)) ip = config.find("client").attrib['ip'] isp = config.find("client").attrib['isp'] @@ -367,7 +367,7 @@ def LoadConfig(self): return {'ip': ip, 'lat': lat, 'lon': lon, 'isp': isp} - def LoadServers(self): + def load_servers(self): # Load server list print_debug("Loading server list...\n") uri = "http://speedtest.net/speedtest-servers.php?x=" + str(time.time()) @@ -375,7 +375,7 @@ def LoadServers(self): response = urllib2.urlopen(request) # Load etree from XML data - servers_xml = etree.fromstring(self.DecompressResponse(response)) + servers_xml = etree.fromstring(self.decompress_response(response)) servers = servers_xml.find("servers").findall("server") server_list = [] @@ -393,20 +393,20 @@ def LoadServers(self): return server_list - def DecompressResponse(sefl, response): + def decompress_response(sefl, response): # Decompress gzipped response data = StringIO(response.read()) gzipper = gzip.GzipFile(fileobj=data) return gzipper.read() - def FindBestServer(self): + def find_best_server(self): print_debug("Looking for closest and best server...\n") - best = self.TestLatency(self.Closest([self.config['lat'], self.config['lon']], - self.server_list, self.bestServers)) + best = self.test_latency(self.closest([self.config['lat'], self.config['lon']], + self.server_list, self.bestServers)) for server in best: self.servers.append(server['url']) - def AsyncRequest(self, url, num, upload=0): + def async_request(self, url, num, upload=0): connections = [] d = Manager().dict() start = time.time() @@ -416,9 +416,9 @@ def AsyncRequest(self, url, num, upload=0): connection = {} connection['parent'], connection['child'] = Pipe() if upload == 1: - connection['connection'] = Process(target=self.AsyncPost, args=(connection['child'], full_url, i, num, d)) + connection['connection'] = Process(target=self.async_post, args=(connection['child'], full_url, i, num, d)) else: - connection['connection'] = Process(target=self.AsyncGet, args=(connection['child'], full_url, i, num, d)) + connection['connection'] = Process(target=self.async_get, args=(connection['child'], full_url, i, num, d)) connection['connection'].start() connections.append(connection) @@ -452,7 +452,7 @@ def AsyncRequest(self, url, num, upload=0): return [sizes, took] - def TestUpload(self): + def test_upload(self): # Testing upload speed url = "upload.php?x=" + str(time.time()) @@ -482,12 +482,12 @@ def TestUpload(self): else: thrds = 2 - sizes, took = self.AsyncRequest(url, thrds, 1) - #sizes, took=self.AsyncRequest(url, (i<4 and 1 or (i<6 and 2 or (i<6 and 4 or 8))), 1) + sizes, took = self.async_request(url, thrds, 1) + #sizes, took=self.async_request(url, (i<4 and 1 or (i<6 and 2 or (i<6 and 4 or 8))), 1) if sizes == 0: continue - size = self.SpeedConversion(sizes) + size = self.speed_conversion(sizes) speed = size / took print_debug("Upload size: %0.2f MiB; Uploaded in %0.2f s\n" % (size, took)) print_debug("\033[92mUpload speed: %0.2f %s/s\033[0m\n" % (speed, self.units)) @@ -498,17 +498,17 @@ def TestUpload(self): if took > 5: break - #print_debug("Upload size: %0.2f MiB; Uploaded in %0.2f s\n" % (self.SpeedConversion(sizes), took)) - #print_debug("Upload speed: %0.2f MiB/s\n" % (self.SpeedConversion(sizes)/took)) + #print_debug("Upload size: %0.2f MiB; Uploaded in %0.2f s\n" % (self.speed_conversion(sizes), took)) + #print_debug("Upload speed: %0.2f MiB/s\n" % (self.speed_conversion(sizes)/took)) - def SpeedConversion(self, data): + def speed_conversion(self, data): if self.unit == 1: result = (float(data) / 1024 / 1024) else: result = (float(data) / 1024 / 1024) * 1.048576 * 8 return result - def TestDownload(self): + def test_download(self): # Testing download speed sizes, took = [0, 0] for i in range(0, len(self.downList)): @@ -531,12 +531,12 @@ def TestDownload(self): else: thrds = 2 - sizes, took = self.AsyncRequest(url, thrds) - #sizes, took=self.AsyncRequest(url, (i<1 and 2 or (i<6 and 4 or (i<10 and 6 or 8))) ) + sizes, took = self.async_request(url, thrds) + #sizes, took=self.async_request(url, (i<1 and 2 or (i<6 and 4 or (i<10 and 6 or 8))) ) if sizes == 0: continue - size = self.SpeedConversion(sizes) + size = self.speed_conversion(sizes) speed = size / took print_debug("Download size: %0.2f MiB; Downloaded in %0.2f s\n" % (size, took)) print_debug("\033[91mDownload speed: %0.2f %s/s\033[0m\n" % (speed, self.units)) @@ -547,30 +547,30 @@ def TestDownload(self): if took > 5: break - #print_debug("Download size: %0.2f MiB; Downloaded in %0.2f s\n" % (self.SpeedConversion(sizes), took)) - #print_debug("Download speed: %0.2f %s/s\n" % (self.SpeedConversion(sizes)/took, self.units)) + #print_debug("Download size: %0.2f MiB; Downloaded in %0.2f s\n" % (self.speed_conversion(sizes), took)) + #print_debug("Download speed: %0.2f %s/s\n" % (self.speed_conversion(sizes)/took, self.units)) - def TestSpeed(self): + def test_speed(self): if self.server == 'list-servers': - self.config = self.LoadConfig() - self.server_list = self.LoadServers() - self.ListServers(self.numTop) + self.config = self.load_config() + self.server_list = self.load_servers() + self.list_servers(self.numTop) return if self.server == '': - self.config = self.LoadConfig() - self.server_list = self.LoadServers() - self.FindBestServer() + self.config = self.load_config() + self.server_list = self.load_servers() + self.find_best_server() - self.TestDownload() - self.TestUpload() + self.test_download() + self.test_upload() print_result("%0.2f,%0.2f,\"%s\",\"%s\"\n" % (self.down_speed, self.up_speed, self.units, self.servers)) - def ListServers(self, num=0): + def list_servers(self, num=0): - allSorted = self.Closest([self.config['lat'], self.config['lon']], self.server_list, num) + allSorted = self.closest([self.config['lat'], self.config['lon']], self.server_list, num) for i in range(0, len(allSorted)): print_result("%s. %s (%s, %s, %s) [%0.2f km]\n" % From dff3421ac5336b5009c81804171520b203657d69 Mon Sep 17 00:00:00 2001 From: David Fischer Date: Fri, 23 May 2014 23:17:11 +0200 Subject: [PATCH 05/24] Use single quotes for string literals, personal taste --- tespeed.py | 95 +++++++++++++++++++++++++++--------------------------- 1 file changed, 47 insertions(+), 48 deletions(-) diff --git a/tespeed.py b/tespeed.py index 36e85a5..ac1cc0c 100755 --- a/tespeed.py +++ b/tespeed.py @@ -52,8 +52,7 @@ def read(self, n=10240): if self.num == 0: percent = float(down) / (self.total) percent = round(percent*100, 2) - print_debug("Uploaded %d of %d bytes (%0.2f%%) in %d threads\r" % - (down, self.total, percent, self.th)) + print_debug('Uploaded %d of %d bytes (%0.2f%%) in %d threads\r' % (down, self.total, percent, self.th)) #if down >= self.total: # print_debug('\n') @@ -67,7 +66,7 @@ def __len__(self): class TeSpeed(object): - def __init__(self, server="", numTop=0, servercount=3, store=False, suppress=False, unit=False, chunksize=10240): + def __init__(self, server='', numTop=0, servercount=3, store=False, suppress=False, unit=False, chunksize=10240): self.headers = { 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', @@ -80,7 +79,7 @@ def __init__(self, server="", numTop=0, servercount=3, store=False, suppress=Fal self.num_servers = servercount self.servers = [] - if server != "": + if server != '': self.servers = [server] self.server = server @@ -89,19 +88,19 @@ def __init__(self, server="", numTop=0, servercount=3, store=False, suppress=Fal self.latencycount = 10 self.bestServers = 5 - self.units = "Mbit" + self.units = 'Mbit' self.unit = 0 self.chunksize = chunksize if unit: - self.units = "MiB" + self.units = 'MiB' self.unit = 1 self.store = store self.suppress = suppress if store: - print_debug("Printing CSV formated results to STDOUT.\n") + print_debug('Printing CSV formated results to STDOUT.\n') self.numTop = int(numTop) #~ self.downList=['350x350', '500x500', '750x750', '1000x1000', #~ '1500x1500', '2000x2000', '2000x2000', '2500x2500', '3000x3000', @@ -143,7 +142,7 @@ def __init__(self, server="", numTop=0, servercount=3, store=False, suppress=Fal # 1024*1024*2, 1024*1024*2] ] - self.postData = "" + self.postData = '' self.test_speed() def distance(self, one, two): @@ -184,14 +183,14 @@ def closest(self, center, points, num=5): def test_latency(self, servers): # Finding servers with lowest latency - print_debug("Testing latency...\n") + print_debug('Testing latency...\n') po = [] for server in servers: - now = self.test_single_latency(server['url'] + "latency.txt?x=" + str(time.time()))*1000 + now = self.test_single_latency(server['url'] + 'latency.txt?x=' + str(time.time()))*1000 now = now / 2 # Evil hack or just pure stupidity? Nobody knows... if now == -1 or now == 0: continue - print_debug("%0.0f ms latency for %s (%s, %s, %s) [%0.2f km]\n" % + print_debug('%0.0f ms latency for %s (%s, %s, %s) [%0.2f km]\n' % (now, server['url'], server['sponsor'], server['name'], server['country'], server['distance'])) server['latency'] = now @@ -264,14 +263,14 @@ def chunk_report(self, bytes_so_far, chunk_size, total_size, num, th, d, w): percent = float(down) / (total_size * th) percent = round(percent * 100, 2) - print_debug("Downloaded %d of %d bytes (%0.2f%%) in %d threads\r" % + print_debug('Downloaded %d of %d bytes (%0.2f%%) in %d threads\r' % (down, total_size*th, percent, th)) #if down >= total_size*th: # print_debug('\n') def chunk_read(self, response, num, th, d, w=0, chunk_size=False, report_hook=None): - #print_debug("Thread num %d %d %d starting to report\n" % (th, num, d)) + #print_debug('Thread num %d %d %d starting to report\n' % (th, num, d)) if not chunk_size: chunk_size = self.chunksize @@ -287,7 +286,7 @@ def chunk_read(self, response, num, th, d, w=0, chunk_size=False, report_hook=No while 1: chunk = 0 if start == 0: - #print_debug("Started receiving data\n") + #print_debug('Started receiving data\n') chunk = response.read(1) start = time.time() @@ -314,10 +313,10 @@ def async_get(self, conn, uri, num, th, d): response = urllib2.urlopen(request, timeout=30) size, start, end = self.chunk_read(response, num, th, d, report_hook=self.chunk_report) #except urllib2.URLError, e: - # print_debug("Failed downloading.\n") + # print_debug('Failed downloading.\n') except: print_debug(' \r') - print_debug("Failed downloading.\n") + print_debug('Failed downloading.\n') conn.send([0, 0, False]) conn.close() return @@ -337,10 +336,10 @@ def async_post(self, conn, uri, num, th, d): response = urllib2.urlopen(request, timeout=30) size, start, end = self.chunk_read(response, num, th, d, 1, report_hook=self.chunk_report) #except urllib2.URLError: - # print_debug("Failed uploading.\n") + # print_debug('Failed uploading.\n') except: print_debug(' \r') - print_debug("Failed uploading.\n") + print_debug('Failed uploading.\n') conn.send([0, 0, False]) conn.close() return @@ -350,33 +349,33 @@ def async_post(self, conn, uri, num, th, d): def load_config(self): # Load the configuration file - print_debug("Loading speedtest configuration...\n") - uri = "http://speedtest.net/speedtest-config.php?x=" + str(time.time()) + print_debug('Loading speedtest configuration...\n') + uri = 'http://speedtest.net/speedtest-config.php?x=' + str(time.time()) request = self.get_request(uri) response = urllib2.urlopen(request) # Load etree from XML data config = etree.fromstring(self.decompress_response(response)) - ip = config.find("client").attrib['ip'] - isp = config.find("client").attrib['isp'] - lat = float(config.find("client").attrib['lat']) - lon = float(config.find("client").attrib['lon']) + ip = config.find('client').attrib['ip'] + isp = config.find('client').attrib['isp'] + lat = float(config.find('client').attrib['lat']) + lon = float(config.find('client').attrib['lon']) - print_debug("IP: %s; Lat: %f; Lon: %f; ISP: %s\n" % (ip, lat, lon, isp)) + print_debug('IP: %s; Lat: %f; Lon: %f; ISP: %s\n' % (ip, lat, lon, isp)) return {'ip': ip, 'lat': lat, 'lon': lon, 'isp': isp} def load_servers(self): # Load server list - print_debug("Loading server list...\n") - uri = "http://speedtest.net/speedtest-servers.php?x=" + str(time.time()) + print_debug('Loading server list...\n') + uri = 'http://speedtest.net/speedtest-servers.php?x=' + str(time.time()) request = self.GetRequest(uri) response = urllib2.urlopen(request) # Load etree from XML data servers_xml = etree.fromstring(self.decompress_response(response)) - servers = servers_xml.find("servers").findall("server") + servers = servers_xml.find('servers').findall('server') server_list = [] for server in servers: @@ -400,7 +399,7 @@ def decompress_response(sefl, response): return gzipper.read() def find_best_server(self): - print_debug("Looking for closest and best server...\n") + print_debug('Looking for closest and best server...\n') best = self.test_latency(self.closest([self.config['lat'], self.config['lon']], self.server_list, self.bestServers)) for server in best: @@ -455,14 +454,14 @@ def async_request(self, url, num, upload=0): def test_upload(self): # Testing upload speed - url = "upload.php?x=" + str(time.time()) + url = 'upload.php?x=' + str(time.time()) sizes, took = [0, 0] - data = "" + data = '' for i in range(0, len(self.upSizes)): if len(data) == 0 or self.upSizes[i] != self.upSizes[i-1]: - #print_debug("Generating new string to upload. Length: %d\n" % (self.upSizes[i])) - data = ''.join("1" for x in xrange(self.upSizes[i])) + #print_debug('Generating new string to upload. Length: %d\n' % (self.upSizes[i])) + data = ''.join('1' for x in xrange(self.upSizes[i])) self.postData = urllib.urlencode({'upload6': data}) if i < 2: @@ -489,8 +488,8 @@ def test_upload(self): size = self.speed_conversion(sizes) speed = size / took - print_debug("Upload size: %0.2f MiB; Uploaded in %0.2f s\n" % (size, took)) - print_debug("\033[92mUpload speed: %0.2f %s/s\033[0m\n" % (speed, self.units)) + print_debug('Upload size: %0.2f MiB; Uploaded in %0.2f s\n' % (size, took)) + print_debug('\033[92mUpload speed: %0.2f %s/s\033[0m\n' % (speed, self.units)) if self.up_speed < speed: self.up_speed = speed @@ -498,8 +497,8 @@ def test_upload(self): if took > 5: break - #print_debug("Upload size: %0.2f MiB; Uploaded in %0.2f s\n" % (self.speed_conversion(sizes), took)) - #print_debug("Upload speed: %0.2f MiB/s\n" % (self.speed_conversion(sizes)/took)) + #print_debug('Upload size: %0.2f MiB; Uploaded in %0.2f s\n' % (self.speed_conversion(sizes), took)) + #print_debug('Upload speed: %0.2f MiB/s\n' % (self.speed_conversion(sizes)/took)) def speed_conversion(self, data): if self.unit == 1: @@ -512,7 +511,7 @@ def test_download(self): # Testing download speed sizes, took = [0, 0] for i in range(0, len(self.downList)): - url = "random" + self.downList[i] + ".jpg?x=" + str(time.time()) + "&y=3" + url = 'random' + self.downList[i] + '.jpg?x=' + str(time.time()) + '&y=3' if i < 2: thrds = 1 @@ -538,8 +537,8 @@ def test_download(self): size = self.speed_conversion(sizes) speed = size / took - print_debug("Download size: %0.2f MiB; Downloaded in %0.2f s\n" % (size, took)) - print_debug("\033[91mDownload speed: %0.2f %s/s\033[0m\n" % (speed, self.units)) + print_debug('Download size: %0.2f MiB; Downloaded in %0.2f s\n' % (size, took)) + print_debug('\033[91mDownload speed: %0.2f %s/s\033[0m\n' % (speed, self.units)) if self.down_speed < speed: self.down_speed = speed @@ -547,8 +546,8 @@ def test_download(self): if took > 5: break - #print_debug("Download size: %0.2f MiB; Downloaded in %0.2f s\n" % (self.speed_conversion(sizes), took)) - #print_debug("Download speed: %0.2f %s/s\n" % (self.speed_conversion(sizes)/took, self.units)) + #print_debug('Download size: %0.2f MiB; Downloaded in %0.2f s\n' % (self.speed_conversion(sizes), took)) + #print_debug('Download speed: %0.2f %s/s\n' % (self.speed_conversion(sizes)/took, self.units)) def test_speed(self): @@ -566,14 +565,14 @@ def test_speed(self): self.test_download() self.test_upload() - print_result("%0.2f,%0.2f,\"%s\",\"%s\"\n" % (self.down_speed, self.up_speed, self.units, self.servers)) + print_result('%0.2f,%0.2f,"%s","%s"\n' % (self.down_speed, self.up_speed, self.units, self.servers)) def list_servers(self, num=0): allSorted = self.closest([self.config['lat'], self.config['lon']], self.server_list, num) for i in range(0, len(allSorted)): - print_result("%s. %s (%s, %s, %s) [%0.2f km]\n" % + print_result('%s. %s (%s, %s, %s) [%0.2f km]\n' % (i + 1, allSorted[i]['url'], allSorted[i]['sponsor'], allSorted[i]['name'], allSorted[i]['country'], allSorted[i]['distance'])) @@ -589,7 +588,7 @@ def print_result(string): # Thanks to Ryan Sears for http://bit.ly/17HhSli -def set_proxy(typ=socks.PROXY_TYPE_SOCKS4, host="127.0.0.1", port=9050): +def set_proxy(typ=socks.PROXY_TYPE_SOCKS4, host='127.0.0.1', port=9050): socks.setdefaultproxy(typ, host, port) socket.socket = socks.socksocket @@ -606,9 +605,9 @@ def main(args): args.store = True if not args.listservers and args.server == '' and not args.store: - print_debug("Getting ready. Use parameter -h or --help to see available features.\n") + print_debug('Getting ready. Use parameter -h or --help to see available features.\n') else: - print_debug("Getting ready\n") + print_debug('Getting ready\n') try: TeSpeed( args.listservers and 'list-servers' or args.server, @@ -619,7 +618,7 @@ def main(args): chunksize=args.chunksize ) except (KeyboardInterrupt, SystemExit): - print_debug("\nTesting stopped.\n") + print_debug('\nTesting stopped.\n') #raise if __name__ == '__main__': From 97bc78926eef232e62ddd14e80d87245c91524f5 Mon Sep 17 00:00:00 2001 From: David Fischer Date: Fri, 23 May 2014 23:31:18 +0200 Subject: [PATCH 06/24] Use store_true to avoid trick with True & False --- tespeed.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/tespeed.py b/tespeed.py index ac1cc0c..8b540c9 100755 --- a/tespeed.py +++ b/tespeed.py @@ -609,14 +609,8 @@ def main(args): else: print_debug('Getting ready\n') try: - TeSpeed( - args.listservers and 'list-servers' or args.server, - args.listservers, args.servercount, - args.store and True or False, - args.suppress and True or False, - args.unit and True or False, - chunksize=args.chunksize - ) + TeSpeed(args.listservers and 'list-servers' or args.server, args.listservers, args.servercount, args.store, + args.suppress, args.unit, chunksize=args.chunksize) except (KeyboardInterrupt, SystemExit): print_debug('\nTesting stopped.\n') #raise @@ -626,9 +620,9 @@ def main(args): parser.add_argument('server', nargs='?', type=str, default='', help='Use the specified server for testing (skip checking for location and closest server).') parser.add_argument('-ls', '--list-servers', dest='listservers', nargs='?', default=0, const=10, help='List the servers sorted by distance, nearest first. Optionally specify number of servers to show.') - parser.add_argument('-w', '--csv', dest='store', action='store_const', const=True, help='Print CSV formated output to STDOUT.') - parser.add_argument('-s', '--suppress', dest='suppress', action='store_const', const=True, help='Suppress debugging (STDERR) output.') - parser.add_argument('-mib', '--mebibit', dest='unit', action='store_const', const=True, help='Show results in mebibits.') + parser.add_argument('-w', '--csv', dest='store', action='store_true', help='Print CSV formated output to STDOUT.') + parser.add_argument('-s', '--suppress', dest='suppress', action='store_true', help='Suppress debugging (STDERR) output.') + parser.add_argument('-mib', '--mebibit', dest='unit', action='store_true', help='Show results in mebibits.') parser.add_argument('-n', '--server-count', dest='servercount', nargs='?', default=1, const=1, help='Specify how many different servers should be used in paralel. (Default: 1) (Increase it for >100Mbit testing.)') parser.add_argument('-p', '--proxy', dest='use_proxy', type=int, nargs='?', const=4, help='Specify 4 or 5 to use SOCKS4 or SOCKS5 proxy.') From 51078f2655a080877955d2baa5bae9275eb060c5 Mon Sep 17 00:00:00 2001 From: David Fischer Date: Fri, 23 May 2014 23:43:28 +0200 Subject: [PATCH 07/24] Revert "Replace submodule by requirements file" This reverts commit dac510f5f4fbe4e57097ea647a4f6a89d31f2b04. --- .gitignore | 4 +++- .gitmodules | 3 +++ SocksiPy | 1 + requirements.txt | 1 - 4 files changed, 7 insertions(+), 2 deletions(-) create mode 100644 .gitmodules create mode 160000 SocksiPy delete mode 100644 requirements.txt diff --git a/.gitignore b/.gitignore index 8a2b429..2bc1b44 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ -src/ +old/ +*~ +~ *.pyc diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..091804b --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "SocksiPy"] + path = SocksiPy + url = git://github.com/Janhouse/SocksiPy.git diff --git a/SocksiPy b/SocksiPy new file mode 160000 index 0000000..842d496 --- /dev/null +++ b/SocksiPy @@ -0,0 +1 @@ +Subproject commit 842d4962cbce16ce4b232d1b7402d0375f9a0c1b diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index 3f105e7..0000000 --- a/requirements.txt +++ /dev/null @@ -1 +0,0 @@ --e git://github.com/Janhouse/SocksiPy@842d4962cbce16ce4b232d1b7402d0375f9a0c1b#egg=SocksiPy From 103eb49615459718ad2647b5284e1fc480e7b4b4 Mon Sep 17 00:00:00 2001 From: David Fischer Date: Fri, 23 May 2014 23:49:49 +0200 Subject: [PATCH 08/24] Fix renaming of methods --- tespeed.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tespeed.py b/tespeed.py index 8b540c9..bbf7da2 100755 --- a/tespeed.py +++ b/tespeed.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- - #!/usr/bin/env python2 -# + # Copyright: # 2012-2013 Janis Jansons (janis.jansons@janhouse.lv) # 2014 David Fischer (david.fischer.ch@gmail.com) @@ -217,7 +216,7 @@ def test_latency(self, servers): def test_single_latency(self, dest_addr): # Checking latency for single server # Does that by loading latency.txt (empty page) - request = self.GetRequest(dest_addr) + request = self.get_request(dest_addr) averagetime = 0 total = 0 @@ -370,7 +369,7 @@ def load_servers(self): # Load server list print_debug('Loading server list...\n') uri = 'http://speedtest.net/speedtest-servers.php?x=' + str(time.time()) - request = self.GetRequest(uri) + request = self.get_request(uri) response = urllib2.urlopen(request) # Load etree from XML data From dd37bd3cb12b6c973fd09a62101fea38be81f0c3 Mon Sep 17 00:00:00 2001 From: David Fischer Date: Sat, 24 May 2014 00:12:34 +0200 Subject: [PATCH 09/24] Split source + other things --- tespeed.py | 158 +++++++++++++++-------------------------------------- utils.py | 77 ++++++++++++++++++++++++++ 2 files changed, 121 insertions(+), 114 deletions(-) create mode 100644 utils.py diff --git a/tespeed.py b/tespeed.py index bbf7da2..6289d25 100755 --- a/tespeed.py +++ b/tespeed.py @@ -1,66 +1,18 @@ -# -*- coding: utf-8 -*- #!/usr/bin/env python2 +# -*- coding: utf-8 -*- # Copyright: # 2012-2013 Janis Jansons (janis.jansons@janhouse.lv) # 2014 David Fischer (david.fischer.ch@gmail.com) -import argparse, socket -from SocksiPy import socks - - -# Magic! -def getaddrinfo(*args): - return [(socket.AF_INET, socket.SOCK_STREAM, 6, '', (args[0], args[1]))] -socket.getaddrinfo = getaddrinfo +from __future__ import absolute_import, division, print_function, unicode_literals -import gzip, urllib, urllib2, sys, time +import argparse, gzip, urllib, urllib2, time from lxml import etree from math import radians, cos, sin, asin, sqrt from multiprocessing import Process, Pipe, Manager -from StringIO import StringIO - - -# Using StringIO with callback to measure upload progress -class CallbackStringIO(StringIO): - - def __init__(self, num, th, d, buf=''): - # Force self.buf to be a string or unicode - if not isinstance(buf, basestring): - buf = str(buf) - self.buf = buf - self.len = len(buf) - self.buflist = [] - self.pos = 0 - self.closed = False - self.softspace = 0 - self.th = th - self.num = num - self.d = d - self.total = self.len*self.th - - def read(self, n=10240): - next = StringIO.read(self, n) - #if 'done' in self.d: - # return - - self.d[self.num] = self.pos - down = 0 - for i in range(self.th): - down = down + self.d.get(i, 0) - if self.num == 0: - percent = float(down) / (self.total) - percent = round(percent*100, 2) - print_debug('Uploaded %d of %d bytes (%0.2f%%) in %d threads\r' % (down, self.total, percent, self.th)) - - #if down >= self.total: - # print_debug('\n') - # self.d['done']=1 - return next - - def __len__(self): - return self.len +from utils import CallbackStringIO, StringIO, print_debug, print_result, set_proxy, socks class TeSpeed(object): @@ -99,7 +51,7 @@ def __init__(self, server='', numTop=0, servercount=3, store=False, suppress=Fal self.store = store self.suppress = suppress if store: - print_debug('Printing CSV formated results to STDOUT.\n') + print_debug(args, 'Printing CSV formated results to STDOUT.\n') self.numTop = int(numTop) #~ self.downList=['350x350', '500x500', '750x750', '1000x1000', #~ '1500x1500', '2000x2000', '2000x2000', '2500x2500', '3000x3000', @@ -154,7 +106,7 @@ def distance(self, one, two): # haversine formula dlon = lon2 - lon1 dlat = lat2 - lat1 - a = sin(dlat/2)**2 + cos(lat1) * cos(lat2) * sin(dlon/2)**2 + a = sin(dlat / 2) ** 2 + cos(lat1) * cos(lat2) * sin(dlon / 2) ** 2 c = 2 * asin(sqrt(a)) km = 6367 * c return km @@ -182,14 +134,14 @@ def closest(self, center, points, num=5): def test_latency(self, servers): # Finding servers with lowest latency - print_debug('Testing latency...\n') + print_debug(args, 'Testing latency...\n') po = [] for server in servers: now = self.test_single_latency(server['url'] + 'latency.txt?x=' + str(time.time()))*1000 now = now / 2 # Evil hack or just pure stupidity? Nobody knows... if now == -1 or now == 0: continue - print_debug('%0.0f ms latency for %s (%s, %s, %s) [%0.2f km]\n' % + print_debug(args, '%0.0f ms latency for %s (%s, %s, %s) [%0.2f km]\n' % (now, server['url'], server['sponsor'], server['name'], server['country'], server['distance'])) server['latency'] = now @@ -259,17 +211,17 @@ def chunk_report(self, bytes_so_far, chunk_size, total_size, num, th, d, w): if num == 0 or down >= total_size * th: - percent = float(down) / (total_size * th) + percent = down / (total_size * th) percent = round(percent * 100, 2) - print_debug('Downloaded %d of %d bytes (%0.2f%%) in %d threads\r' % + print_debug(args, 'Downloaded %d of %d bytes (%0.2f%%) in %d threads\r' % (down, total_size*th, percent, th)) #if down >= total_size*th: - # print_debug('\n') + # print_debug(args, '\n') def chunk_read(self, response, num, th, d, w=0, chunk_size=False, report_hook=None): - #print_debug('Thread num %d %d %d starting to report\n' % (th, num, d)) + #print_debug(args, 'Thread num %d %d %d starting to report\n' % (th, num, d)) if not chunk_size: chunk_size = self.chunksize @@ -285,7 +237,7 @@ def chunk_read(self, response, num, th, d, w=0, chunk_size=False, report_hook=No while 1: chunk = 0 if start == 0: - #print_debug('Started receiving data\n') + #print_debug(args, 'Started receiving data\n') chunk = response.read(1) start = time.time() @@ -312,7 +264,7 @@ def async_get(self, conn, uri, num, th, d): response = urllib2.urlopen(request, timeout=30) size, start, end = self.chunk_read(response, num, th, d, report_hook=self.chunk_report) #except urllib2.URLError, e: - # print_debug('Failed downloading.\n') + # print_debug(args, 'Failed downloading.\n') except: print_debug(' \r') print_debug('Failed downloading.\n') @@ -335,10 +287,10 @@ def async_post(self, conn, uri, num, th, d): response = urllib2.urlopen(request, timeout=30) size, start, end = self.chunk_read(response, num, th, d, 1, report_hook=self.chunk_report) #except urllib2.URLError: - # print_debug('Failed uploading.\n') + # print_debug(args, 'Failed uploading.\n') except: - print_debug(' \r') - print_debug('Failed uploading.\n') + print_debug(args, ' \r') + print_debug(args, 'Failed uploading.\n') conn.send([0, 0, False]) conn.close() return @@ -348,7 +300,7 @@ def async_post(self, conn, uri, num, th, d): def load_config(self): # Load the configuration file - print_debug('Loading speedtest configuration...\n') + print_debug(args, 'Loading speedtest configuration...\n') uri = 'http://speedtest.net/speedtest-config.php?x=' + str(time.time()) request = self.get_request(uri) response = urllib2.urlopen(request) @@ -361,13 +313,13 @@ def load_config(self): lat = float(config.find('client').attrib['lat']) lon = float(config.find('client').attrib['lon']) - print_debug('IP: %s; Lat: %f; Lon: %f; ISP: %s\n' % (ip, lat, lon, isp)) + print_debug(args, 'IP: %s; Lat: %f; Lon: %f; ISP: %s\n' % (ip, lat, lon, isp)) return {'ip': ip, 'lat': lat, 'lon': lon, 'isp': isp} def load_servers(self): # Load server list - print_debug('Loading server list...\n') + print_debug(args, 'Loading server list...\n') uri = 'http://speedtest.net/speedtest-servers.php?x=' + str(time.time()) request = self.get_request(uri) response = urllib2.urlopen(request) @@ -398,7 +350,7 @@ def decompress_response(sefl, response): return gzipper.read() def find_best_server(self): - print_debug('Looking for closest and best server...\n') + print_debug(args, 'Looking for closest and best server...\n') best = self.test_latency(self.closest([self.config['lat'], self.config['lon']], self.server_list, self.bestServers)) for server in best: @@ -413,10 +365,8 @@ def async_request(self, url, num, upload=0): #print full_url connection = {} connection['parent'], connection['child'] = Pipe() - if upload == 1: - connection['connection'] = Process(target=self.async_post, args=(connection['child'], full_url, i, num, d)) - else: - connection['connection'] = Process(target=self.async_get, args=(connection['child'], full_url, i, num, d)) + connection['connection'] = Process(target=self.async_post if upload == 1 else self.async_get, + args=(connection['child'], full_url, i, num, d)) connection['connection'].start() connections.append(connection) @@ -426,7 +376,7 @@ def async_request(self, url, num, upload=0): end = time.time() - print_debug(' \r') + print_debug(args, ' \r') sizes = 0 #tspeed=0 @@ -459,7 +409,7 @@ def test_upload(self): data = '' for i in range(0, len(self.upSizes)): if len(data) == 0 or self.upSizes[i] != self.upSizes[i-1]: - #print_debug('Generating new string to upload. Length: %d\n' % (self.upSizes[i])) + #print_debug(args, 'Generating new string to upload. Length: %d\n' % (self.upSizes[i])) data = ''.join('1' for x in xrange(self.upSizes[i])) self.postData = urllib.urlencode({'upload6': data}) @@ -487,8 +437,8 @@ def test_upload(self): size = self.speed_conversion(sizes) speed = size / took - print_debug('Upload size: %0.2f MiB; Uploaded in %0.2f s\n' % (size, took)) - print_debug('\033[92mUpload speed: %0.2f %s/s\033[0m\n' % (speed, self.units)) + print_debug(args, 'Upload size: %0.2f MiB; Uploaded in %0.2f s\n' % (size, took)) + print_debug(args, '\033[92mUpload speed: %0.2f %s/s\033[0m\n' % (speed, self.units)) if self.up_speed < speed: self.up_speed = speed @@ -496,20 +446,16 @@ def test_upload(self): if took > 5: break - #print_debug('Upload size: %0.2f MiB; Uploaded in %0.2f s\n' % (self.speed_conversion(sizes), took)) - #print_debug('Upload speed: %0.2f MiB/s\n' % (self.speed_conversion(sizes)/took)) + #print_debug(args, 'Upload size: %0.2f MiB; Uploaded in %0.2f s\n' % (self.speed_conversion(sizes), took)) + #print_debug(args, 'Upload speed: %0.2f MiB/s\n' % (self.speed_conversion(sizes)/took)) def speed_conversion(self, data): - if self.unit == 1: - result = (float(data) / 1024 / 1024) - else: - result = (float(data) / 1024 / 1024) * 1.048576 * 8 - return result + return data / 1024 ** 2 * (1 if self.unit == 1 else 1.048576 * 8) def test_download(self): # Testing download speed sizes, took = [0, 0] - for i in range(0, len(self.downList)): + for i in xrange(0, len(self.downList)): url = 'random' + self.downList[i] + '.jpg?x=' + str(time.time()) + '&y=3' if i < 2: @@ -536,8 +482,8 @@ def test_download(self): size = self.speed_conversion(sizes) speed = size / took - print_debug('Download size: %0.2f MiB; Downloaded in %0.2f s\n' % (size, took)) - print_debug('\033[91mDownload speed: %0.2f %s/s\033[0m\n' % (speed, self.units)) + print_debug(args, 'Download size: %0.2f MiB; Downloaded in %0.2f s\n' % (size, took)) + print_debug(args, '\033[91mDownload speed: %0.2f %s/s\033[0m\n' % (speed, self.units)) if self.down_speed < speed: self.down_speed = speed @@ -545,8 +491,8 @@ def test_download(self): if took > 5: break - #print_debug('Download size: %0.2f MiB; Downloaded in %0.2f s\n' % (self.speed_conversion(sizes), took)) - #print_debug('Download speed: %0.2f %s/s\n' % (self.speed_conversion(sizes)/took, self.units)) + #print_debug(args, 'Download size: %0.2f MiB; Downloaded in %0.2f s\n' % (self.speed_conversion(sizes), took)) + #print_debug(args, 'Download speed: %0.2f %s/s\n' % (self.speed_conversion(sizes)/took, self.units)) def test_speed(self): @@ -564,32 +510,16 @@ def test_speed(self): self.test_download() self.test_upload() - print_result('%0.2f,%0.2f,"%s","%s"\n' % (self.down_speed, self.up_speed, self.units, self.servers)) + print_result(args, '%0.2f,%0.2f,"%s","%s"\n' % (self.down_speed, self.up_speed, self.units, self.servers)) def list_servers(self, num=0): - allSorted = self.closest([self.config['lat'], self.config['lon']], self.server_list, num) - - for i in range(0, len(allSorted)): - print_result('%s. %s (%s, %s, %s) [%0.2f km]\n' % - (i + 1, allSorted[i]['url'], allSorted[i]['sponsor'], allSorted[i]['name'], - allSorted[i]['country'], allSorted[i]['distance'])) - - -def print_debug(string): - if not args.suppress: - sys.stderr.write(string.encode('utf8')) - - -def print_result(string): - if args.store: - sys.stdout.write(string.encode('utf8')) - + all_sorted = self.closest([self.config['lat'], self.config['lon']], self.server_list, num) -# Thanks to Ryan Sears for http://bit.ly/17HhSli -def set_proxy(typ=socks.PROXY_TYPE_SOCKS4, host='127.0.0.1', port=9050): - socks.setdefaultproxy(typ, host, port) - socket.socket = socks.socksocket + for i in xrange(0, len(all_sorted)): + print_result(args, '%s. %s (%s, %s, %s) [%0.2f km]\n' % + (i + 1, all_sorted[i]['url'], all_sorted[i]['sponsor'], all_sorted[i]['name'], + all_sorted[i]['country'], all_sorted[i]['distance'])) def main(args): @@ -604,14 +534,14 @@ def main(args): args.store = True if not args.listservers and args.server == '' and not args.store: - print_debug('Getting ready. Use parameter -h or --help to see available features.\n') + print_debug(args, 'Getting ready. Use parameter -h or --help to see available features.\n') else: - print_debug('Getting ready\n') + print_debug(args, 'Getting ready\n') try: TeSpeed(args.listservers and 'list-servers' or args.server, args.listservers, args.servercount, args.store, args.suppress, args.unit, chunksize=args.chunksize) except (KeyboardInterrupt, SystemExit): - print_debug('\nTesting stopped.\n') + print_debug(args, '\nTesting stopped.\n') #raise if __name__ == '__main__': diff --git a/utils.py b/utils.py new file mode 100644 index 0000000..4731e9d --- /dev/null +++ b/utils.py @@ -0,0 +1,77 @@ +# -*- coding: utf-8 -*- + +# Copyright: +# 2012-2013 Janis Jansons (janis.jansons@janhouse.lv) +# 2014 David Fischer (david.fischer.ch@gmail.com) + +from __future__ import absolute_import, division, print_function, unicode_literals + +import socket, sys +from SocksiPy import socks +from StringIO import StringIO + +__all__ = ('CallbackStringIO', 'StringIO', 'print_debug', 'print_result', 'set_proxy', 'socks') + + +# Magic! +def getaddrinfo(*args): + return [(socket.AF_INET, socket.SOCK_STREAM, 6, '', (args[0], args[1]))] +socket.getaddrinfo = getaddrinfo + + +# Using StringIO with callback to measure upload progress +class CallbackStringIO(StringIO): + + def __init__(self, num, th, d, buf=''): + # Force self.buf to be a string or unicode + if not isinstance(buf, basestring): + buf = str(buf) + self.buf = buf + self.len = len(buf) + self.buflist = [] + self.pos = 0 + self.closed = False + self.softspace = 0 + self.th = th + self.num = num + self.d = d + self.total = self.len*self.th + + def read(self, n=10240): + next_chunk = StringIO.read(self, n) + #if 'done' in self.d: + # return + + self.d[self.num] = self.pos + down = 0 + for i in xrange(self.th): + down = down + self.d.get(i, 0) + if self.num == 0: + percent = down / self.total + percent = round(percent * 100, 2) + print_debug(None, 'Uploaded %d of %d bytes (%0.2f%%) in %d threads\r' % (down, self.total, percent, self.th)) + + #if down >= self.total: + # print_debug('\n') + # self.d['done']=1 + + return next_chunk + + def __len__(self): + return self.len + + +def print_debug(args, string): + if not args or not args.suppress: + sys.stderr.write(string.encode('utf8')) + + +def print_result(args, string): + if args.store: + sys.stdout.write(string.encode('utf8')) + + +# Thanks to Ryan Sears for http://bit.ly/17HhSli +def set_proxy(typ=socks.PROXY_TYPE_SOCKS4, host='127.0.0.1', port=9050): + socks.setdefaultproxy(typ, host, port) + socket.socket = socks.socksocket From 09030934e911dab5f06532e3e0a5d2aeda1919d0 Mon Sep 17 00:00:00 2001 From: David Fischer Date: Sat, 24 May 2014 00:42:36 +0200 Subject: [PATCH 10/24] Add a logging class (not yet using stdlib.logging because of \n) --- tespeed.py | 92 +++++++++++++++++++++++++++--------------------------- utils.py | 26 +++++++++------ 2 files changed, 63 insertions(+), 55 deletions(-) diff --git a/tespeed.py b/tespeed.py index 6289d25..6a3404f 100755 --- a/tespeed.py +++ b/tespeed.py @@ -12,12 +12,12 @@ from math import radians, cos, sin, asin, sqrt from multiprocessing import Process, Pipe, Manager -from utils import CallbackStringIO, StringIO, print_debug, print_result, set_proxy, socks +from utils import CallbackStringIO, StringIO, Log, set_proxy, socks class TeSpeed(object): - def __init__(self, server='', numTop=0, servercount=3, store=False, suppress=False, unit=False, chunksize=10240): + def __init__(self, server='', numTop=0, servercount=3, unit=False, chunksize=10240, log=None): self.headers = { 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', @@ -43,15 +43,15 @@ def __init__(self, server='', numTop=0, servercount=3, store=False, suppress=Fal self.unit = 0 self.chunksize = chunksize + self.log = log if unit: self.units = 'MiB' self.unit = 1 - self.store = store - self.suppress = suppress - if store: - print_debug(args, 'Printing CSV formated results to STDOUT.\n') + if log.store: + log.debug('Printing CSV formated results to STDOUT.\n') + self.numTop = int(numTop) #~ self.downList=['350x350', '500x500', '750x750', '1000x1000', #~ '1500x1500', '2000x2000', '2000x2000', '2500x2500', '3000x3000', @@ -134,15 +134,16 @@ def closest(self, center, points, num=5): def test_latency(self, servers): # Finding servers with lowest latency - print_debug(args, 'Testing latency...\n') + self.log.debug('Testing latency...\n') po = [] for server in servers: now = self.test_single_latency(server['url'] + 'latency.txt?x=' + str(time.time()))*1000 now = now / 2 # Evil hack or just pure stupidity? Nobody knows... if now == -1 or now == 0: continue - print_debug(args, '%0.0f ms latency for %s (%s, %s, %s) [%0.2f km]\n' % - (now, server['url'], server['sponsor'], server['name'], server['country'], server['distance'])) + self.log.debug('%0.0f ms latency for %s (%s, %s, %s) [%0.2f km]\n' % + (now, server['url'], server['sponsor'], server['name'], server['country'], + server['distance'])) server['latency'] = now @@ -214,14 +215,13 @@ def chunk_report(self, bytes_so_far, chunk_size, total_size, num, th, d, w): percent = down / (total_size * th) percent = round(percent * 100, 2) - print_debug(args, 'Downloaded %d of %d bytes (%0.2f%%) in %d threads\r' % - (down, total_size*th, percent, th)) + self.log.debug('Downloaded %d of %d bytes (%0.2f%%) in %d threads\r' % (down, total_size*th, percent, th)) #if down >= total_size*th: - # print_debug(args, '\n') + # self.log.debug('\n') def chunk_read(self, response, num, th, d, w=0, chunk_size=False, report_hook=None): - #print_debug(args, 'Thread num %d %d %d starting to report\n' % (th, num, d)) + # self.log.debug('Thread num %d %d %d starting to report\n' % (th, num, d)) if not chunk_size: chunk_size = self.chunksize @@ -237,7 +237,7 @@ def chunk_read(self, response, num, th, d, w=0, chunk_size=False, report_hook=No while 1: chunk = 0 if start == 0: - #print_debug(args, 'Started receiving data\n') + # self.log.debug('Started receiving data\n') chunk = response.read(1) start = time.time() @@ -264,10 +264,10 @@ def async_get(self, conn, uri, num, th, d): response = urllib2.urlopen(request, timeout=30) size, start, end = self.chunk_read(response, num, th, d, report_hook=self.chunk_report) #except urllib2.URLError, e: - # print_debug(args, 'Failed downloading.\n') + # self.log.debug('Failed downloading.\n') except: - print_debug(' \r') - print_debug('Failed downloading.\n') + self.log.debug(' \r') + self.log.debug('Failed downloading.\n') conn.send([0, 0, False]) conn.close() return @@ -277,7 +277,7 @@ def async_get(self, conn, uri, num, th, d): def async_post(self, conn, uri, num, th, d): postlen = len(self.postData) - stream = CallbackStringIO(num, th, d, self.postData) + stream = CallbackStringIO(num, th, d, self.postData, log=self.log) request = self.post_request(uri, stream) start = 0 @@ -287,10 +287,10 @@ def async_post(self, conn, uri, num, th, d): response = urllib2.urlopen(request, timeout=30) size, start, end = self.chunk_read(response, num, th, d, 1, report_hook=self.chunk_report) #except urllib2.URLError: - # print_debug(args, 'Failed uploading.\n') + # self.log.debug('Failed uploading.\n') except: - print_debug(args, ' \r') - print_debug(args, 'Failed uploading.\n') + self.log.debug(' \r') + self.log.debug('Failed uploading.\n') conn.send([0, 0, False]) conn.close() return @@ -300,7 +300,7 @@ def async_post(self, conn, uri, num, th, d): def load_config(self): # Load the configuration file - print_debug(args, 'Loading speedtest configuration...\n') + self.log.debug('Loading speedtest configuration...\n') uri = 'http://speedtest.net/speedtest-config.php?x=' + str(time.time()) request = self.get_request(uri) response = urllib2.urlopen(request) @@ -313,13 +313,13 @@ def load_config(self): lat = float(config.find('client').attrib['lat']) lon = float(config.find('client').attrib['lon']) - print_debug(args, 'IP: %s; Lat: %f; Lon: %f; ISP: %s\n' % (ip, lat, lon, isp)) + self.log.debug('IP: %s; Lat: %f; Lon: %f; ISP: %s\n' % (ip, lat, lon, isp)) return {'ip': ip, 'lat': lat, 'lon': lon, 'isp': isp} def load_servers(self): # Load server list - print_debug(args, 'Loading server list...\n') + self.log.debug('Loading server list...\n') uri = 'http://speedtest.net/speedtest-servers.php?x=' + str(time.time()) request = self.get_request(uri) response = urllib2.urlopen(request) @@ -350,7 +350,7 @@ def decompress_response(sefl, response): return gzipper.read() def find_best_server(self): - print_debug(args, 'Looking for closest and best server...\n') + self.log.debug('Looking for closest and best server...\n') best = self.test_latency(self.closest([self.config['lat'], self.config['lon']], self.server_list, self.bestServers)) for server in best: @@ -376,7 +376,7 @@ def async_request(self, url, num, upload=0): end = time.time() - print_debug(args, ' \r') + self.log.debug(' \r') sizes = 0 #tspeed=0 @@ -409,7 +409,7 @@ def test_upload(self): data = '' for i in range(0, len(self.upSizes)): if len(data) == 0 or self.upSizes[i] != self.upSizes[i-1]: - #print_debug(args, 'Generating new string to upload. Length: %d\n' % (self.upSizes[i])) + #self.log.debug('Generating new string to upload. Length: %d\n' % (self.upSizes[i])) data = ''.join('1' for x in xrange(self.upSizes[i])) self.postData = urllib.urlencode({'upload6': data}) @@ -437,8 +437,8 @@ def test_upload(self): size = self.speed_conversion(sizes) speed = size / took - print_debug(args, 'Upload size: %0.2f MiB; Uploaded in %0.2f s\n' % (size, took)) - print_debug(args, '\033[92mUpload speed: %0.2f %s/s\033[0m\n' % (speed, self.units)) + self.log.debug('Upload size: %0.2f MiB; Uploaded in %0.2f s\n' % (size, took)) + self.log.debug('\033[92mUpload speed: %0.2f %s/s\033[0m\n' % (speed, self.units)) if self.up_speed < speed: self.up_speed = speed @@ -446,8 +446,8 @@ def test_upload(self): if took > 5: break - #print_debug(args, 'Upload size: %0.2f MiB; Uploaded in %0.2f s\n' % (self.speed_conversion(sizes), took)) - #print_debug(args, 'Upload speed: %0.2f MiB/s\n' % (self.speed_conversion(sizes)/took)) + #self.log.debug('Upload size: %0.2f MiB; Uploaded in %0.2f s\n' % (self.speed_conversion(sizes), took)) + #self.log.debug('Upload speed: %0.2f MiB/s\n' % (self.speed_conversion(sizes)/took)) def speed_conversion(self, data): return data / 1024 ** 2 * (1 if self.unit == 1 else 1.048576 * 8) @@ -482,8 +482,8 @@ def test_download(self): size = self.speed_conversion(sizes) speed = size / took - print_debug(args, 'Download size: %0.2f MiB; Downloaded in %0.2f s\n' % (size, took)) - print_debug(args, '\033[91mDownload speed: %0.2f %s/s\033[0m\n' % (speed, self.units)) + self.log.debug('Download size: %0.2f MiB; Downloaded in %0.2f s\n' % (size, took)) + self.log.debug('\033[91mDownload speed: %0.2f %s/s\033[0m\n' % (speed, self.units)) if self.down_speed < speed: self.down_speed = speed @@ -491,8 +491,8 @@ def test_download(self): if took > 5: break - #print_debug(args, 'Download size: %0.2f MiB; Downloaded in %0.2f s\n' % (self.speed_conversion(sizes), took)) - #print_debug(args, 'Download speed: %0.2f %s/s\n' % (self.speed_conversion(sizes)/took, self.units)) + #self.log.debug('Download size: %0.2f MiB; Downloaded in %0.2f s\n' % (self.speed_conversion(sizes), took)) + #self.log.debug('Download speed: %0.2f %s/s\n' % (self.speed_conversion(sizes)/took, self.units)) def test_speed(self): @@ -510,16 +510,16 @@ def test_speed(self): self.test_download() self.test_upload() - print_result(args, '%0.2f,%0.2f,"%s","%s"\n' % (self.down_speed, self.up_speed, self.units, self.servers)) + self.log.result('%0.2f,%0.2f,"%s","%s"\n' % (self.down_speed, self.up_speed, self.units, self.servers)) def list_servers(self, num=0): all_sorted = self.closest([self.config['lat'], self.config['lon']], self.server_list, num) for i in xrange(0, len(all_sorted)): - print_result(args, '%s. %s (%s, %s, %s) [%0.2f km]\n' % - (i + 1, all_sorted[i]['url'], all_sorted[i]['sponsor'], all_sorted[i]['name'], - all_sorted[i]['country'], all_sorted[i]['distance'])) + self.log.result('%s. %s (%s, %s, %s) [%0.2f km]\n' % + (i + 1, all_sorted[i]['url'], all_sorted[i]['sponsor'], all_sorted[i]['name'], + all_sorted[i]['country'], all_sorted[i]['distance'])) def main(args): @@ -533,15 +533,16 @@ def main(args): if args.listservers: args.store = True + log = Log(suppress=args.suppress, store=args.store) if not args.listservers and args.server == '' and not args.store: - print_debug(args, 'Getting ready. Use parameter -h or --help to see available features.\n') + log.debug('Getting ready. Use parameter -h or --help to see available features.\n') else: - print_debug(args, 'Getting ready\n') + log.debug('Getting ready\n') try: - TeSpeed(args.listservers and 'list-servers' or args.server, args.listservers, args.servercount, args.store, - args.suppress, args.unit, chunksize=args.chunksize) + TeSpeed(args.listservers and 'list-servers' or args.server, args.listservers, args.servercount, args.unit, + chunksize=args.chunksize, log=log) except (KeyboardInterrupt, SystemExit): - print_debug(args, '\nTesting stopped.\n') + log.debug('\nTesting stopped.\n') #raise if __name__ == '__main__': @@ -562,5 +563,4 @@ def main(args): #parser.add_argument('-i', '--interface', dest='interface', nargs='?', help='If specified, measures speed from data for the whole network interface.') - args = parser.parse_args() - main(args) + main(parser.parse_args()) diff --git a/utils.py b/utils.py index 4731e9d..827df8e 100644 --- a/utils.py +++ b/utils.py @@ -22,7 +22,7 @@ def getaddrinfo(*args): # Using StringIO with callback to measure upload progress class CallbackStringIO(StringIO): - def __init__(self, num, th, d, buf=''): + def __init__(self, num, th, d, buf='', log=None): # Force self.buf to be a string or unicode if not isinstance(buf, basestring): buf = str(buf) @@ -36,6 +36,7 @@ def __init__(self, num, th, d, buf=''): self.num = num self.d = d self.total = self.len*self.th + self.log = log def read(self, n=10240): next_chunk = StringIO.read(self, n) @@ -49,10 +50,12 @@ def read(self, n=10240): if self.num == 0: percent = down / self.total percent = round(percent * 100, 2) - print_debug(None, 'Uploaded %d of %d bytes (%0.2f%%) in %d threads\r' % (down, self.total, percent, self.th)) + if self.log: + self.log.debug('Uploaded %d of %d bytes (%0.2f%%) in %d threads\r' % + (down, self.total, percent, self.th)) #if down >= self.total: - # print_debug('\n') + # if self.log: self.log.debug('\n') # self.d['done']=1 return next_chunk @@ -61,14 +64,19 @@ def __len__(self): return self.len -def print_debug(args, string): - if not args or not args.suppress: - sys.stderr.write(string.encode('utf8')) +class Log(object): + def __init__(self, suppress=False, store=False): + self.suppress = suppress + self.store = store -def print_result(args, string): - if args.store: - sys.stdout.write(string.encode('utf8')) + def debug(self, string): + if not self.suppress: + sys.stderr.write(string.encode('utf8')) + + def result(self, string): + if self.store: + sys.stdout.write(string.encode('utf8')) # Thanks to Ryan Sears for http://bit.ly/17HhSli From b66c97513ee46f9017c4e457c185bbaf780608e8 Mon Sep 17 00:00:00 2001 From: David Fischer Date: Sat, 24 May 2014 01:01:05 +0200 Subject: [PATCH 11/24] Cleanup usage of "constants" --- tespeed.py | 79 ++++++++++++++++++++++++------------------------------ 1 file changed, 35 insertions(+), 44 deletions(-) diff --git a/tespeed.py b/tespeed.py index 6a3404f..e421d19 100755 --- a/tespeed.py +++ b/tespeed.py @@ -17,6 +17,36 @@ class TeSpeed(object): + DOWNLOAD_LIST = [ + '350x350', '350x350', '500x500', '500x500', '750x750', '750x750', '1000x1000', '1500x1500', '2000x2000', + '2500x2500', + + '3000x3000', '3500x3500', '4000x4000', '1000x1000', '1000x1000', '1000x1000', '1000x1000', '1000x1000', + '1000x1000', '1000x1000', '1000x1000', '1000x1000', '1000x1000', '1000x1000', '1000x1000', '1000x1000', + '1000x1000', '1000x1000', '1000x1000', '1000x1000', + + '2000x2000', '2000x2000', '2000x2000', '2000x2000', '2000x2000', '2000x2000', '2000x2000', '2000x2000', + '2000x2000', '2000x2000', '2000x2000', '2000x2000', '2000x2000', '2000x2000', '2000x2000', '2000x2000', + '2000x2000', '2000x2000', '2000x2000', '2000x2000', + + '4000x4000', '4000x4000', '4000x4000', '4000x4000', '4000x4000' + ] + + UPLOAD_SIZES = [ + 1024*256, 1024*256, 1024*512, 1024*512, 1024*1024, 1024*1024, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*512, + 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, + + 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, + 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, + + 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, + 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, + + 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, + 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, + 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2 + ] + def __init__(self, server='', numTop=0, servercount=3, unit=False, chunksize=10240, log=None): self.headers = { @@ -53,45 +83,6 @@ def __init__(self, server='', numTop=0, servercount=3, unit=False, chunksize=102 log.debug('Printing CSV formated results to STDOUT.\n') self.numTop = int(numTop) - #~ self.downList=['350x350', '500x500', '750x750', '1000x1000', - #~ '1500x1500', '2000x2000', '2000x2000', '2500x2500', '3000x3000', - #~ '3500x3500', '4000x4000', '4000x4000', '4000x4000', '4000x4000'] - #~ self.upSizes=[1024*256, 1024*256, 1024*512, 1024*512, - #~ 1024*1024, 1024*1024, 1024*1024*2, 1024*1024*2, - #~ 1024*1024*2, 1024*1024*2] - - self.downList = [ -'350x350', '350x350', '500x500', '500x500', '750x750', '750x750', '1000x1000', '1500x1500', '2000x2000', '2500x2500', - -'3000x3000','3500x3500','4000x4000','1000x1000','1000x1000','1000x1000','1000x1000','1000x1000','1000x1000','1000x1000', -'1000x1000','1000x1000','1000x1000','1000x1000','1000x1000','1000x1000','1000x1000','1000x1000','1000x1000','1000x1000', - -'2000x2000','2000x2000','2000x2000','2000x2000','2000x2000','2000x2000','2000x2000','2000x2000','2000x2000','2000x2000', -'2000x2000','2000x2000','2000x2000','2000x2000','2000x2000','2000x2000','2000x2000','2000x2000','2000x2000','2000x2000', - -'4000x4000', '4000x4000', '4000x4000', '4000x4000', '4000x4000' -] - -#'350x350', '500x500', '750x750', '1000x1000', -# '1500x1500', '2000x2000', '2000x2000', '2500x2500', '3000x3000', -# '3500x3500', '4000x4000', '4000x4000', '4000x4000', '4000x4000' -#] - self.upSizes = [ - -1024*256, 1024*256, 1024*512, 1024*512, 1024*1024, 1024*1024, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*512, -1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, - -1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, -1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, - -1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, -1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, - -1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, -1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, -# 1024*1024, 1024*1024, 1024*1024*2, 1024*1024*2, -# 1024*1024*2, 1024*1024*2] -] self.postData = '' self.test_speed() @@ -407,10 +398,10 @@ def test_upload(self): sizes, took = [0, 0] data = '' - for i in range(0, len(self.upSizes)): - if len(data) == 0 or self.upSizes[i] != self.upSizes[i-1]: + for i in range(0, len(self.UPLOAD_SIZES)): + if len(data) == 0 or self.UPLOAD_SIZES[i] != self.UPLOAD_SIZES[i-1]: #self.log.debug('Generating new string to upload. Length: %d\n' % (self.upSizes[i])) - data = ''.join('1' for x in xrange(self.upSizes[i])) + data = ''.join('1' for x in xrange(self.UPLOAD_SIZES[i])) self.postData = urllib.urlencode({'upload6': data}) if i < 2: @@ -455,8 +446,8 @@ def speed_conversion(self, data): def test_download(self): # Testing download speed sizes, took = [0, 0] - for i in xrange(0, len(self.downList)): - url = 'random' + self.downList[i] + '.jpg?x=' + str(time.time()) + '&y=3' + for i in xrange(0, len(self.DOWNLOAD_LIST)): + url = 'random' + self.DOWNLOAD_LIST[i] + '.jpg?x=' + str(time.time()) + '&y=3' if i < 2: thrds = 1 From 02b75d089e9aeb03f049d4d9fa255e49c93697c4 Mon Sep 17 00:00:00 2001 From: David Fischer Date: Sat, 24 May 2014 01:02:46 +0200 Subject: [PATCH 12/24] Rename attributes & variables to follow recommendations of PEP-8 --- tespeed.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/tespeed.py b/tespeed.py index e421d19..fb51524 100755 --- a/tespeed.py +++ b/tespeed.py @@ -47,7 +47,7 @@ class TeSpeed(object): 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2 ] - def __init__(self, server='', numTop=0, servercount=3, unit=False, chunksize=10240, log=None): + def __init__(self, server='', num_top=0, servercount=3, unit=False, chunk_size=10240, log=None): self.headers = { 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', @@ -67,12 +67,12 @@ def __init__(self, server='', numTop=0, servercount=3, unit=False, chunksize=102 self.down_speed = -1 self.up_speed = -1 self.latencycount = 10 - self.bestServers = 5 + self.best_servers = 5 self.units = 'Mbit' self.unit = 0 - self.chunksize = chunksize + self.chunk_size = chunk_size self.log = log if unit: @@ -82,9 +82,9 @@ def __init__(self, server='', numTop=0, servercount=3, unit=False, chunksize=102 if log.store: log.debug('Printing CSV formated results to STDOUT.\n') - self.numTop = int(numTop) + self.num_top = int(num_top) - self.postData = '' + self.post_data = '' self.test_speed() def distance(self, one, two): @@ -162,24 +162,24 @@ def test_single_latency(self, dest_addr): # Does that by loading latency.txt (empty page) request = self.get_request(dest_addr) - averagetime = 0 + average_time = 0 total = 0 for i in range(self.latencycount): error = 0 - startTime = time.time() + start_time = time.time() try: urllib2.urlopen(request, timeout=5) except urllib2.URLError: error = 1 if error == 0: - averagetime = averagetime + (time.time() - startTime) + average_time = average_time + (time.time() - start_time) total = total + 1 if total == 0: return False - return averagetime / total + return average_time / total def get_request(self, uri): # Generates a GET request to be used with urlopen @@ -215,7 +215,7 @@ def chunk_read(self, response, num, th, d, w=0, chunk_size=False, report_hook=No # self.log.debug('Thread num %d %d %d starting to report\n' % (th, num, d)) if not chunk_size: - chunk_size = self.chunksize + chunk_size = self.chunk_size if w == 1: return [0, 0, 0] @@ -267,8 +267,8 @@ def async_get(self, conn, uri, num, th, d): conn.close() def async_post(self, conn, uri, num, th, d): - postlen = len(self.postData) - stream = CallbackStringIO(num, th, d, self.postData, log=self.log) + postlen = len(self.post_data) + stream = CallbackStringIO(num, th, d, self.post_data, log=self.log) request = self.post_request(uri, stream) start = 0 @@ -343,7 +343,7 @@ def decompress_response(sefl, response): def find_best_server(self): self.log.debug('Looking for closest and best server...\n') best = self.test_latency(self.closest([self.config['lat'], self.config['lon']], - self.server_list, self.bestServers)) + self.server_list, self.best_servers)) for server in best: self.servers.append(server['url']) @@ -490,7 +490,7 @@ def test_speed(self): if self.server == 'list-servers': self.config = self.load_config() self.server_list = self.load_servers() - self.list_servers(self.numTop) + self.list_servers(self.num_top) return if self.server == '': @@ -531,7 +531,7 @@ def main(args): log.debug('Getting ready\n') try: TeSpeed(args.listservers and 'list-servers' or args.server, args.listservers, args.servercount, args.unit, - chunksize=args.chunksize, log=log) + chunk_size=args.chunksize, log=log) except (KeyboardInterrupt, SystemExit): log.debug('\nTesting stopped.\n') #raise From 9666ac24fbcd8d40391e864c273c27c4990e86ee Mon Sep 17 00:00:00 2001 From: David Fischer Date: Sat, 24 May 2014 01:04:36 +0200 Subject: [PATCH 13/24] Fix renaming of attributes and variables --- tespeed.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tespeed.py b/tespeed.py index fb51524..0200552 100755 --- a/tespeed.py +++ b/tespeed.py @@ -400,9 +400,9 @@ def test_upload(self): data = '' for i in range(0, len(self.UPLOAD_SIZES)): if len(data) == 0 or self.UPLOAD_SIZES[i] != self.UPLOAD_SIZES[i-1]: - #self.log.debug('Generating new string to upload. Length: %d\n' % (self.upSizes[i])) + #self.log.debug('Generating new string to upload. Length: %d\n' % (self.UPLOAD_SIZES[i])) data = ''.join('1' for x in xrange(self.UPLOAD_SIZES[i])) - self.postData = urllib.urlencode({'upload6': data}) + self.post_data = urllib.urlencode({'upload6': data}) if i < 2: thrds = 1 From bd3af2428ccd1a23e80095b4d6a196059001f114 Mon Sep 17 00:00:00 2001 From: David Fischer Date: Sat, 24 May 2014 01:14:32 +0200 Subject: [PATCH 14/24] Use xrange instead of range --- tespeed.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tespeed.py b/tespeed.py index 0200552..eb22713 100755 --- a/tespeed.py +++ b/tespeed.py @@ -105,7 +105,7 @@ def distance(self, one, two): def closest(self, center, points, num=5): # Returns object that is closest to center closest = {} - for p in range(len(points)): + for p in xrange(len(points)): now = self.distance(center, [points[p]['lat'], points[p]['lon']]) points[p]['distance'] = now while True: @@ -144,7 +144,7 @@ def test_latency(self, servers): else: largest = -1 - for x in range(len(po)): + for x in xrange(len(po)): if largest < 0: if now < po[x]['latency']: largest = x @@ -164,7 +164,7 @@ def test_single_latency(self, dest_addr): average_time = 0 total = 0 - for i in range(self.latencycount): + for i in xrange(self.latencycount): error = 0 start_time = time.time() try: @@ -198,7 +198,7 @@ def chunk_report(self, bytes_so_far, chunk_size, total_size, num, th, d, w): return d[num] = bytes_so_far down = 0 - for i in range(th): + for i in xrange(th): down = down + d.get(i, 0) if num == 0 or down >= total_size * th: @@ -361,7 +361,7 @@ def async_request(self, url, num, upload=0): connection['connection'].start() connections.append(connection) - for c in range(num): + for c in xrange(num): connections[c]['size'], connections[c]['start'], connections[c]['end'] = connections[c]['parent'].recv() connections[c]['connection'].join() @@ -371,7 +371,7 @@ def async_request(self, url, num, upload=0): sizes = 0 #tspeed=0 - for c in range(num): + for c in xrange(num): if connections[c]['end'] is not False: #tspeed=tspeed+(connections[c]['size']/(connections[c]['end']-connections[c]['start'])) sizes = sizes + connections[c]['size'] @@ -398,7 +398,7 @@ def test_upload(self): sizes, took = [0, 0] data = '' - for i in range(0, len(self.UPLOAD_SIZES)): + for i in xrange(0, len(self.UPLOAD_SIZES)): if len(data) == 0 or self.UPLOAD_SIZES[i] != self.UPLOAD_SIZES[i-1]: #self.log.debug('Generating new string to upload. Length: %d\n' % (self.UPLOAD_SIZES[i])) data = ''.join('1' for x in xrange(self.UPLOAD_SIZES[i])) From 8516b761d96c0ff70521111e8ef1bc4a441695a1 Mon Sep 17 00:00:00 2001 From: David Fischer Date: Sat, 24 May 2014 01:15:35 +0200 Subject: [PATCH 15/24] Make code shorter --- tespeed.py | 33 +++++++++------------------------ 1 file changed, 9 insertions(+), 24 deletions(-) diff --git a/tespeed.py b/tespeed.py index eb22713..e62fadb 100755 --- a/tespeed.py +++ b/tespeed.py @@ -59,25 +59,20 @@ def __init__(self, server='', num_top=0, servercount=3, unit=False, chunk_size=1 } self.num_servers = servercount - self.servers = [] - if server != '': - self.servers = [server] - + self.servers = [] if server == '' else [server] self.server = server - self.down_speed = -1 - self.up_speed = -1 - self.latencycount = 10 + self.down_speed = self.up_speed = -1 + self.latency_count = 10 self.best_servers = 5 - - self.units = 'Mbit' - self.unit = 0 - self.chunk_size = chunk_size self.log = log if unit: self.units = 'MiB' self.unit = 1 + else: + self.units = 'Mbit' + self.unit = 0 if log.store: log.debug('Printing CSV formated results to STDOUT.\n') @@ -164,7 +159,7 @@ def test_single_latency(self, dest_addr): average_time = 0 total = 0 - for i in xrange(self.latencycount): + for i in xrange(self.latency_count): error = 0 start_time = time.time() try: @@ -244,13 +239,8 @@ def chunk_read(self, response, num, th, d, w=0, chunk_size=False, report_hook=No return [bytes_so_far, start, end] def async_get(self, conn, uri, num, th, d): - request = self.get_request(uri) - - start = 0 - end = 0 - size = 0 - + start = end = size = 0 try: response = urllib2.urlopen(request, timeout=30) size, start, end = self.chunk_read(response, num, th, d, report_hook=self.chunk_report) @@ -262,7 +252,6 @@ def async_get(self, conn, uri, num, th, d): conn.send([0, 0, False]) conn.close() return - conn.send([size, start, end]) conn.close() @@ -270,10 +259,7 @@ def async_post(self, conn, uri, num, th, d): postlen = len(self.post_data) stream = CallbackStringIO(num, th, d, self.post_data, log=self.log) request = self.post_request(uri, stream) - - start = 0 - end = 0 - + start = end = 0 try: response = urllib2.urlopen(request, timeout=30) size, start, end = self.chunk_read(response, num, th, d, 1, report_hook=self.chunk_report) @@ -285,7 +271,6 @@ def async_post(self, conn, uri, num, th, d): conn.send([0, 0, False]) conn.close() return - conn.send([postlen, start, end]) conn.close() From 8baed79f9a7c80a1fce6d8ea4df3f3dd060ba76b Mon Sep 17 00:00:00 2001 From: David Fischer Date: Sat, 24 May 2014 01:32:31 +0200 Subject: [PATCH 16/24] Convert comments to docstrings + simplify code --- tespeed.py | 116 +++++++++++++++++++++++------------------------------ utils.py | 2 +- 2 files changed, 52 insertions(+), 66 deletions(-) diff --git a/tespeed.py b/tespeed.py index e62fadb..ff2cfbe 100755 --- a/tespeed.py +++ b/tespeed.py @@ -83,11 +83,10 @@ def __init__(self, server='', num_top=0, servercount=3, unit=False, chunk_size=1 self.test_speed() def distance(self, one, two): - #Calculate the great circle distance between two points - #on the earth specified in decimal degrees (haversine formula) - #(http://stackoverflow.com/posts/4913653/revisions) - # convert decimal degrees to radians - + """ + Compute the great circle distance between two points on the earth specified in decimal degrees `haversine + formula `_ convert decimal degrees to radians. + """ lon1, lat1, lon2, lat2 = map(radians, [one[0], one[1], two[0], two[1]]) # haversine formula dlon = lon2 - lon1 @@ -98,7 +97,7 @@ def distance(self, one, two): return km def closest(self, center, points, num=5): - # Returns object that is closest to center + """Return object that is closest to center.""" closest = {} for p in xrange(len(points)): now = self.distance(center, [points[p]['lat'], points[p]['lon']]) @@ -109,21 +108,20 @@ def closest(self, center, points, num=5): else: break closest[now] = points[p] - n = 0 - ret = [] + closest_objects = [] + # FIXME can be simplified for key in sorted(closest): - ret.append(closest[key]) - n += 1 - if n >= num and num != 0: + closest_objects.append(closest[key]) + if len(closest_objects) >= num and num != 0: break - return ret + return closest_objects def test_latency(self, servers): - # Finding servers with lowest latency + """Find servers with lowest latency.""" self.log.debug('Testing latency...\n') po = [] for server in servers: - now = self.test_single_latency(server['url'] + 'latency.txt?x=' + str(time.time()))*1000 + now = self.test_single_latency(server['url'] + 'latency.txt?x=' + str(time.time())) * 1000 now = now / 2 # Evil hack or just pure stupidity? Nobody knows... if now == -1 or now == 0: continue @@ -153,8 +151,7 @@ def test_latency(self, servers): return po def test_single_latency(self, dest_addr): - # Checking latency for single server - # Does that by loading latency.txt (empty page) + """Check latency for single server. Does that by loading latency.txt (empty page).""" request = self.get_request(dest_addr) average_time = 0 @@ -177,34 +174,29 @@ def test_single_latency(self, dest_addr): return average_time / total def get_request(self, uri): - # Generates a GET request to be used with urlopen - req = urllib2.Request(uri, headers=self.headers) - return req + """Generate a GET request to be used with urlopen.""" + return urllib2.Request(uri, headers=self.headers) def post_request(self, uri, stream): - # Generate a POST request to be used with urlopen - req = urllib2.Request(uri, stream, headers=self.headers) - return req + """Generate a POST request to be used with urlopen.""" + return urllib2.Request(uri, stream, headers=self.headers) def chunk_report(self, bytes_so_far, chunk_size, total_size, num, th, d, w): - # Receiving status update from download thread - - if w == 1: - return - d[num] = bytes_so_far - down = 0 - for i in xrange(th): - down = down + d.get(i, 0) + """Receive status update from download thread.""" - if num == 0 or down >= total_size * th: + if w != 1: + d[num] = bytes_so_far + down = 0 + for i in xrange(th): + down = down + d.get(i, 0) - percent = down / (total_size * th) - percent = round(percent * 100, 2) + if num == 0 or down >= total_size * th: + percent = round(down / (total_size * th) * 100, 2) + self.log.debug('Downloaded %d of %d bytes (%0.2f%%) in %d threads\r' % + (down, total_size*th, percent, th)) - self.log.debug('Downloaded %d of %d bytes (%0.2f%%) in %d threads\r' % (down, total_size*th, percent, th)) - - #if down >= total_size*th: - # self.log.debug('\n') + #if down >= total_size*th: + # self.log.debug('\n') def chunk_read(self, response, num, th, d, w=0, chunk_size=False, report_hook=None): # self.log.debug('Thread num %d %d %d starting to report\n' % (th, num, d)) @@ -215,8 +207,7 @@ def chunk_read(self, response, num, th, d, w=0, chunk_size=False, report_hook=No if w == 1: return [0, 0, 0] - total_size = response.info().getheader('Content-Length').strip() - total_size = int(total_size) + total_size = int(response.info().getheader('Content-Length').strip()) bytes_so_far = 0 start = 0 @@ -275,7 +266,7 @@ def async_post(self, conn, uri, num, th, d): conn.close() def load_config(self): - # Load the configuration file + """Load the configuration file.""" self.log.debug('Loading speedtest configuration...\n') uri = 'http://speedtest.net/speedtest-config.php?x=' + str(time.time()) request = self.get_request(uri) @@ -294,7 +285,7 @@ def load_config(self): return {'ip': ip, 'lat': lat, 'lon': lon, 'isp': isp} def load_servers(self): - # Load server list + """Load server list.""" self.log.debug('Loading server list...\n') uri = 'http://speedtest.net/speedtest-servers.php?x=' + str(time.time()) request = self.get_request(uri) @@ -320,7 +311,7 @@ def load_servers(self): return server_list def decompress_response(sefl, response): - # Decompress gzipped response + """Decompress gzipped response.""" data = StringIO(response.read()) gzipper = gzip.GzipFile(fileobj=data) return gzipper.read() @@ -335,7 +326,9 @@ def find_best_server(self): def async_request(self, url, num, upload=0): connections = [] d = Manager().dict() - start = time.time() + + start_time = time.time() + for i in xrange(num): full_url = self.servers[i % len(self.servers)] + url #print full_url @@ -350,7 +343,7 @@ def async_request(self, url, num, upload=0): connections[c]['size'], connections[c]['start'], connections[c]['end'] = connections[c]['parent'].recv() connections[c]['connection'].join() - end = time.time() + end_time = time.time() self.log.debug(' \r') @@ -359,30 +352,25 @@ def async_request(self, url, num, upload=0): for c in xrange(num): if connections[c]['end'] is not False: #tspeed=tspeed+(connections[c]['size']/(connections[c]['end']-connections[c]['start'])) - sizes = sizes + connections[c]['size'] + sizes += connections[c]['size'] # Using more precise times for downloads if upload == 0: if c == 0: - start = connections[c]['start'] - end = connections[c]['end'] + start_time = connections[c]['start'] + end_time = connections[c]['end'] else: - if connections[c]['start'] < start: - start = connections[c]['start'] - if connections[c]['end'] > end: - end = connections[c]['end'] - - took = end - start + if connections[c]['start'] < start_time: + start_time = connections[c]['start'] + if connections[c]['end'] > end_time: + end_time = connections[c]['end'] - return [sizes, took] + return [sizes, end_time - start_time] def test_upload(self): - # Testing upload speed - - url = 'upload.php?x=' + str(time.time()) - - sizes, took = [0, 0] - data = '' + """Test upload speed.""" + data, url = '', 'upload.php?x=' + str(time.time()) + sizes = took = 0 for i in xrange(0, len(self.UPLOAD_SIZES)): if len(data) == 0 or self.UPLOAD_SIZES[i] != self.UPLOAD_SIZES[i-1]: #self.log.debug('Generating new string to upload. Length: %d\n' % (self.UPLOAD_SIZES[i])) @@ -429,7 +417,7 @@ def speed_conversion(self, data): return data / 1024 ** 2 * (1 if self.unit == 1 else 1.048576 * 8) def test_download(self): - # Testing download speed + """Test download speed.""" sizes, took = [0, 0] for i in xrange(0, len(self.DOWNLOAD_LIST)): url = 'random' + self.DOWNLOAD_LIST[i] + '.jpg?x=' + str(time.time()) + '&y=3' @@ -501,10 +489,8 @@ def list_servers(self, num=0): def main(args): if args.use_proxy: - if args.use_proxy == 5: - set_proxy(typ=socks.PROXY_TYPE_SOCKS5, host=args.proxy_host, port=args.proxy_port) - else: - set_proxy(typ=socks.PROXY_TYPE_SOCKS4, host=args.proxy_host, port=args.proxy_port) + set_proxy(typ=socks.PROXY_TYPE_SOCKS5 if args.use_proxy == 5 else socks.PROXY_TYPE_SOCKS4, host=args.proxy_host, + port=args.proxy_port) if args.listservers: args.store = True diff --git a/utils.py b/utils.py index 827df8e..813a3c3 100644 --- a/utils.py +++ b/utils.py @@ -19,8 +19,8 @@ def getaddrinfo(*args): socket.getaddrinfo = getaddrinfo -# Using StringIO with callback to measure upload progress class CallbackStringIO(StringIO): + """Using StringIO with callback to measure upload progress""" def __init__(self, num, th, d, buf='', log=None): # Force self.buf to be a string or unicode From 2e631c2d2f25fa31e106628e4257b8864c09b42e Mon Sep 17 00:00:00 2001 From: David Fischer Date: Sat, 24 May 2014 11:22:41 +0200 Subject: [PATCH 17/24] Various cleanups & improvements --- tespeed.py | 499 ++++++++++++++++++++--------------------------------- utils.py | 86 ++++++++- 2 files changed, 265 insertions(+), 320 deletions(-) diff --git a/tespeed.py b/tespeed.py index ff2cfbe..f308b13 100755 --- a/tespeed.py +++ b/tespeed.py @@ -7,12 +7,14 @@ from __future__ import absolute_import, division, print_function, unicode_literals -import argparse, gzip, urllib, urllib2, time +import argparse, urllib, urllib2, time from lxml import etree -from math import radians, cos, sin, asin, sqrt from multiprocessing import Process, Pipe, Manager -from utils import CallbackStringIO, StringIO, Log, set_proxy, socks +from utils import ( + socks, + CallbackStringIO, Log, closest, decompress_response, num_download_threads_for, num_upload_threads_for, set_proxy +) class TeSpeed(object): @@ -33,8 +35,8 @@ class TeSpeed(object): ] UPLOAD_SIZES = [ - 1024*256, 1024*256, 1024*512, 1024*512, 1024*1024, 1024*1024, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*512, - 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, + 1024*256, 1024*256, 1024*512, 1024*512, 1024*1024, 1024*1024, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*512, + 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, @@ -42,287 +44,119 @@ class TeSpeed(object): 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, - 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, - 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, + 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, + 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2 ] - def __init__(self, server='', num_top=0, servercount=3, unit=False, chunk_size=10240, log=None): - - self.headers = { - 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', - 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:11.0) Gecko/20100101 Firefox/11.0', - 'Accept-Language': 'en-us,en;q=0.5', - 'Connection': 'keep-alive', - 'Accept-Encoding': 'gzip, deflate', - #'Referer' : 'http://c.speedtest.net/flash/speedtest.swf?v=301256', - } - - self.num_servers = servercount - self.servers = [] if server == '' else [server] + HEADERS = { + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', + 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:11.0) Gecko/20100101 Firefox/11.0', + 'Accept-Language': 'en-us,en;q=0.5', + 'Connection': 'keep-alive', + 'Accept-Encoding': 'gzip, deflate', + #'Referer' : 'http://c.speedtest.net/flash/speedtest.swf?v=301256', + } + + def __init__(self, server=None, num_servers=3, num_top=0, unit=False, chunk_size=10240, log=None): + # FIXME having server, num_servers, ... looks like a hack that should be the responsibility of the caller self.server = server - self.down_speed = self.up_speed = -1 - self.latency_count = 10 - self.best_servers = 5 - self.chunk_size = chunk_size - self.log = log - + self.servers = [server] if server else [] + self.num_servers = num_servers + self.num_top = int(num_top) if unit: self.units = 'MiB' self.unit = 1 else: self.units = 'Mbit' self.unit = 0 + self.chunk_size = chunk_size + self.log = log if log.store: log.debug('Printing CSV formated results to STDOUT.\n') - self.num_top = int(num_top) - + self.best_servers = 5 + self.latency_count = 10 self.post_data = '' - self.test_speed() - - def distance(self, one, two): - """ - Compute the great circle distance between two points on the earth specified in decimal degrees `haversine - formula `_ convert decimal degrees to radians. - """ - lon1, lat1, lon2, lat2 = map(radians, [one[0], one[1], two[0], two[1]]) - # haversine formula - dlon = lon2 - lon1 - dlat = lat2 - lat1 - a = sin(dlat / 2) ** 2 + cos(lat1) * cos(lat2) * sin(dlon / 2) ** 2 - c = 2 * asin(sqrt(a)) - km = 6367 * c - return km - - def closest(self, center, points, num=5): - """Return object that is closest to center.""" - closest = {} - for p in xrange(len(points)): - now = self.distance(center, [points[p]['lat'], points[p]['lon']]) - points[p]['distance'] = now - while True: - if now in closest: - now = now + 00.1 - else: - break - closest[now] = points[p] - closest_objects = [] - # FIXME can be simplified - for key in sorted(closest): - closest_objects.append(closest[key]) - if len(closest_objects) >= num and num != 0: - break - return closest_objects - - def test_latency(self, servers): - """Find servers with lowest latency.""" - self.log.debug('Testing latency...\n') - po = [] - for server in servers: - now = self.test_single_latency(server['url'] + 'latency.txt?x=' + str(time.time())) * 1000 - now = now / 2 # Evil hack or just pure stupidity? Nobody knows... - if now == -1 or now == 0: - continue - self.log.debug('%0.0f ms latency for %s (%s, %s, %s) [%0.2f km]\n' % - (now, server['url'], server['sponsor'], server['name'], server['country'], - server['distance'])) - - server['latency'] = now - - # Pick specified ammount of servers with best latency for testing - if int(len(po)) < int(self.num_servers): - po.append(server) - else: - largest = -1 - - for x in xrange(len(po)): - if largest < 0: - if now < po[x]['latency']: - largest = x - elif po[largest]['latency'] < po[x]['latency']: - largest = x - #if cur['latency'] - - if largest >= 0: - po[largest] = server - - return po - - def test_single_latency(self, dest_addr): - """Check latency for single server. Does that by loading latency.txt (empty page).""" - request = self.get_request(dest_addr) - - average_time = 0 - total = 0 - for i in xrange(self.latency_count): - error = 0 - start_time = time.time() - try: - urllib2.urlopen(request, timeout=5) - except urllib2.URLError: - error = 1 - - if error == 0: - average_time = average_time + (time.time() - start_time) - total = total + 1 - - if total == 0: - return False - - return average_time / total def get_request(self, uri): """Generate a GET request to be used with urlopen.""" - return urllib2.Request(uri, headers=self.headers) + return urllib2.Request(uri, headers=self.HEADERS) def post_request(self, uri, stream): """Generate a POST request to be used with urlopen.""" - return urllib2.Request(uri, stream, headers=self.headers) + return urllib2.Request(uri, stream, headers=self.HEADERS) - def chunk_report(self, bytes_so_far, chunk_size, total_size, num, th, d, w): + def chunk_report(self, bytes_so_far, chunk_size, total_size, num, num_threads, d, w): """Receive status update from download thread.""" - if w != 1: d[num] = bytes_so_far down = 0 - for i in xrange(th): + for i in xrange(num_threads): down = down + d.get(i, 0) - - if num == 0 or down >= total_size * th: - percent = round(down / (total_size * th) * 100, 2) + if num == 0 or down >= total_size * num_threads: + percent = round(down / (total_size * num_threads) * 100, 2) self.log.debug('Downloaded %d of %d bytes (%0.2f%%) in %d threads\r' % - (down, total_size*th, percent, th)) + (down, total_size * num_threads, percent, num_threads)) - #if down >= total_size*th: - # self.log.debug('\n') + def chunk_read(self, response, num, num_threads, d, w=0, chunk_size=False, report_hook=None): - def chunk_read(self, response, num, th, d, w=0, chunk_size=False, report_hook=None): - # self.log.debug('Thread num %d %d %d starting to report\n' % (th, num, d)) - - if not chunk_size: - chunk_size = self.chunk_size + chunk_size = chunk_size or self.chunk_size if w == 1: return [0, 0, 0] total_size = int(response.info().getheader('Content-Length').strip()) - bytes_so_far = 0 - - start = 0 - while 1: + bytes_so_far = start_time = 0 + while True: chunk = 0 - if start == 0: - # self.log.debug('Started receiving data\n') + if not start_time: chunk = response.read(1) - start = time.time() - + start_time = time.time() else: chunk = response.read(chunk_size) if not chunk: break bytes_so_far += len(chunk) if report_hook: - report_hook(bytes_so_far, chunk_size, total_size, num, th, d, w) - end = time.time() + report_hook(bytes_so_far, chunk_size, total_size, num, num_threads, d, w) - return [bytes_so_far, start, end] + return [bytes_so_far, start_time, time.time()] - def async_get(self, conn, uri, num, th, d): + def async_get(self, conn, uri, num, num_threads, d): request = self.get_request(uri) - start = end = size = 0 + start_time = end_time = size = 0 try: response = urllib2.urlopen(request, timeout=30) - size, start, end = self.chunk_read(response, num, th, d, report_hook=self.chunk_report) - #except urllib2.URLError, e: - # self.log.debug('Failed downloading.\n') + size, start_time, end_time = self.chunk_read(response, num, num_threads, d, report_hook=self.chunk_report) except: - self.log.debug(' \r') + self.log.debug(Log.BLANK_LINE) self.log.debug('Failed downloading.\n') conn.send([0, 0, False]) conn.close() return - conn.send([size, start, end]) + conn.send([size, start_time, end_time]) conn.close() - def async_post(self, conn, uri, num, th, d): + def async_post(self, conn, uri, num, num_threads, d): postlen = len(self.post_data) - stream = CallbackStringIO(num, th, d, self.post_data, log=self.log) + stream = CallbackStringIO(num, num_threads, d, self.post_data, log=self.log) request = self.post_request(uri, stream) - start = end = 0 + start_time = end_time = 0 try: response = urllib2.urlopen(request, timeout=30) - size, start, end = self.chunk_read(response, num, th, d, 1, report_hook=self.chunk_report) - #except urllib2.URLError: - # self.log.debug('Failed uploading.\n') + size, start_time, end_time = self.chunk_read(response, num, num_threads, d, 1, + report_hook=self.chunk_report) except: - self.log.debug(' \r') + self.log.debug(Log.BLANK_LINE) self.log.debug('Failed uploading.\n') conn.send([0, 0, False]) conn.close() return - conn.send([postlen, start, end]) + conn.send([postlen, start_time, end_time]) conn.close() - def load_config(self): - """Load the configuration file.""" - self.log.debug('Loading speedtest configuration...\n') - uri = 'http://speedtest.net/speedtest-config.php?x=' + str(time.time()) - request = self.get_request(uri) - response = urllib2.urlopen(request) - - # Load etree from XML data - config = etree.fromstring(self.decompress_response(response)) - - ip = config.find('client').attrib['ip'] - isp = config.find('client').attrib['isp'] - lat = float(config.find('client').attrib['lat']) - lon = float(config.find('client').attrib['lon']) - - self.log.debug('IP: %s; Lat: %f; Lon: %f; ISP: %s\n' % (ip, lat, lon, isp)) - - return {'ip': ip, 'lat': lat, 'lon': lon, 'isp': isp} - - def load_servers(self): - """Load server list.""" - self.log.debug('Loading server list...\n') - uri = 'http://speedtest.net/speedtest-servers.php?x=' + str(time.time()) - request = self.get_request(uri) - response = urllib2.urlopen(request) - - # Load etree from XML data - servers_xml = etree.fromstring(self.decompress_response(response)) - servers = servers_xml.find('servers').findall('server') - server_list = [] - - for server in servers: - server_list.append({ - 'lat': float(server.attrib['lat']), - 'lon': float(server.attrib['lon']), - 'url': server.attrib['url'].rsplit('/', 1)[0] + '/', - #'url2': server.attrib['url2'].rsplit('/', 1)[0] + '/', - 'name': server.attrib['name'], - 'country': server.attrib['country'], - 'sponsor': server.attrib['sponsor'], - 'id': server.attrib['id'], - }) - - return server_list - - def decompress_response(sefl, response): - """Decompress gzipped response.""" - data = StringIO(response.read()) - gzipper = gzip.GzipFile(fileobj=data) - return gzipper.read() - - def find_best_server(self): - self.log.debug('Looking for closest and best server...\n') - best = self.test_latency(self.closest([self.config['lat'], self.config['lon']], - self.server_list, self.best_servers)) - for server in best: - self.servers.append(server['url']) - def async_request(self, url, num, upload=0): connections = [] d = Manager().dict() @@ -331,7 +165,6 @@ def async_request(self, url, num, upload=0): for i in xrange(num): full_url = self.servers[i % len(self.servers)] + url - #print full_url connection = {} connection['parent'], connection['child'] = Pipe() connection['connection'] = Process(target=self.async_post if upload == 1 else self.async_get, @@ -345,13 +178,11 @@ def async_request(self, url, num, upload=0): end_time = time.time() - self.log.debug(' \r') + self.log.debug(Log.BLANK_LINE) sizes = 0 - #tspeed=0 for c in xrange(num): if connections[c]['end'] is not False: - #tspeed=tspeed+(connections[c]['size']/(connections[c]['end']-connections[c]['start'])) sizes += connections[c]['size'] # Using more precise times for downloads @@ -367,123 +198,161 @@ def async_request(self, url, num, upload=0): return [sizes, end_time - start_time] - def test_upload(self): - """Test upload speed.""" - data, url = '', 'upload.php?x=' + str(time.time()) - sizes = took = 0 - for i in xrange(0, len(self.UPLOAD_SIZES)): - if len(data) == 0 or self.UPLOAD_SIZES[i] != self.UPLOAD_SIZES[i-1]: - #self.log.debug('Generating new string to upload. Length: %d\n' % (self.UPLOAD_SIZES[i])) - data = ''.join('1' for x in xrange(self.UPLOAD_SIZES[i])) - self.post_data = urllib.urlencode({'upload6': data}) + def load_config(self): + """Load the configuration file and return it as a dictionary.""" + self.log.debug('Loading speedtest configuration...\n') + uri = 'http://speedtest.net/speedtest-config.php?x=' + str(time.time()) + request = self.get_request(uri) + response = urllib2.urlopen(request) - if i < 2: - thrds = 1 - elif i < 5: - thrds = 2 - elif i < 7: - thrds = 2 - elif i < 10: - thrds = 3 - elif i < 25: - thrds = 6 - elif i < 45: - thrds = 4 - elif i < 65: - thrds = 3 - else: - thrds = 2 + # Load etree from XML data + client = etree.fromstring(decompress_response(response)).find('client') + config_dict = { + 'ip': client.attrib['ip'], + 'isp': client.attrib['isp'], + 'lat': float(client.attrib['lat']), + 'lon': float(client.attrib['lon']) + } + self.log.debug('IP: {ip}; Lat: {lat}; Lon: {lon}; ISP: {isp}\n'.format(**config_dict)) + return config_dict - sizes, took = self.async_request(url, thrds, 1) - #sizes, took=self.async_request(url, (i<4 and 1 or (i<6 and 2 or (i<6 and 4 or 8))), 1) - if sizes == 0: - continue + def find_best_server(self): + self.log.debug('Looking for closest and best server...\n') + best_servers = self.test_latency(closest([self.config['lat'], self.config['lon']], self.server_list, + self.best_servers)) + self.servers.extend(server['url'] for server in best_servers) - size = self.speed_conversion(sizes) - speed = size / took - self.log.debug('Upload size: %0.2f MiB; Uploaded in %0.2f s\n' % (size, took)) - self.log.debug('\033[92mUpload speed: %0.2f %s/s\033[0m\n' % (speed, self.units)) + def load_servers(self): + """Load and return server list.""" + self.log.debug('Loading server list...\n') + uri = 'http://speedtest.net/speedtest-servers.php?x=' + str(time.time()) + request = self.get_request(uri) + response = urllib2.urlopen(request) - if self.up_speed < speed: - self.up_speed = speed + # Load etree from XML data + servers_xml = etree.fromstring(decompress_response(response)) + return [ + { + 'lat': float(server.attrib['lat']), + 'lon': float(server.attrib['lon']), + 'url': server.attrib['url'].rsplit('/', 1)[0] + '/', + #'url2': server.attrib['url2'].rsplit('/', 1)[0] + '/', + 'name': server.attrib['name'], + 'country': server.attrib['country'], + 'sponsor': server.attrib['sponsor'], + 'id': server.attrib['id'], + } for server in servers_xml.find('servers').findall('server') + ] - if took > 5: - break + def list_servers(self, num=0): + + all_sorted = closest([self.config['lat'], self.config['lon']], self.server_list, num) - #self.log.debug('Upload size: %0.2f MiB; Uploaded in %0.2f s\n' % (self.speed_conversion(sizes), took)) - #self.log.debug('Upload speed: %0.2f MiB/s\n' % (self.speed_conversion(sizes)/took)) + for i in xrange(len(all_sorted)): + self.log.result('%s. %s (%s, %s, %s) [%0.2f km]\n' % + (i + 1, all_sorted[i]['url'], all_sorted[i]['sponsor'], all_sorted[i]['name'], + all_sorted[i]['country'], all_sorted[i]['distance'])) def speed_conversion(self, data): return data / 1024 ** 2 * (1 if self.unit == 1 else 1.048576 * 8) + def test_latency(self, servers): + """Find servers with lowest latency.""" + self.log.debug('Testing latency...\n') + po = [] + for server in servers: + latency = self.test_single_latency(server['url'] + 'latency.txt?x=' + str(time.time())) * 1000 + if not latency: + continue + self.log.debug('%0.0f ms latency for %s (%s, %s, %s) [%0.2f km]\n' % + (latency, server['url'], server['sponsor'], server['name'], server['country'], + server['distance'])) + server['latency'] = latency + # Pick specified amount of servers with best latency for testing + if int(len(po)) < int(self.num_servers): + po.append(server) + else: + largest = -1 + for x in xrange(len(po)): + if largest < 0: + if latency < po[x]['latency']: + largest = x + elif po[largest]['latency'] < po[x]['latency']: + largest = x + if largest >= 0: + po[largest] = server + return po + + def test_single_latency(self, destination_address): + """Check latency for single server. Does that by loading latency.txt (empty page).""" + request = self.get_request(destination_address) + average_time = total = 0 + for i in xrange(self.latency_count): + start_time = time.time() + try: + urllib2.urlopen(request, timeout=5) + average_time += time.time() - start_time + total += 2 # Already "multiply" by 2 to return half of the round-trip (the latency) for free + except urllib2.URLError: + pass + return average_time / total if total else None + def test_download(self): """Test download speed.""" - sizes, took = [0, 0] - for i in xrange(0, len(self.DOWNLOAD_LIST)): + max_speed = -1 + for i in xrange(len(self.DOWNLOAD_LIST)): url = 'random' + self.DOWNLOAD_LIST[i] + '.jpg?x=' + str(time.time()) + '&y=3' - if i < 2: - thrds = 1 - elif i < 5: - thrds = 2 - elif i < 11: - thrds = 2 - elif i < 13: - thrds = 4 - elif i < 25: - thrds = 2 - elif i < 45: - thrds = 3 - elif i < 65: - thrds = 2 - else: - thrds = 2 - - sizes, took = self.async_request(url, thrds) - #sizes, took=self.async_request(url, (i<1 and 2 or (i<6 and 4 or (i<10 and 6 or 8))) ) + sizes, took = self.async_request(url, num_download_threads_for(i)) if sizes == 0: continue - size = self.speed_conversion(sizes) + size = self.speed_conversion(sizes) # FIXME rename sizes to something more meaningful speed = size / took + max_speed = max(speed, max_speed) self.log.debug('Download size: %0.2f MiB; Downloaded in %0.2f s\n' % (size, took)) self.log.debug('\033[91mDownload speed: %0.2f %s/s\033[0m\n' % (speed, self.units)) - if self.down_speed < speed: - self.down_speed = speed - if took > 5: break + return max_speed + + def test_upload(self): + """Test upload speed.""" + max_speed = -1 + data, url = '', 'upload.php?x=' + str(time.time()) + for i in xrange(len(self.UPLOAD_SIZES)): + if len(data) == 0 or self.UPLOAD_SIZES[i] != self.UPLOAD_SIZES[i-1]: + data = ''.join('1' for x in xrange(self.UPLOAD_SIZES[i])) + self.post_data = urllib.urlencode({'upload6': data}) + + sizes, took = self.async_request(url, num_upload_threads_for(i), 1) + if sizes == 0: + continue - #self.log.debug('Download size: %0.2f MiB; Downloaded in %0.2f s\n' % (self.speed_conversion(sizes), took)) - #self.log.debug('Download speed: %0.2f %s/s\n' % (self.speed_conversion(sizes)/took, self.units)) + size = self.speed_conversion(sizes) # FIXME rename sizes to something more meaningful + speed = size / took + max_speed = max(speed, max_speed) + self.log.debug('Upload size: %0.2f MiB; Uploaded in %0.2f s\n' % (size, took)) + self.log.debug('\033[92mUpload speed: %0.2f %s/s\033[0m\n' % (speed, self.units)) - def test_speed(self): + if took > 5: + break + return max_speed + def run_tests(self): if self.server == 'list-servers': self.config = self.load_config() self.server_list = self.load_servers() self.list_servers(self.num_top) - return - - if self.server == '': - self.config = self.load_config() - self.server_list = self.load_servers() - self.find_best_server() - - self.test_download() - self.test_upload() - - self.log.result('%0.2f,%0.2f,"%s","%s"\n' % (self.down_speed, self.up_speed, self.units, self.servers)) - - def list_servers(self, num=0): - - all_sorted = self.closest([self.config['lat'], self.config['lon']], self.server_list, num) - - for i in xrange(0, len(all_sorted)): - self.log.result('%s. %s (%s, %s, %s) [%0.2f km]\n' % - (i + 1, all_sorted[i]['url'], all_sorted[i]['sponsor'], all_sorted[i]['name'], - all_sorted[i]['country'], all_sorted[i]['distance'])) + else: + if not self.server: + self.config = self.load_config() + self.server_list = self.load_servers() + self.find_best_server() + download_speed = self.test_download() + upload_speed = self.test_upload() + self.log.result('%0.2f,%0.2f,"%s","%s"\n' % (download_speed, upload_speed, self.units, self.servers)) def main(args): @@ -501,11 +370,11 @@ def main(args): else: log.debug('Getting ready\n') try: - TeSpeed(args.listservers and 'list-servers' or args.server, args.listservers, args.servercount, args.unit, - chunk_size=args.chunksize, log=log) + tespeed = TeSpeed(server=args.listservers and 'list-servers' or args.server, num_servers=args.servercount, + num_top=args.listservers, unit=args.unit, chunk_size=args.chunksize, log=log) + tespeed.run_tests() except (KeyboardInterrupt, SystemExit): - log.debug('\nTesting stopped.\n') - #raise + log.debug('\nSpeed test stopped.\n') if __name__ == '__main__': parser = argparse.ArgumentParser(description='TeSpeed, CLI SpeedTest.net') diff --git a/utils.py b/utils.py index 813a3c3..05a73b9 100644 --- a/utils.py +++ b/utils.py @@ -6,11 +6,16 @@ from __future__ import absolute_import, division, print_function, unicode_literals -import socket, sys +import gzip, socket, sys +from math import radians, cos, sin, asin, sqrt from SocksiPy import socks from StringIO import StringIO -__all__ = ('CallbackStringIO', 'StringIO', 'print_debug', 'print_result', 'set_proxy', 'socks') +__all__ = ( + 'socks', 'StringIO', # also include things from other modules + 'CallbackStringIO', 'Log', 'closest', 'decompress_response', 'distance', 'num_download_threads_for', + 'num_upload_threads_for', 'set_proxy' +) # Magic! @@ -42,7 +47,6 @@ def read(self, n=10240): next_chunk = StringIO.read(self, n) #if 'done' in self.d: # return - self.d[self.num] = self.pos down = 0 for i in xrange(self.th): @@ -53,11 +57,9 @@ def read(self, n=10240): if self.log: self.log.debug('Uploaded %d of %d bytes (%0.2f%%) in %d threads\r' % (down, self.total, percent, self.th)) - #if down >= self.total: # if self.log: self.log.debug('\n') # self.d['done']=1 - return next_chunk def __len__(self): @@ -66,6 +68,8 @@ def __len__(self): class Log(object): + BLANK_LINE = ' \r' + def __init__(self, suppress=False, store=False): self.suppress = suppress self.store = store @@ -79,6 +83,78 @@ def result(self, string): sys.stdout.write(string.encode('utf8')) +def closest(center, points, num=5): + """Return object that is closest to center.""" + closest = {} + for p in xrange(len(points)): + p_distance = distance(center, [points[p]['lat'], points[p]['lon']]) + points[p]['distance'] = p_distance + while True: + if p_distance in closest: + p_distance = p_distance + 00.1 + else: + break + closest[p_distance] = points[p] + closest_objects = [] + # FIXME can be simplified + for key in sorted(closest): + closest_objects.append(closest[key]) + if len(closest_objects) >= num and num != 0: + break + return closest_objects + + +def decompress_response(response): + """Return decompressed gzip response.""" + data = StringIO(response.read()) + gzipper = gzip.GzipFile(fileobj=data) + return gzipper.read() + + +def distance(one, two): + """ + Compute the great circle distance between two points on the earth specified in decimal degrees `haversine + formula `_ convert decimal degrees to radians. + """ + lon1, lat1, lon2, lat2 = map(radians, [one[0], one[1], two[0], two[1]]) + a = sin((lat2 - lat1) / 2) ** 2 + cos(lat1) * cos(lat2) * sin((lon2 - lon1) / 2) ** 2 + c = 2 * asin(sqrt(a)) + km = 6367 * c + return km + + +def num_download_threads_for(num_downloads): + if num_downloads < 2: + return 1 + elif num_downloads < 11: + return 2 + elif num_downloads < 13: + return 4 + elif num_downloads < 25: + return 2 + elif num_downloads < 45: + return 3 + elif num_downloads < 65: + return 2 + return 2 + + +def num_upload_threads_for(num_uploads): + if num_uploads < 2: + return 1 + elif num_uploads < 7: + return 2 + elif num_uploads < 10: + return 3 + elif num_uploads < 25: + return 6 + elif num_uploads < 45: + return 4 + elif num_uploads < 65: + return 3 + return 2 + + # Thanks to Ryan Sears for http://bit.ly/17HhSli def set_proxy(typ=socks.PROXY_TYPE_SOCKS4, host='127.0.0.1', port=9050): socks.setdefaultproxy(typ, host, port) From 8c570e9b6ee272bbc921b5c721b2b7163a607ae9 Mon Sep 17 00:00:00 2001 From: David Fischer Date: Sat, 24 May 2014 13:10:07 +0200 Subject: [PATCH 18/24] Use a += b instead of a = a + b --- tespeed.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tespeed.py b/tespeed.py index f308b13..7cffeae 100755 --- a/tespeed.py +++ b/tespeed.py @@ -94,7 +94,7 @@ def chunk_report(self, bytes_so_far, chunk_size, total_size, num, num_threads, d d[num] = bytes_so_far down = 0 for i in xrange(num_threads): - down = down + d.get(i, 0) + down += d.get(i, 0) if num == 0 or down >= total_size * num_threads: percent = round(down / (total_size * num_threads) * 100, 2) self.log.debug('Downloaded %d of %d bytes (%0.2f%%) in %d threads\r' % From aa010fbe7d574621bdaeb49b829d43d9b2f008b2 Mon Sep 17 00:00:00 2001 From: David Fischer Date: Sat, 24 May 2014 13:10:40 +0200 Subject: [PATCH 19/24] Rename speed_conversion to convert_size --- tespeed.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tespeed.py b/tespeed.py index 7cffeae..de388ca 100755 --- a/tespeed.py +++ b/tespeed.py @@ -80,6 +80,9 @@ def __init__(self, server=None, num_servers=3, num_top=0, unit=False, chunk_size self.latency_count = 10 self.post_data = '' + def convert_size(self, value): + return value / 1024 ** 2 * (1 if self.unit == 1 else 1.048576 * 8) + def get_request(self, uri): """Generate a GET request to be used with urlopen.""" return urllib2.Request(uri, headers=self.HEADERS) @@ -253,9 +256,6 @@ def list_servers(self, num=0): (i + 1, all_sorted[i]['url'], all_sorted[i]['sponsor'], all_sorted[i]['name'], all_sorted[i]['country'], all_sorted[i]['distance'])) - def speed_conversion(self, data): - return data / 1024 ** 2 * (1 if self.unit == 1 else 1.048576 * 8) - def test_latency(self, servers): """Find servers with lowest latency.""" self.log.debug('Testing latency...\n') @@ -307,7 +307,7 @@ def test_download(self): if sizes == 0: continue - size = self.speed_conversion(sizes) # FIXME rename sizes to something more meaningful + size = self.convert_size(sizes) # FIXME rename sizes to something more meaningful speed = size / took max_speed = max(speed, max_speed) self.log.debug('Download size: %0.2f MiB; Downloaded in %0.2f s\n' % (size, took)) @@ -330,7 +330,7 @@ def test_upload(self): if sizes == 0: continue - size = self.speed_conversion(sizes) # FIXME rename sizes to something more meaningful + size = self.convert_size(sizes) # FIXME rename sizes to something more meaningful speed = size / took max_speed = max(speed, max_speed) self.log.debug('Upload size: %0.2f MiB; Uploaded in %0.2f s\n' % (size, took)) From 881d55b7524ee45c836d766ce04ee249ac9a6cb0 Mon Sep 17 00:00:00 2001 From: David Fischer Date: Sat, 24 May 2014 13:11:17 +0200 Subject: [PATCH 20/24] Load methods actually update attributes of the instance --- tespeed.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/tespeed.py b/tespeed.py index de388ca..cd51e39 100755 --- a/tespeed.py +++ b/tespeed.py @@ -202,7 +202,7 @@ def async_request(self, url, num, upload=0): return [sizes, end_time - start_time] def load_config(self): - """Load the configuration file and return it as a dictionary.""" + """Load the configuration file.""" self.log.debug('Loading speedtest configuration...\n') uri = 'http://speedtest.net/speedtest-config.php?x=' + str(time.time()) request = self.get_request(uri) @@ -210,14 +210,13 @@ def load_config(self): # Load etree from XML data client = etree.fromstring(decompress_response(response)).find('client') - config_dict = { + self.config = { 'ip': client.attrib['ip'], 'isp': client.attrib['isp'], 'lat': float(client.attrib['lat']), 'lon': float(client.attrib['lon']) } - self.log.debug('IP: {ip}; Lat: {lat}; Lon: {lon}; ISP: {isp}\n'.format(**config_dict)) - return config_dict + self.log.debug('IP: {ip}; Lat: {lat}; Lon: {lon}; ISP: {isp}\n'.format(**self.config)) def find_best_server(self): self.log.debug('Looking for closest and best server...\n') @@ -226,7 +225,7 @@ def find_best_server(self): self.servers.extend(server['url'] for server in best_servers) def load_servers(self): - """Load and return server list.""" + """Load server list.""" self.log.debug('Loading server list...\n') uri = 'http://speedtest.net/speedtest-servers.php?x=' + str(time.time()) request = self.get_request(uri) @@ -234,7 +233,7 @@ def load_servers(self): # Load etree from XML data servers_xml = etree.fromstring(decompress_response(response)) - return [ + self.server_list = [ { 'lat': float(server.attrib['lat']), 'lon': float(server.attrib['lon']), @@ -342,13 +341,13 @@ def test_upload(self): def run_tests(self): if self.server == 'list-servers': - self.config = self.load_config() - self.server_list = self.load_servers() + self.load_config() + self.load_servers() self.list_servers(self.num_top) else: if not self.server: - self.config = self.load_config() - self.server_list = self.load_servers() + self.load_config() + self.load_servers() self.find_best_server() download_speed = self.test_download() upload_speed = self.test_upload() From 639010100242c717125abf796ba424d0106331a2 Mon Sep 17 00:00:00 2001 From: David Fischer Date: Sat, 24 May 2014 13:20:29 +0200 Subject: [PATCH 21/24] Fix wording + use underscores --- tespeed.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tespeed.py b/tespeed.py index cd51e39..73e5bc2 100755 --- a/tespeed.py +++ b/tespeed.py @@ -360,17 +360,17 @@ def main(args): set_proxy(typ=socks.PROXY_TYPE_SOCKS5 if args.use_proxy == 5 else socks.PROXY_TYPE_SOCKS4, host=args.proxy_host, port=args.proxy_port) - if args.listservers: + if args.list_servers: args.store = True log = Log(suppress=args.suppress, store=args.store) - if not args.listservers and args.server == '' and not args.store: + if not args.list_servers and args.server == '' and not args.store: log.debug('Getting ready. Use parameter -h or --help to see available features.\n') else: log.debug('Getting ready\n') try: - tespeed = TeSpeed(server=args.listservers and 'list-servers' or args.server, num_servers=args.servercount, - num_top=args.listservers, unit=args.unit, chunk_size=args.chunksize, log=log) + tespeed = TeSpeed(server=args.list_servers and 'list-servers' or args.server, num_servers=args.server_count, + num_top=args.list_servers, unit=args.unit, chunk_size=args.chunk_size, log=log) tespeed.run_tests() except (KeyboardInterrupt, SystemExit): log.debug('\nSpeed test stopped.\n') @@ -379,17 +379,17 @@ def main(args): parser = argparse.ArgumentParser(description='TeSpeed, CLI SpeedTest.net') parser.add_argument('server', nargs='?', type=str, default='', help='Use the specified server for testing (skip checking for location and closest server).') - parser.add_argument('-ls', '--list-servers', dest='listservers', nargs='?', default=0, const=10, help='List the servers sorted by distance, nearest first. Optionally specify number of servers to show.') + parser.add_argument('-ls', '--list-servers', dest='list_servers', nargs='?', default=0, const=10, help='List the servers sorted by distance, nearest first. Optionally specify number of servers to show.') parser.add_argument('-w', '--csv', dest='store', action='store_true', help='Print CSV formated output to STDOUT.') parser.add_argument('-s', '--suppress', dest='suppress', action='store_true', help='Suppress debugging (STDERR) output.') parser.add_argument('-mib', '--mebibit', dest='unit', action='store_true', help='Show results in mebibits.') - parser.add_argument('-n', '--server-count', dest='servercount', nargs='?', default=1, const=1, help='Specify how many different servers should be used in paralel. (Default: 1) (Increase it for >100Mbit testing.)') + parser.add_argument('-n', '--server-count', dest='server_count', nargs='?', default=1, const=1, help='Specify how many different servers should be used in parallel. (Default: 1) (Increase it for >100Mbit testing.)') parser.add_argument('-p', '--proxy', dest='use_proxy', type=int, nargs='?', const=4, help='Specify 4 or 5 to use SOCKS4 or SOCKS5 proxy.') parser.add_argument('-ph', '--proxy-host', dest='proxy_host', type=str, nargs='?', default='127.0.0.1', help='Specify socks proxy host. (Default: 127.0.0.1)') parser.add_argument('-pp', '--proxy-port', dest='proxy_port', type=int, nargs='?', default=9050, help='Specify socks proxy port. (Default: 9050)') - parser.add_argument('-cs', '--chunk-size', dest='chunksize', nargs='?', type=int, default=10240, help='Specify chunk size after wich tespeed calculates speed. Increase this number 4 or 5 times if you use weak hardware like RaspberryPi. (Default: 10240)') + parser.add_argument('-cs', '--chunk-size', dest='chunk_size', nargs='?', type=int, default=10240, help='Specify chunk size after which tespeed calculates speed. Increase this number 4 or 5 times if you use weak hardware like RaspberryPi. (Default: 10240)') #parser.add_argument('-i', '--interface', dest='interface', nargs='?', help='If specified, measures speed from data for the whole network interface.') From d4a51b45da2746c7cacff6a9ae7cf7101c912130 Mon Sep 17 00:00:00 2001 From: David Fischer Date: Sat, 24 May 2014 14:57:20 +0200 Subject: [PATCH 22/24] Make this project a package with setup.py and make it available as a console script --- .gitignore | 4 +- .gitmodules | 3 - README | 123 ------------------------------- README.rst | 104 ++++++++++++++++++++++++++ SocksiPy | 1 - requirements.txt | 3 + serverlist.txt => serverlist.xml | 0 setup.py | 60 +++++++++++++++ tespeed/__init__.py | 0 tespeed/bin.py | 66 +++++++++++++++++ tespeed.py => tespeed/core.py | 52 +------------ utils.py => tespeed/utils.py | 8 +- 12 files changed, 243 insertions(+), 181 deletions(-) delete mode 100644 .gitmodules delete mode 100644 README create mode 100644 README.rst delete mode 160000 SocksiPy create mode 100644 requirements.txt rename serverlist.txt => serverlist.xml (100%) create mode 100755 setup.py create mode 100644 tespeed/__init__.py create mode 100644 tespeed/bin.py rename tespeed.py => tespeed/core.py (82%) rename utils.py => tespeed/utils.py (94%) diff --git a/.gitignore b/.gitignore index 2bc1b44..ade8000 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ -old/ +build/ +dist/ +*.egg* *~ ~ *.pyc diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 091804b..0000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "SocksiPy"] - path = SocksiPy - url = git://github.com/Janhouse/SocksiPy.git diff --git a/README b/README deleted file mode 100644 index a1ac4ab..0000000 --- a/README +++ /dev/null @@ -1,123 +0,0 @@ -Tespeed (terminal speedtest) - Copyright 2012 Janis Jansons (janis.jansons@janhouse.lv) - - - This is a new Tespeed version written in Python (for the purpose of learning it). - - The old one was written in PHP years ago and wasn't really made for general - public (was fine tuned and possibly working only on my server). - - Even though the old version didn't work on most boxes, it somehow got - almost 17 000 downloads on Sourceforge. I guess some people could use this - (those who hate Flash, JavaScript, has GUI-less servers, etc.) so I'll - try to make this one a bit better working in time. - - Let's call this version 1.0-alpha - - - Of course, this script could not work like this without the best speed - testing site out there - http://www.speedtest.net/ - Support them in any way you can (going to their website and clicking on - ads could probably make them a bit happier). :) - - -Requirements: - - This script requires recent Python (preferably 2.7 or newer) and Python2 - modules lxml and argparse. - Install python-lxml and python-argparse (Debian) or python2-lxml (Archlinux). - -Installation: - - If you have Debian, you might have to change the python executable in tuper.py - or create a symlink for your existing python2.x executable by doing: - - sudo ln -s /usr/bin/python2.7 /usr/bin/python2 - - If you have python2.6 then replace python2.7 with python2.6. - - - - When doing the checkout, remember to pull submodules. - - If you have a decent git version (1.6.5 and up), get everything by doing: - - git clone --recursive git://github.com/Janhouse/tespeed.git - - Otherwise do: - - git clone git://github.com/Janhouse/tespeed.git - cd tespeed - git submodule init - git submodule update - - -Usage: - - usage: tespeed.py [-h] [-ls [LISTSERVERS]] [-w] [-s] [-mib] [-n [SERVERCOUNT]] - [-p [USE_PROXY]] [-ph [PROXY_HOST]] [-pp [PROXY_PORT]] - [server] - - TeSpeed, CLI SpeedTest.net - - positional arguments: - server Use the specified server for testing (skip checking - for location and closest server). - - optional arguments: - -h, --help show this help message and exit - -ls [LISTSERVERS], --list-servers [LISTSERVERS] - List the servers sorted by distance, nearest first. - Optionally specify number of servers to show. - -w, --csv Print CSV formated output to STDOUT. - -s, --suppress Suppress debugging (STDERR) output. - -mib, --mebibit Show results in mebibits. - -n [SERVERCOUNT], --server-count [SERVERCOUNT] - Specify how many different servers should be used in - paralel. (Defaults to 1.) (Increase it for >100Mbit - testing.) - -p [USE_PROXY], --proxy [USE_PROXY] - Specify 4 or 5 to use SOCKS4 or SOCKS5 proxy. - -ph [PROXY_HOST], --proxy-host [PROXY_HOST] - Specify socks proxy host (defaults to 127.0.0.1). - -pp [PROXY_PORT], --proxy-port [PROXY_PORT] - Specify socks proxy port (defaults to 9050). - -cs [CHUNKSIZE], --chunk-size [CHUNKSIZE] - Specify chunk size after wich tespeed calculates - speed. Increase this number 4 or 5 times if you use - weak hardware like RaspberryPi. (Default: 10240) - - -What the script does: - - * Loads config from speedtest.net (http://speedtest.net/speedtest-config.php). - - * Gets server list (http://speedtest.net/speedtest-servers.php). - - * Picks 5 closests servers using the coordinates provides by speedtest.net - config and serverlist. - - * Checks latency for those servers and picks one with the lowest. - - * Does download speed test and returns results. - - * Does upload speed test and returns results. - - * Optionally can return CSV formated results. - - * Can measure through SOCKS proxy. - -TODO (ideas): - - * Add more error checking. - * Make it less messy. - * Send found results to speedtest.net API (needs some hash?) and get the link - to the generated image. - * Store the measurement data and draw graphs. - * Measure the speed for the whole network interface (similar like it was in the - old version of Tespeed). - * Start upload timer only after 1st byte is read. - * Figure out the ammount of data that was transfered only when all threads were - actively sendong/receiving data at the same time. (Should provide more precise - test results.) - - diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..35035b2 --- /dev/null +++ b/README.rst @@ -0,0 +1,104 @@ +============================ +Tespeed (terminal speedtest) +============================ + +:copyright: Copyright 2012 Janis Jansons (janis.jansons@janhouse.lv) + +This is a new TeSpeed version written in Python (for the purpose of learning it). + +The old one was written in PHP years ago and wasn't really made for general public (was fine tuned and possibly working +only on my server). + +Even though the old version didn't work on most boxes, it somehow got almost 17'000 downloads on Sourceforge. +I guess some people could use this (those who hate Flash, JavaScript, has GUI-less servers, etc.) so I'll try to make +this one a bit better working in time. + +Let's call this version 0.1.0-alpha + +Of course, this script could not work like this without the best speed testing site out there - +http://www.speedtest.net/ +Support them in any way you can (going to their website and clicking on ads could probably make them a bit happier). :) + +------------ +Installation +------------ + +When doing the checkout, remember to pull submodules. + +If you have a decent git version (1.6.5 and up), get everything by doing:: + + git clone --recursive git://github.com/Janhouse/tespeed.git + +Otherwise do:: + + git clone git://github.com/Janhouse/tespeed.git + cd tespeed + git submodule init + git submodule update + +Then install it thanks to the project's setup script:: + + sudo ./setup.py install + +----- +Usage +----- + +:: + + usage: tespeed.py [-h] [-ls [LISTSERVERS]] [-w] [-s] [-mib] [-n [SERVERCOUNT]] + [-p [USE_PROXY]] [-ph [PROXY_HOST]] [-pp [PROXY_PORT]] + [server] + + TeSpeed, CLI SpeedTest.net + + positional arguments: + server Use the specified server for testing (skip checking + for location and closest server). + + optional arguments: + -h, --help show this help message and exit + -ls [LISTSERVERS], --list-servers [LISTSERVERS] + List the servers sorted by distance, nearest first. + Optionally specify number of servers to show. + -w, --csv Print CSV formated output to STDOUT. + -s, --suppress Suppress debugging (STDERR) output. + -mib, --mebibit Show results in mebibits. + -n [SERVERCOUNT], --server-count [SERVERCOUNT] + Specify how many different servers should be used in + paralel. (Defaults to 1.) (Increase it for >100Mbit + testing.) + -p [USE_PROXY], --proxy [USE_PROXY] + Specify 4 or 5 to use SOCKS4 or SOCKS5 proxy. + -ph [PROXY_HOST], --proxy-host [PROXY_HOST] + Specify socks proxy host (defaults to 127.0.0.1). + -pp [PROXY_PORT], --proxy-port [PROXY_PORT] + Specify socks proxy port (defaults to 9050). + -cs [CHUNKSIZE], --chunk-size [CHUNKSIZE] + Specify chunk size after wich tespeed calculates + speed. Increase this number 4 or 5 times if you use + weak hardware like RaspberryPi. (Default: 10240) + +What the script does: + +* Loads configuration from speedtest.net (http://speedtest.net/speedtest-config.php). +* Gets server list (http://speedtest.net/speedtest-servers.php). +* Picks 5 closest servers using the coordinates provides by speedtest.net config and serverlist. +* Checks latency for those servers and picks one with the lowest. +* Does download speed test and returns results. +* Does upload speed test and returns results. +* Optionally can return CSV formated results. +* Can measure through SOCKS proxy. + +TODO (ideas): + +* Add more error checking. +* Make it less messy. +* Send found results to speedtest.net API (needs some hash?) and get the link to the generated image. +* Store the measurement data and draw graphs. +* Measure the speed for the whole network interface (similar like it was in the old version of Tespeed). +* Start upload timer only after 1st byte is read. +* Figure out the amount of data that was transfered only when all threads were actively sending/receiving data at the + same time. (Should provide more precise test results.) + + diff --git a/SocksiPy b/SocksiPy deleted file mode 160000 index 842d496..0000000 --- a/SocksiPy +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 842d4962cbce16ce4b232d1b7402d0375f9a0c1b diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..0564b6c --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +argparse +lxml +-e git://github.com/Anorov/PySocks.git#egg=PySocks diff --git a/serverlist.txt b/serverlist.xml similarity index 100% rename from serverlist.txt rename to serverlist.xml diff --git a/setup.py b/setup.py new file mode 100755 index 0000000..c3c2170 --- /dev/null +++ b/setup.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright: +# 2012-2013 Janis Jansons (janis.jansons@janhouse.lv) +# 2014 David Fischer (david.fischer.ch@gmail.com) + +from __future__ import absolute_import, division, print_function, unicode_literals + +import sys +from codecs import open +from pip.req import parse_requirements +from setuptools import setup, find_packages + +# https://pypi.python.org/pypi?%3Aaction=list_classifiers + +classifiers = """ +Development Status :: 3 - Alpha +Environment :: Console +Intended Audience :: Developers +Intended Audience :: End Users/Desktop +License :: OSI Approved :: MIT License +Natural Language :: English +Operating System :: POSIX :: Linux +Programming Language :: Python +Programming Language :: Python :: 2 +Programming Language :: Python :: 2.6 +Programming Language :: Python :: 2.7 +Programming Language :: Python :: Implementation :: CPython +Topic :: Internet :: WWW/HTTP +Topic :: Utilities +""" + +not_yet_tested = """ +Programming Language :: Python :: 3 +Programming Language :: Python :: 3.3 +Programming Language :: Python :: 3.4 +Operating System :: MacOS :: MacOS X +Operating System :: Unix +""" + +setup(name='tespeed', + version='0.1.0-alpha', + packages=find_packages(include=['tespeed']), + description='TeSpeed, CLI SpeedTest.net', + long_description=open('README.rst', 'r', encoding='utf-8').read(), + author='Janis Jansons & David Fischer', + author_email='janis.jansons@janhouse.lv', + url='https://github.com/davidfischer-ch/tespeed', + license='MIT', + classifiers=filter(None, classifiers.split('\n')), + keywords=['benchmark', 'download', 'upload', 'network', 'speed', 'internet', 'speedtest.net'], + dependency_links=[str(requirement.url) for requirement in parse_requirements('requirements.txt')], + install_requires=[str(requirement.req) for requirement in parse_requirements('requirements.txt')], + entry_points={ + 'console_scripts': [ + 'tespeed=tespeed.bin:tespeed' + ] + }, + use_2to3=sys.version_info[0] > 2) diff --git a/tespeed/__init__.py b/tespeed/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tespeed/bin.py b/tespeed/bin.py new file mode 100644 index 0000000..004552d --- /dev/null +++ b/tespeed/bin.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- + +# Copyright: +# 2012-2013 Janis Jansons (janis.jansons@janhouse.lv) +# 2014 David Fischer (david.fischer.ch@gmail.com) + +from __future__ import absolute_import, division, print_function, unicode_literals + +import argparse + +from .core import TeSpeed +from .utils import Log, set_proxy + +__all__ = ('tespeed', ) + + +def tespeed(): + + parser = argparse.ArgumentParser(description='TeSpeed, CLI SpeedTest.net') + + parser.add_argument('server', nargs='?', type=str, default='', help='Use the specified server for testing (skip ' + 'checking for location and closest server).') + parser.add_argument('-ls', '--list-servers', dest='list_servers', nargs='?', default=0, const=10, + help='List the servers sorted by distance, nearest first. Optionally specify number of servers ' + 'to show.') + parser.add_argument('-w', '--csv', dest='store', action='store_true', help='Print CSV formated output to STDOUT.') + parser.add_argument('-s', '--suppress', dest='suppress', action='store_true', + help='Suppress debugging (STDERR) output.') + parser.add_argument('-mib', '--mebibit', dest='unit', action='store_true', help='Show results in mebibits.') + parser.add_argument('-n', '--server-count', dest='server_count', nargs='?', default=1, const=1, + help='Specify how many different servers should be used in parallel. (Default: 1) ' + '(Increase it for >100Mbit testing.)') + + parser.add_argument('-p', '--proxy', dest='use_proxy', type=int, nargs='?', const=4, + help='Specify 4 or 5 to use SOCKS4 or SOCKS5 proxy.') + parser.add_argument('-ph', '--proxy-host', dest='proxy_host', type=str, nargs='?', default='127.0.0.1', + help='Specify socks proxy host. (Default: 127.0.0.1)') + parser.add_argument('-pp', '--proxy-port', dest='proxy_port', type=int, nargs='?', default=9050, + help='Specify socks proxy port. (Default: 9050)') + + parser.add_argument('-cs', '--chunk-size', dest='chunk_size', nargs='?', type=int, default=10240, + help='Specify chunk size after which tespeed calculates speed. Increase this number 4 or 5 ' + 'times if you use weak hardware like RaspberryPi. (Default: 10240)') + + # parser.add_argument('-i', '--interface', dest='interface', nargs='?', + # help='If specified, measures speed from data for the whole network interface.') + + args = parser.parse_args() + + if args.use_proxy: + set_proxy(version=args.use_proxy, host=args.proxy_host, port=args.proxy_port) + + if args.list_servers: + args.store = True + + log = Log(suppress=args.suppress, store=args.store) + if not args.list_servers and args.server == '' and not args.store: + log.debug('Getting ready. Use parameter -h or --help to see available features.\n') + else: + log.debug('Getting ready\n') + try: + tespeed = TeSpeed(server=args.list_servers and 'list-servers' or args.server, num_servers=args.server_count, + num_top=args.list_servers, unit=args.unit, chunk_size=args.chunk_size, log=log) + tespeed.run_tests() + except (KeyboardInterrupt, SystemExit): + log.debug('\nSpeed test stopped.\n') diff --git a/tespeed.py b/tespeed/core.py similarity index 82% rename from tespeed.py rename to tespeed/core.py index 73e5bc2..511d104 100755 --- a/tespeed.py +++ b/tespeed/core.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python2 # -*- coding: utf-8 -*- # Copyright: @@ -7,14 +6,13 @@ from __future__ import absolute_import, division, print_function, unicode_literals -import argparse, urllib, urllib2, time +import urllib, urllib2, time from lxml import etree from multiprocessing import Process, Pipe, Manager -from utils import ( - socks, - CallbackStringIO, Log, closest, decompress_response, num_download_threads_for, num_upload_threads_for, set_proxy -) +from .utils import CallbackStringIO, Log, closest, decompress_response, num_download_threads_for, num_upload_threads_for + +__all__ = ('TeSpeed', ) class TeSpeed(object): @@ -352,45 +350,3 @@ def run_tests(self): download_speed = self.test_download() upload_speed = self.test_upload() self.log.result('%0.2f,%0.2f,"%s","%s"\n' % (download_speed, upload_speed, self.units, self.servers)) - - -def main(args): - - if args.use_proxy: - set_proxy(typ=socks.PROXY_TYPE_SOCKS5 if args.use_proxy == 5 else socks.PROXY_TYPE_SOCKS4, host=args.proxy_host, - port=args.proxy_port) - - if args.list_servers: - args.store = True - - log = Log(suppress=args.suppress, store=args.store) - if not args.list_servers and args.server == '' and not args.store: - log.debug('Getting ready. Use parameter -h or --help to see available features.\n') - else: - log.debug('Getting ready\n') - try: - tespeed = TeSpeed(server=args.list_servers and 'list-servers' or args.server, num_servers=args.server_count, - num_top=args.list_servers, unit=args.unit, chunk_size=args.chunk_size, log=log) - tespeed.run_tests() - except (KeyboardInterrupt, SystemExit): - log.debug('\nSpeed test stopped.\n') - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='TeSpeed, CLI SpeedTest.net') - - parser.add_argument('server', nargs='?', type=str, default='', help='Use the specified server for testing (skip checking for location and closest server).') - parser.add_argument('-ls', '--list-servers', dest='list_servers', nargs='?', default=0, const=10, help='List the servers sorted by distance, nearest first. Optionally specify number of servers to show.') - parser.add_argument('-w', '--csv', dest='store', action='store_true', help='Print CSV formated output to STDOUT.') - parser.add_argument('-s', '--suppress', dest='suppress', action='store_true', help='Suppress debugging (STDERR) output.') - parser.add_argument('-mib', '--mebibit', dest='unit', action='store_true', help='Show results in mebibits.') - parser.add_argument('-n', '--server-count', dest='server_count', nargs='?', default=1, const=1, help='Specify how many different servers should be used in parallel. (Default: 1) (Increase it for >100Mbit testing.)') - - parser.add_argument('-p', '--proxy', dest='use_proxy', type=int, nargs='?', const=4, help='Specify 4 or 5 to use SOCKS4 or SOCKS5 proxy.') - parser.add_argument('-ph', '--proxy-host', dest='proxy_host', type=str, nargs='?', default='127.0.0.1', help='Specify socks proxy host. (Default: 127.0.0.1)') - parser.add_argument('-pp', '--proxy-port', dest='proxy_port', type=int, nargs='?', default=9050, help='Specify socks proxy port. (Default: 9050)') - - parser.add_argument('-cs', '--chunk-size', dest='chunk_size', nargs='?', type=int, default=10240, help='Specify chunk size after which tespeed calculates speed. Increase this number 4 or 5 times if you use weak hardware like RaspberryPi. (Default: 10240)') - - #parser.add_argument('-i', '--interface', dest='interface', nargs='?', help='If specified, measures speed from data for the whole network interface.') - - main(parser.parse_args()) diff --git a/utils.py b/tespeed/utils.py similarity index 94% rename from utils.py rename to tespeed/utils.py index 05a73b9..a4594c6 100644 --- a/utils.py +++ b/tespeed/utils.py @@ -6,13 +6,11 @@ from __future__ import absolute_import, division, print_function, unicode_literals -import gzip, socket, sys +import gzip, socket, socks, sys from math import radians, cos, sin, asin, sqrt -from SocksiPy import socks from StringIO import StringIO __all__ = ( - 'socks', 'StringIO', # also include things from other modules 'CallbackStringIO', 'Log', 'closest', 'decompress_response', 'distance', 'num_download_threads_for', 'num_upload_threads_for', 'set_proxy' ) @@ -156,6 +154,6 @@ def num_upload_threads_for(num_uploads): # Thanks to Ryan Sears for http://bit.ly/17HhSli -def set_proxy(typ=socks.PROXY_TYPE_SOCKS4, host='127.0.0.1', port=9050): - socks.setdefaultproxy(typ, host, port) +def set_proxy(version, host='127.0.0.1', port=9050): + socks.setdefaultproxy(getattr(socks, 'PROXY_TYPE_SOCKS' + version), host, port) socket.socket = socks.socksocket From a9c3bfb0657c312c8d33b7d75b6903c2d1761e16 Mon Sep 17 00:00:00 2001 From: David Fischer Date: Sat, 24 May 2014 15:26:08 +0200 Subject: [PATCH 23/24] Update README --- README.rst | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/README.rst b/README.rst index 35035b2..7fe5ede 100644 --- a/README.rst +++ b/README.rst @@ -15,8 +15,8 @@ this one a bit better working in time. Let's call this version 0.1.0-alpha -Of course, this script could not work like this without the best speed testing site out there - -http://www.speedtest.net/ +Of course, this script could not work like this without the best speed testing site out there - http://www.speedtest.net/ + Support them in any way you can (going to their website and clicking on ads could probably make them a bit happier). :) ------------ @@ -46,9 +46,10 @@ Usage :: - usage: tespeed.py [-h] [-ls [LISTSERVERS]] [-w] [-s] [-mib] [-n [SERVERCOUNT]] - [-p [USE_PROXY]] [-ph [PROXY_HOST]] [-pp [PROXY_PORT]] - [server] + usage: tespeed [-h] [-ls [LIST_SERVERS]] [-w] [-s] [-mib] [-n [SERVER_COUNT]] + [-p [USE_PROXY]] [-ph [PROXY_HOST]] [-pp [PROXY_PORT]] + [-cs [CHUNK_SIZE]] + [server] TeSpeed, CLI SpeedTest.net @@ -58,24 +59,24 @@ Usage optional arguments: -h, --help show this help message and exit - -ls [LISTSERVERS], --list-servers [LISTSERVERS] + -ls [LIST_SERVERS], --list-servers [LIST_SERVERS] List the servers sorted by distance, nearest first. Optionally specify number of servers to show. -w, --csv Print CSV formated output to STDOUT. -s, --suppress Suppress debugging (STDERR) output. -mib, --mebibit Show results in mebibits. - -n [SERVERCOUNT], --server-count [SERVERCOUNT] + -n [SERVER_COUNT], --server-count [SERVER_COUNT] Specify how many different servers should be used in - paralel. (Defaults to 1.) (Increase it for >100Mbit + parallel. (Default: 1) (Increase it for >100Mbit testing.) -p [USE_PROXY], --proxy [USE_PROXY] Specify 4 or 5 to use SOCKS4 or SOCKS5 proxy. -ph [PROXY_HOST], --proxy-host [PROXY_HOST] - Specify socks proxy host (defaults to 127.0.0.1). + Specify socks proxy host. (Default: 127.0.0.1) -pp [PROXY_PORT], --proxy-port [PROXY_PORT] - Specify socks proxy port (defaults to 9050). - -cs [CHUNKSIZE], --chunk-size [CHUNKSIZE] - Specify chunk size after wich tespeed calculates + Specify socks proxy port. (Default: 9050) + -cs [CHUNK_SIZE], --chunk-size [CHUNK_SIZE] + Specify chunk size after which tespeed calculates speed. Increase this number 4 or 5 times if you use weak hardware like RaspberryPi. (Default: 10240) From f77827423a44c45733110ea598c614454559a8b0 Mon Sep 17 00:00:00 2001 From: David Fischer Date: Sun, 25 May 2014 20:32:01 +0200 Subject: [PATCH 24/24] Use enumerate, load config & server list on access --- tespeed/core.py | 139 ++++++++++++++++++++++-------------------------- 1 file changed, 65 insertions(+), 74 deletions(-) diff --git a/tespeed/core.py b/tespeed/core.py index 511d104..a3a4d45 100755 --- a/tespeed/core.py +++ b/tespeed/core.py @@ -78,6 +78,9 @@ def __init__(self, server=None, num_servers=3, num_top=0, unit=False, chunk_size self.latency_count = 10 self.post_data = '' + self._config = None + self._server_list = None + def convert_size(self, value): return value / 1024 ** 2 * (1 if self.unit == 1 else 1.048576 * 8) @@ -173,85 +176,79 @@ def async_request(self, url, num, upload=0): connection['connection'].start() connections.append(connection) - for c in xrange(num): - connections[c]['size'], connections[c]['start'], connections[c]['end'] = connections[c]['parent'].recv() - connections[c]['connection'].join() + for i in xrange(num): + connections[i]['size'], connections[i]['start'], connections[i]['end'] = connections[i]['parent'].recv() + connections[i]['connection'].join() end_time = time.time() self.log.debug(Log.BLANK_LINE) sizes = 0 - for c in xrange(num): - if connections[c]['end'] is not False: - sizes += connections[c]['size'] + for i in xrange(num): + if connections[i]['end'] is not False: + sizes += connections[i]['size'] # Using more precise times for downloads if upload == 0: - if c == 0: - start_time = connections[c]['start'] - end_time = connections[c]['end'] + if i == 0: + start_time = connections[i]['start'] + end_time = connections[i]['end'] else: - if connections[c]['start'] < start_time: - start_time = connections[c]['start'] - if connections[c]['end'] > end_time: - end_time = connections[c]['end'] + if connections[i]['start'] < start_time: + start_time = connections[i]['start'] + if connections[i]['end'] > end_time: + end_time = connections[i]['end'] return [sizes, end_time - start_time] - def load_config(self): - """Load the configuration file.""" - self.log.debug('Loading speedtest configuration...\n') - uri = 'http://speedtest.net/speedtest-config.php?x=' + str(time.time()) - request = self.get_request(uri) - response = urllib2.urlopen(request) - - # Load etree from XML data - client = etree.fromstring(decompress_response(response)).find('client') - self.config = { - 'ip': client.attrib['ip'], - 'isp': client.attrib['isp'], - 'lat': float(client.attrib['lat']), - 'lon': float(client.attrib['lon']) - } - self.log.debug('IP: {ip}; Lat: {lat}; Lon: {lon}; ISP: {isp}\n'.format(**self.config)) - - def find_best_server(self): - self.log.debug('Looking for closest and best server...\n') - best_servers = self.test_latency(closest([self.config['lat'], self.config['lon']], self.server_list, + def config(self, force_reload=False): + """Return the configuration loaded from the speedtest.net server API.""" + if not self._config or force_reload: + self.log.debug('Loading the configuration...\n') + uri = 'http://speedtest.net/speedtest-config.php?x=' + str(time.time()) + response = urllib2.urlopen(self.get_request(uri)) + client = etree.fromstring(decompress_response(response)).find('client') + self._config = { + 'ip': client.attrib['ip'], + 'isp': client.attrib['isp'], + 'lat': float(client.attrib['lat']), + 'lon': float(client.attrib['lon']) + } + self.log.debug('IP: {ip}; Lat: {lat}; Lon: {lon}; ISP: {isp}\n'.format(**self._config)) + return self._config + + def server_list(self, force_reload=False): + """Return the list of servers loaded from the speedtest.net server API.""" + if not self._server_list or force_reload: + self.log.debug('Loading the list of servers...\n') + uri = 'http://speedtest.net/speedtest-servers.php?x=' + str(time.time()) + response = urllib2.urlopen(self.get_request(uri)) + servers_xml = etree.fromstring(decompress_response(response)) + self._server_list = [ + { + 'lat': float(server.attrib['lat']), + 'lon': float(server.attrib['lon']), + 'url': server.attrib['url'].rsplit('/', 1)[0] + '/', + #'url2': server.attrib['url2'].rsplit('/', 1)[0] + '/', + 'name': server.attrib['name'], + 'country': server.attrib['country'], + 'sponsor': server.attrib['sponsor'], + 'id': server.attrib['id'], + } for server in servers_xml.find('servers').findall('server') + ] + return self._server_list + + def find_best_servers(self): + self.log.debug('Looking for closest and best servers...\n') + best_servers = self.test_latency(closest([self.config()['lat'], self.config()['lon']], self.server_list(), self.best_servers)) self.servers.extend(server['url'] for server in best_servers) - def load_servers(self): - """Load server list.""" - self.log.debug('Loading server list...\n') - uri = 'http://speedtest.net/speedtest-servers.php?x=' + str(time.time()) - request = self.get_request(uri) - response = urllib2.urlopen(request) - - # Load etree from XML data - servers_xml = etree.fromstring(decompress_response(response)) - self.server_list = [ - { - 'lat': float(server.attrib['lat']), - 'lon': float(server.attrib['lon']), - 'url': server.attrib['url'].rsplit('/', 1)[0] + '/', - #'url2': server.attrib['url2'].rsplit('/', 1)[0] + '/', - 'name': server.attrib['name'], - 'country': server.attrib['country'], - 'sponsor': server.attrib['sponsor'], - 'id': server.attrib['id'], - } for server in servers_xml.find('servers').findall('server') - ] - def list_servers(self, num=0): - - all_sorted = closest([self.config['lat'], self.config['lon']], self.server_list, num) - - for i in xrange(len(all_sorted)): - self.log.result('%s. %s (%s, %s, %s) [%0.2f km]\n' % - (i + 1, all_sorted[i]['url'], all_sorted[i]['sponsor'], all_sorted[i]['name'], - all_sorted[i]['country'], all_sorted[i]['distance'])) + for i, server in enumerate(closest([self.config()['lat'], self.config()['lon']], self.server_list(), num), 1): + self.log.result('%s. %s (%s, %s, %s) [%0.2f km]\n' % (i, server['url'], server['sponsor'], server['name'], + server['country'], server['distance'])) def test_latency(self, servers): """Find servers with lowest latency.""" @@ -261,9 +258,8 @@ def test_latency(self, servers): latency = self.test_single_latency(server['url'] + 'latency.txt?x=' + str(time.time())) * 1000 if not latency: continue - self.log.debug('%0.0f ms latency for %s (%s, %s, %s) [%0.2f km]\n' % - (latency, server['url'], server['sponsor'], server['name'], server['country'], - server['distance'])) + self.log.debug('%0.0f ms latency for %s (%s, %s, %s) [%0.2f km]\n' % (latency, server['url'], + server['sponsor'], server['name'], server['country'], server['distance'])) server['latency'] = latency # Pick specified amount of servers with best latency for testing if int(len(po)) < int(self.num_servers): @@ -297,10 +293,9 @@ def test_single_latency(self, destination_address): def test_download(self): """Test download speed.""" max_speed = -1 - for i in xrange(len(self.DOWNLOAD_LIST)): - url = 'random' + self.DOWNLOAD_LIST[i] + '.jpg?x=' + str(time.time()) + '&y=3' - - sizes, took = self.async_request(url, num_download_threads_for(i)) + for counter, download in enumerate(self.DOWNLOAD_LIST): + url = 'random' + download + '.jpg?x=' + str(time.time()) + '&y=3' + sizes, took = self.async_request(url, num_download_threads_for(counter)) if sizes == 0: continue @@ -339,14 +334,10 @@ def test_upload(self): def run_tests(self): if self.server == 'list-servers': - self.load_config() - self.load_servers() self.list_servers(self.num_top) else: if not self.server: - self.load_config() - self.load_servers() - self.find_best_server() + self.find_best_servers() download_speed = self.test_download() upload_speed = self.test_upload() self.log.result('%0.2f,%0.2f,"%s","%s"\n' % (download_speed, upload_speed, self.units, self.servers))