diff --git a/.gitignore b/.gitignore index 2bc1b44..ade8000 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ -old/ +build/ +dist/ +*.egg* *~ ~ *.pyc diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index 091804b..0000000 --- a/.gitmodules +++ /dev/null @@ -1,3 +0,0 @@ -[submodule "SocksiPy"] - path = SocksiPy - url = git://github.com/Janhouse/SocksiPy.git diff --git a/README b/README deleted file mode 100644 index a1ac4ab..0000000 --- a/README +++ /dev/null @@ -1,123 +0,0 @@ -Tespeed (terminal speedtest) - Copyright 2012 Janis Jansons (janis.jansons@janhouse.lv) - - - This is a new Tespeed version written in Python (for the purpose of learning it). - - The old one was written in PHP years ago and wasn't really made for general - public (was fine tuned and possibly working only on my server). - - Even though the old version didn't work on most boxes, it somehow got - almost 17 000 downloads on Sourceforge. I guess some people could use this - (those who hate Flash, JavaScript, has GUI-less servers, etc.) so I'll - try to make this one a bit better working in time. - - Let's call this version 1.0-alpha - - - Of course, this script could not work like this without the best speed - testing site out there - http://www.speedtest.net/ - Support them in any way you can (going to their website and clicking on - ads could probably make them a bit happier). :) - - -Requirements: - - This script requires recent Python (preferably 2.7 or newer) and Python2 - modules lxml and argparse. - Install python-lxml and python-argparse (Debian) or python2-lxml (Archlinux). - -Installation: - - If you have Debian, you might have to change the python executable in tuper.py - or create a symlink for your existing python2.x executable by doing: - - sudo ln -s /usr/bin/python2.7 /usr/bin/python2 - - If you have python2.6 then replace python2.7 with python2.6. - - - - When doing the checkout, remember to pull submodules. - - If you have a decent git version (1.6.5 and up), get everything by doing: - - git clone --recursive git://github.com/Janhouse/tespeed.git - - Otherwise do: - - git clone git://github.com/Janhouse/tespeed.git - cd tespeed - git submodule init - git submodule update - - -Usage: - - usage: tespeed.py [-h] [-ls [LISTSERVERS]] [-w] [-s] [-mib] [-n [SERVERCOUNT]] - [-p [USE_PROXY]] [-ph [PROXY_HOST]] [-pp [PROXY_PORT]] - [server] - - TeSpeed, CLI SpeedTest.net - - positional arguments: - server Use the specified server for testing (skip checking - for location and closest server). - - optional arguments: - -h, --help show this help message and exit - -ls [LISTSERVERS], --list-servers [LISTSERVERS] - List the servers sorted by distance, nearest first. - Optionally specify number of servers to show. - -w, --csv Print CSV formated output to STDOUT. - -s, --suppress Suppress debugging (STDERR) output. - -mib, --mebibit Show results in mebibits. - -n [SERVERCOUNT], --server-count [SERVERCOUNT] - Specify how many different servers should be used in - paralel. (Defaults to 1.) (Increase it for >100Mbit - testing.) - -p [USE_PROXY], --proxy [USE_PROXY] - Specify 4 or 5 to use SOCKS4 or SOCKS5 proxy. - -ph [PROXY_HOST], --proxy-host [PROXY_HOST] - Specify socks proxy host (defaults to 127.0.0.1). - -pp [PROXY_PORT], --proxy-port [PROXY_PORT] - Specify socks proxy port (defaults to 9050). - -cs [CHUNKSIZE], --chunk-size [CHUNKSIZE] - Specify chunk size after wich tespeed calculates - speed. Increase this number 4 or 5 times if you use - weak hardware like RaspberryPi. (Default: 10240) - - -What the script does: - - * Loads config from speedtest.net (http://speedtest.net/speedtest-config.php). - - * Gets server list (http://speedtest.net/speedtest-servers.php). - - * Picks 5 closests servers using the coordinates provides by speedtest.net - config and serverlist. - - * Checks latency for those servers and picks one with the lowest. - - * Does download speed test and returns results. - - * Does upload speed test and returns results. - - * Optionally can return CSV formated results. - - * Can measure through SOCKS proxy. - -TODO (ideas): - - * Add more error checking. - * Make it less messy. - * Send found results to speedtest.net API (needs some hash?) and get the link - to the generated image. - * Store the measurement data and draw graphs. - * Measure the speed for the whole network interface (similar like it was in the - old version of Tespeed). - * Start upload timer only after 1st byte is read. - * Figure out the ammount of data that was transfered only when all threads were - actively sendong/receiving data at the same time. (Should provide more precise - test results.) - - diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..7fe5ede --- /dev/null +++ b/README.rst @@ -0,0 +1,105 @@ +============================ +Tespeed (terminal speedtest) +============================ + +:copyright: Copyright 2012 Janis Jansons (janis.jansons@janhouse.lv) + +This is a new TeSpeed version written in Python (for the purpose of learning it). + +The old one was written in PHP years ago and wasn't really made for general public (was fine tuned and possibly working +only on my server). + +Even though the old version didn't work on most boxes, it somehow got almost 17'000 downloads on Sourceforge. +I guess some people could use this (those who hate Flash, JavaScript, has GUI-less servers, etc.) so I'll try to make +this one a bit better working in time. + +Let's call this version 0.1.0-alpha + +Of course, this script could not work like this without the best speed testing site out there - http://www.speedtest.net/ + +Support them in any way you can (going to their website and clicking on ads could probably make them a bit happier). :) + +------------ +Installation +------------ + +When doing the checkout, remember to pull submodules. + +If you have a decent git version (1.6.5 and up), get everything by doing:: + + git clone --recursive git://github.com/Janhouse/tespeed.git + +Otherwise do:: + + git clone git://github.com/Janhouse/tespeed.git + cd tespeed + git submodule init + git submodule update + +Then install it thanks to the project's setup script:: + + sudo ./setup.py install + +----- +Usage +----- + +:: + + usage: tespeed [-h] [-ls [LIST_SERVERS]] [-w] [-s] [-mib] [-n [SERVER_COUNT]] + [-p [USE_PROXY]] [-ph [PROXY_HOST]] [-pp [PROXY_PORT]] + [-cs [CHUNK_SIZE]] + [server] + + TeSpeed, CLI SpeedTest.net + + positional arguments: + server Use the specified server for testing (skip checking + for location and closest server). + + optional arguments: + -h, --help show this help message and exit + -ls [LIST_SERVERS], --list-servers [LIST_SERVERS] + List the servers sorted by distance, nearest first. + Optionally specify number of servers to show. + -w, --csv Print CSV formated output to STDOUT. + -s, --suppress Suppress debugging (STDERR) output. + -mib, --mebibit Show results in mebibits. + -n [SERVER_COUNT], --server-count [SERVER_COUNT] + Specify how many different servers should be used in + parallel. (Default: 1) (Increase it for >100Mbit + testing.) + -p [USE_PROXY], --proxy [USE_PROXY] + Specify 4 or 5 to use SOCKS4 or SOCKS5 proxy. + -ph [PROXY_HOST], --proxy-host [PROXY_HOST] + Specify socks proxy host. (Default: 127.0.0.1) + -pp [PROXY_PORT], --proxy-port [PROXY_PORT] + Specify socks proxy port. (Default: 9050) + -cs [CHUNK_SIZE], --chunk-size [CHUNK_SIZE] + Specify chunk size after which tespeed calculates + speed. Increase this number 4 or 5 times if you use + weak hardware like RaspberryPi. (Default: 10240) + +What the script does: + +* Loads configuration from speedtest.net (http://speedtest.net/speedtest-config.php). +* Gets server list (http://speedtest.net/speedtest-servers.php). +* Picks 5 closest servers using the coordinates provides by speedtest.net config and serverlist. +* Checks latency for those servers and picks one with the lowest. +* Does download speed test and returns results. +* Does upload speed test and returns results. +* Optionally can return CSV formated results. +* Can measure through SOCKS proxy. + +TODO (ideas): + +* Add more error checking. +* Make it less messy. +* Send found results to speedtest.net API (needs some hash?) and get the link to the generated image. +* Store the measurement data and draw graphs. +* Measure the speed for the whole network interface (similar like it was in the old version of Tespeed). +* Start upload timer only after 1st byte is read. +* Figure out the amount of data that was transfered only when all threads were actively sending/receiving data at the + same time. (Should provide more precise test results.) + + diff --git a/SocksiPy b/SocksiPy deleted file mode 160000 index 842d496..0000000 --- a/SocksiPy +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 842d4962cbce16ce4b232d1b7402d0375f9a0c1b diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..0564b6c --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +argparse +lxml +-e git://github.com/Anorov/PySocks.git#egg=PySocks diff --git a/serverlist.txt b/serverlist.xml similarity index 100% rename from serverlist.txt rename to serverlist.xml diff --git a/setup.py b/setup.py new file mode 100755 index 0000000..c3c2170 --- /dev/null +++ b/setup.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +# Copyright: +# 2012-2013 Janis Jansons (janis.jansons@janhouse.lv) +# 2014 David Fischer (david.fischer.ch@gmail.com) + +from __future__ import absolute_import, division, print_function, unicode_literals + +import sys +from codecs import open +from pip.req import parse_requirements +from setuptools import setup, find_packages + +# https://pypi.python.org/pypi?%3Aaction=list_classifiers + +classifiers = """ +Development Status :: 3 - Alpha +Environment :: Console +Intended Audience :: Developers +Intended Audience :: End Users/Desktop +License :: OSI Approved :: MIT License +Natural Language :: English +Operating System :: POSIX :: Linux +Programming Language :: Python +Programming Language :: Python :: 2 +Programming Language :: Python :: 2.6 +Programming Language :: Python :: 2.7 +Programming Language :: Python :: Implementation :: CPython +Topic :: Internet :: WWW/HTTP +Topic :: Utilities +""" + +not_yet_tested = """ +Programming Language :: Python :: 3 +Programming Language :: Python :: 3.3 +Programming Language :: Python :: 3.4 +Operating System :: MacOS :: MacOS X +Operating System :: Unix +""" + +setup(name='tespeed', + version='0.1.0-alpha', + packages=find_packages(include=['tespeed']), + description='TeSpeed, CLI SpeedTest.net', + long_description=open('README.rst', 'r', encoding='utf-8').read(), + author='Janis Jansons & David Fischer', + author_email='janis.jansons@janhouse.lv', + url='https://github.com/davidfischer-ch/tespeed', + license='MIT', + classifiers=filter(None, classifiers.split('\n')), + keywords=['benchmark', 'download', 'upload', 'network', 'speed', 'internet', 'speedtest.net'], + dependency_links=[str(requirement.url) for requirement in parse_requirements('requirements.txt')], + install_requires=[str(requirement.req) for requirement in parse_requirements('requirements.txt')], + entry_points={ + 'console_scripts': [ + 'tespeed=tespeed.bin:tespeed' + ] + }, + use_2to3=sys.version_info[0] > 2) diff --git a/tespeed.py b/tespeed.py deleted file mode 100755 index bc3d33a..0000000 --- a/tespeed.py +++ /dev/null @@ -1,661 +0,0 @@ -#!/usr/bin/env python2 -# -# Copyright 2012-2013 Janis Jansons (janis.jansons@janhouse.lv) -# - -import argparse - -from SocksiPy import socks -import socket - -# Magic! -def getaddrinfo(*args): - return [(socket.AF_INET, socket.SOCK_STREAM, 6, '', (args[0], args[1]))] -socket.getaddrinfo = getaddrinfo - -import urllib -import urllib2 -import gzip -import sys -from multiprocessing import Process, Pipe, Manager -from lxml import etree -import time -from math import radians, cos, sin, asin, sqrt - -from StringIO import StringIO - -# Using StringIO with callback to measure upload progress -class CallbackStringIO(StringIO): - def __init__(self, num, th, d, buf = ''): - # Force self.buf to be a string or unicode - if not isinstance(buf, basestring): - buf = str(buf) - self.buf = buf - self.len = len(buf) - self.buflist = [] - self.pos = 0 - self.closed = False - self.softspace = 0 - self.th=th - self.num=num - self.d=d - self.total=self.len*self.th - - def read(self, n=10240): - next = StringIO.read(self, n) - #if 'done' in self.d: - # return - - self.d[self.num]=self.pos - down=0 - for i in range(self.th): - down=down+self.d.get(i, 0) - if self.num==0: - percent = float(down) / (self.total) - percent = round(percent*100, 2) - print_debug("Uploaded %d of %d bytes (%0.2f%%) in %d threads\r" % - (down, self.total, percent, self.th)) - - #if down >= self.total: - # print_debug('\n') - # self.d['done']=1 - - return next - - def __len__(self): - return self.len - - -class TeSpeed: - - def __init__(self, server = "", numTop = 0, servercount = 3, store = False, suppress = False, unit = False, chunksize=10240): - - self.headers = { - 'Accept' : 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', - 'User-Agent' : 'Mozilla/5.0 (X11; Linux x86_64; rv:11.0) Gecko/20100101 Firefox/11.0', - 'Accept-Language' : 'en-us,en;q=0.5', - 'Connection' : 'keep-alive', - 'Accept-Encoding' : 'gzip, deflate', - #'Referer' : 'http://c.speedtest.net/flash/speedtest.swf?v=301256', - } - - self.num_servers=servercount; - self.servers=[] - if server != "": - self.servers=[server] - - self.server=server - self.down_speed=-1 - self.up_speed=-1 - self.latencycount=10 - self.bestServers=5 - - self.units="Mbit" - self.unit=0 - - self.chunksize=chunksize - - if unit: - self.units="MiB" - self.unit=1 - - self.store=store - self.suppress=suppress - if store: - print_debug("Printing CSV formated results to STDOUT.\n") - self.numTop=int(numTop) - #~ self.downList=['350x350', '500x500', '750x750', '1000x1000', - #~ '1500x1500', '2000x2000', '2000x2000', '2500x2500', '3000x3000', - #~ '3500x3500', '4000x4000', '4000x4000', '4000x4000', '4000x4000'] - #~ self.upSizes=[1024*256, 1024*256, 1024*512, 1024*512, - #~ 1024*1024, 1024*1024, 1024*1024*2, 1024*1024*2, - #~ 1024*1024*2, 1024*1024*2] - - self.downList=[ -'350x350', '350x350', '500x500', '500x500', '750x750', '750x750', '1000x1000', '1500x1500', '2000x2000', '2500x2500', - -'3000x3000','3500x3500','4000x4000','1000x1000','1000x1000','1000x1000','1000x1000','1000x1000','1000x1000','1000x1000', -'1000x1000','1000x1000','1000x1000','1000x1000','1000x1000','1000x1000','1000x1000','1000x1000','1000x1000','1000x1000', - -'2000x2000','2000x2000','2000x2000','2000x2000','2000x2000','2000x2000','2000x2000','2000x2000','2000x2000','2000x2000', -'2000x2000','2000x2000','2000x2000','2000x2000','2000x2000','2000x2000','2000x2000','2000x2000','2000x2000','2000x2000', - -'4000x4000', '4000x4000', '4000x4000', '4000x4000', '4000x4000' -] - -#'350x350', '500x500', '750x750', '1000x1000', -# '1500x1500', '2000x2000', '2000x2000', '2500x2500', '3000x3000', -# '3500x3500', '4000x4000', '4000x4000', '4000x4000', '4000x4000' -#] - self.upSizes=[ - -1024*256, 1024*256, 1024*512, 1024*512, 1024*1024, 1024*1024, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*512, -1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, - -1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, -1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, - -1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, -1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, - -1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, -1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, -# 1024*1024, 1024*1024, 1024*1024*2, 1024*1024*2, -# 1024*1024*2, 1024*1024*2] -] - - self.postData="" - self.TestSpeed() - - - def Distance(self, one, two): - #Calculate the great circle distance between two points - #on the earth specified in decimal degrees (haversine formula) - #(http://stackoverflow.com/posts/4913653/revisions) - # convert decimal degrees to radians - - lon1, lat1, lon2, lat2 = map(radians, [one[0], one[1], two[0], two[1]]) - # haversine formula - dlon = lon2 - lon1 - dlat = lat2 - lat1 - a = sin(dlat/2)**2 + cos(lat1) * cos(lat2) * sin(dlon/2)**2 - c = 2 * asin(sqrt(a)) - km = 6367 * c - return km - - - def Closest(self, center, points, num=5): - # Returns object that is closest to center - closest={} - for p in range(len(points)): - now = self.Distance(center, [points[p]['lat'], points[p]['lon']]) - points[p]['distance']=now - while True: - if now in closest: - now=now+00.1 - else: - break - closest[now]=points[p] - n=0 - ret=[] - for key in sorted(closest): - ret.append(closest[key]) - n+=1 - if n >= num and num!=0: - break - return ret - - - def TestLatency(self, servers): - # Finding servers with lowest latency - print_debug("Testing latency...\n") - po = [] - for server in servers: - now=self.TestSingleLatency(server['url']+"latency.txt?x=" + str( time.time() ))*1000 - now=now/2 # Evil hack or just pure stupidity? Nobody knows... - if now == -1 or now == 0: - continue - print_debug("%0.0f ms latency for %s (%s, %s, %s) [%0.2f km]\n" % - (now, server['url'], server['sponsor'], server['name'], server['country'], server['distance'])) - - server['latency']=now - - # Pick specified ammount of servers with best latency for testing - if int(len(po)) < int(self.num_servers): - po.append(server) - else: - largest = -1 - - for x in range(len(po)): - if largest < 0: - if now < po[x]['latency']: - largest = x - elif po[largest]['latency'] < po[x]['latency']: - largest = x - #if cur['latency'] - - if largest >= 0: - po[largest]=server - - return po - - - def TestSingleLatency(self, dest_addr): - # Checking latency for single server - # Does that by loading latency.txt (empty page) - request=self.GetRequest(dest_addr) - - averagetime=0 - total=0 - for i in range(self.latencycount): - error=0 - startTime = time.time() - try: - response = urllib2.urlopen(request, timeout = 5) - except urllib2.URLError, e: - error=1 - - if error==0: - averagetime = averagetime + (time.time() - startTime) - total=total+1 - - if total==0: - return False - - return averagetime/total - - - def GetRequest(self, uri): - # Generates a GET request to be used with urlopen - req = urllib2.Request(uri, headers = self.headers) - return req - - - def PostRequest(self, uri, stream): - # Generate a POST request to be used with urlopen - req = urllib2.Request(uri, stream, headers = self.headers) - return req - - - def ChunkReport(self, bytes_so_far, chunk_size, total_size, num, th, d, w): - # Receiving status update from download thread - - if w==1: - return - d[num]=bytes_so_far - down=0 - for i in range(th): - down=down+d.get(i, 0) - - if num==0 or down >= total_size*th: - - percent = float(down) / (total_size*th) - percent = round(percent*100, 2) - - print_debug("Downloaded %d of %d bytes (%0.2f%%) in %d threads\r" % - (down, total_size*th, percent, th)) - - #if down >= total_size*th: - # print_debug('\n') - - - def ChunkRead(self, response, num, th, d, w=0, chunk_size=False, report_hook=None): - #print_debug("Thread num %d %d %d starting to report\n" % (th, num, d)) - - if not chunk_size: - chunk_size=self.chunksize - - if(w==1): - return [0,0,0] - - total_size = response.info().getheader('Content-Length').strip() - total_size = int(total_size) - bytes_so_far = 0 - - start=0 - while 1: - chunk=0 - if start == 0: - #print_debug("Started receiving data\n") - chunk = response.read(1) - start = time.time() - - else: - chunk = response.read(chunk_size) - if not chunk: - break - bytes_so_far += len(chunk) - if report_hook: - report_hook(bytes_so_far, chunk_size, total_size, num, th, d, w) - end = time.time() - - return [ bytes_so_far, start, end ] - - - def AsyncGet(self, conn, uri, num, th, d): - - request=self.GetRequest(uri) - - start=0 - end=0 - size=0 - - try: - response = urllib2.urlopen(request, timeout = 30); - size, start, end=self.ChunkRead(response, num, th, d, report_hook=self.ChunkReport) - #except urllib2.URLError, e: - # print_debug("Failed downloading.\n") - except: - print_debug(' \r') - print_debug("Failed downloading.\n") - conn.send([0, 0, False]) - conn.close() - return - - conn.send([size, start, end]) - conn.close() - - - def AsyncPost(self, conn, uri, num, th, d): - postlen=len(self.postData) - stream = CallbackStringIO(num, th, d, self.postData) - request=self.PostRequest(uri, stream) - - start=0 - end=0 - - try: - response = urllib2.urlopen(request, timeout = 30); - size, start, end=self.ChunkRead(response, num, th, d, 1, report_hook=self.ChunkReport) - #except urllib2.URLError, e: - # print_debug("Failed uploading.\n") - except: - print_debug(' \r') - print_debug("Failed uploading.\n") - conn.send([0, 0, False]) - conn.close() - return - - conn.send([postlen, start, end]) - conn.close() - - - def LoadConfig(self): - # Load the configuration file - print_debug("Loading speedtest configuration...\n") - uri = "http://speedtest.net/speedtest-config.php?x=" + str( time.time() ) - request=self.GetRequest(uri) - response = urllib2.urlopen(request) - - - # Load etree from XML data - config = etree.fromstring(self.DecompressResponse(response)) - - ip=config.find("client").attrib['ip'] - isp=config.find("client").attrib['isp'] - lat=float(config.find("client").attrib['lat']) - lon=float(config.find("client").attrib['lon']) - - print_debug("IP: %s; Lat: %f; Lon: %f; ISP: %s\n" % (ip, lat, lon, isp)) - - return { 'ip': ip, 'lat': lat, 'lon': lon, 'isp': isp } - - - def LoadServers(self): - # Load server list - print_debug("Loading server list...\n") - uri = "http://speedtest.net/speedtest-servers.php?x=" + str( time.time() ) - request=self.GetRequest(uri) - response = urllib2.urlopen(request); - - # Load etree from XML data - servers_xml = etree.fromstring(self.DecompressResponse(response)) - servers=servers_xml.find("servers").findall("server") - server_list=[] - - for server in servers: - server_list.append({ - 'lat': float(server.attrib['lat']), - 'lon': float(server.attrib['lon']), - 'url': server.attrib['url'].rsplit('/', 1)[0] + '/', - #'url2': server.attrib['url2'].rsplit('/', 1)[0] + '/', - 'name': server.attrib['name'], - 'country': server.attrib['country'], - 'sponsor': server.attrib['sponsor'], - 'id': server.attrib['id'], - }) - - return server_list - - - def DecompressResponse(sefl, response): - # Decompress gzipped response - data = StringIO(response.read()) - gzipper = gzip.GzipFile(fileobj=data) - return gzipper.read() - - - def FindBestServer(self): - print_debug("Looking for closest and best server...\n") - best=self.TestLatency(self.Closest([self.config['lat'], self.config['lon']], self.server_list, self.bestServers)) - for server in best: - self.servers.append(server['url']) - - def AsyncRequest(self, url, num, upload=0): - connections=[] - d=Manager().dict() - start=time.time() - for i in range(num): - full_url=self.servers[i % len(self.servers)]+url - #print full_url - connection={} - connection['parent'], connection['child']= Pipe() - if upload==1: - connection['connection'] = Process(target=self.AsyncPost, args=(connection['child'], full_url, i, num, d)) - else: - connection['connection'] = Process(target=self.AsyncGet, args=(connection['child'], full_url, i, num, d)) - connection['connection'].start() - connections.append(connection) - - for c in range(num): - connections[c]['size'], connections[c]['start'], connections[c]['end']=connections[c]['parent'].recv() - connections[c]['connection'].join() - - end=time.time() - - print_debug(' \r') - - sizes=0 - #tspeed=0 - for c in range(num): - if connections[c]['end'] is not False: - #tspeed=tspeed+(connections[c]['size']/(connections[c]['end']-connections[c]['start'])) - sizes=sizes+connections[c]['size'] - - # Using more precise times for downloads - if upload==0: - if c==0: - start=connections[c]['start'] - end=connections[c]['end'] - else: - if connections[c]['start'] < start: - start=connections[c]['start'] - if connections[c]['end'] > end: - end=connections[c]['end'] - - took=end-start - - return [sizes, took] - - def TestUpload(self): - # Testing upload speed - - url="upload.php?x=" + str( time.time() ) - - sizes, took=[0,0] - data="" - for i in range(0, len(self.upSizes)): - if len(data) == 0 or self.upSizes[i] != self.upSizes[i-1]: - #print_debug("Generating new string to upload. Length: %d\n" % (self.upSizes[i])) - data=''.join("1" for x in xrange(self.upSizes[i])) - self.postData=urllib.urlencode({'upload6': data }) - - if i<2: - thrds=1 - elif i<5: - thrds=2 - elif i<7: - thrds=2 - elif i<10: - thrds=3 - elif i<25: - thrds=6 - elif i<45: - thrds=4 - elif i<65: - thrds=3 - else: - thrds=2 - - sizes, took=self.AsyncRequest(url, thrds, 1) - #sizes, took=self.AsyncRequest(url, (i<4 and 1 or (i<6 and 2 or (i<6 and 4 or 8))), 1) - if sizes==0: - continue - - size=self.SpeedConversion(sizes) - speed=size/took - print_debug("Upload size: %0.2f MiB; Uploaded in %0.2f s\n" % - (size, took)) - print_debug("\033[92mUpload speed: %0.2f %s/s\033[0m\n" % - (speed, self.units)) - - if self.up_speed5: - break - - #print_debug("Upload size: %0.2f MiB; Uploaded in %0.2f s\n" % (self.SpeedConversion(sizes), took)) - #print_debug("Upload speed: %0.2f MiB/s\n" % (self.SpeedConversion(sizes)/took)) - - def SpeedConversion(self, data): - if self.unit==1: - result=(float(data)/1024/1024) - else: - result=(float(data)/1024/1024)*1.048576*8 - return result - - def TestDownload(self): - # Testing download speed - sizes, took=[0,0] - for i in range(0, len(self.downList)): - url="random"+self.downList[i]+".jpg?x=" + str( time.time() ) + "&y=3" - - - if i<2: - thrds=1 - elif i<5: - thrds=2 - elif i<11: - thrds=2 - elif i<13: - thrds=4 - elif i<25: - thrds=2 - elif i<45: - thrds=3 - elif i<65: - thrds=2 - else: - thrds=2 - - sizes, took=self.AsyncRequest(url, thrds ) - #sizes, took=self.AsyncRequest(url, (i<1 and 2 or (i<6 and 4 or (i<10 and 6 or 8))) ) - if sizes==0: - continue - - size=self.SpeedConversion(sizes) - speed=size/took - print_debug("Download size: %0.2f MiB; Downloaded in %0.2f s\n" % - (size, took)) - print_debug("\033[91mDownload speed: %0.2f %s/s\033[0m\n" % - (speed, self.units)) - - if self.down_speed5: - break - - #print_debug("Download size: %0.2f MiB; Downloaded in %0.2f s\n" % (self.SpeedConversion(sizes), took)) - #print_debug("Download speed: %0.2f %s/s\n" % (self.SpeedConversion(sizes)/took, self.units)) - - def TestSpeed(self): - - if self.server=='list-servers': - self.config=self.LoadConfig() - self.server_list=self.LoadServers() - self.ListServers(self.numTop) - return - - if self.server == '': - self.config=self.LoadConfig() - self.server_list=self.LoadServers() - self.FindBestServer() - - self.TestDownload() - self.TestUpload() - - print_result("%0.2f,%0.2f,\"%s\",\"%s\"\n" % (self.down_speed, self.up_speed, self.units, self.servers)) - - def ListServers(self, num=0): - - allSorted=self.Closest([self.config['lat'], self.config['lon']], self.server_list, num) - - for i in range(0, len(allSorted)): - print_result("%s. %s (%s, %s, %s) [%0.2f km]\n" % - (i+1, allSorted[i]['url'], allSorted[i]['sponsor'], allSorted[i]['name'], allSorted[i]['country'], allSorted[i]['distance'])) - -def print_debug(string): - if args.suppress!=True: - sys.stderr.write(string.encode('utf8')) - #return - -def print_result(string): - if args.store==True: - sys.stdout.write(string.encode('utf8')) - #return - -# Thx to Ryan Sears for http://bit.ly/17HhSli -def set_proxy(typ=socks.PROXY_TYPE_SOCKS4, host="127.0.0.1", port=9050): - socks.setdefaultproxy(typ, host, port) - socket.socket = socks.socksocket - -def main(args): - - if args.use_proxy: - if args.use_proxy==5: - set_proxy(typ=socks.PROXY_TYPE_SOCKS5, host=args.proxy_host, port=args.proxy_port) - else: - set_proxy(typ=socks.PROXY_TYPE_SOCKS4, host=args.proxy_host, port=args.proxy_port) - - if args.listservers: - args.store=True - - if args.listservers!=True and args.server=='' and args.store!=True: - print_debug("Getting ready. Use parameter -h or --help to see available features.\n") - else: - print_debug("Getting ready\n") - try: - t=TeSpeed( - args.listservers and 'list-servers' or args.server, - args.listservers, args.servercount, - args.store and True or False, - args.suppress and True or False, - args.unit and True or False, - chunksize=args.chunksize - ) - except (KeyboardInterrupt, SystemExit): - print_debug("\nTesting stopped.\n") - #raise - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='TeSpeed, CLI SpeedTest.net') - - parser.add_argument('server', nargs='?', type=str, default='', help='Use the specified server for testing (skip checking for location and closest server).') - parser.add_argument('-ls', '--list-servers', dest='listservers', nargs='?', default=0, const=10, help='List the servers sorted by distance, nearest first. Optionally specify number of servers to show.') - parser.add_argument('-w', '--csv', dest='store', action='store_const', const=True, help='Print CSV formated output to STDOUT.') - parser.add_argument('-s', '--suppress', dest='suppress', action='store_const', const=True, help='Suppress debugging (STDERR) output.') - parser.add_argument('-mib', '--mebibit', dest='unit', action='store_const', const=True, help='Show results in mebibits.') - parser.add_argument('-n', '--server-count', dest='servercount', nargs='?', default=1, const=1, help='Specify how many different servers should be used in paralel. (Default: 1) (Increase it for >100Mbit testing.)') - - parser.add_argument('-p', '--proxy', dest='use_proxy', type=int, nargs='?', const=4, help='Specify 4 or 5 to use SOCKS4 or SOCKS5 proxy.') - parser.add_argument('-ph', '--proxy-host', dest='proxy_host', type=str, nargs='?', default='127.0.0.1', help='Specify socks proxy host. (Default: 127.0.0.1)') - parser.add_argument('-pp', '--proxy-port', dest='proxy_port', type=int, nargs='?', default=9050, help='Specify socks proxy port. (Default: 9050)') - - parser.add_argument('-cs', '--chunk-size', dest='chunksize', nargs='?', type=int, default=10240, help='Specify chunk size after wich tespeed calculates speed. Increase this number 4 or 5 times if you use weak hardware like RaspberryPi. (Default: 10240)') - - #parser.add_argument('-i', '--interface', dest='interface', nargs='?', help='If specified, measures speed from data for the whole network interface.') - - args = parser.parse_args() - main(args) diff --git a/tespeed/__init__.py b/tespeed/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tespeed/bin.py b/tespeed/bin.py new file mode 100644 index 0000000..004552d --- /dev/null +++ b/tespeed/bin.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- + +# Copyright: +# 2012-2013 Janis Jansons (janis.jansons@janhouse.lv) +# 2014 David Fischer (david.fischer.ch@gmail.com) + +from __future__ import absolute_import, division, print_function, unicode_literals + +import argparse + +from .core import TeSpeed +from .utils import Log, set_proxy + +__all__ = ('tespeed', ) + + +def tespeed(): + + parser = argparse.ArgumentParser(description='TeSpeed, CLI SpeedTest.net') + + parser.add_argument('server', nargs='?', type=str, default='', help='Use the specified server for testing (skip ' + 'checking for location and closest server).') + parser.add_argument('-ls', '--list-servers', dest='list_servers', nargs='?', default=0, const=10, + help='List the servers sorted by distance, nearest first. Optionally specify number of servers ' + 'to show.') + parser.add_argument('-w', '--csv', dest='store', action='store_true', help='Print CSV formated output to STDOUT.') + parser.add_argument('-s', '--suppress', dest='suppress', action='store_true', + help='Suppress debugging (STDERR) output.') + parser.add_argument('-mib', '--mebibit', dest='unit', action='store_true', help='Show results in mebibits.') + parser.add_argument('-n', '--server-count', dest='server_count', nargs='?', default=1, const=1, + help='Specify how many different servers should be used in parallel. (Default: 1) ' + '(Increase it for >100Mbit testing.)') + + parser.add_argument('-p', '--proxy', dest='use_proxy', type=int, nargs='?', const=4, + help='Specify 4 or 5 to use SOCKS4 or SOCKS5 proxy.') + parser.add_argument('-ph', '--proxy-host', dest='proxy_host', type=str, nargs='?', default='127.0.0.1', + help='Specify socks proxy host. (Default: 127.0.0.1)') + parser.add_argument('-pp', '--proxy-port', dest='proxy_port', type=int, nargs='?', default=9050, + help='Specify socks proxy port. (Default: 9050)') + + parser.add_argument('-cs', '--chunk-size', dest='chunk_size', nargs='?', type=int, default=10240, + help='Specify chunk size after which tespeed calculates speed. Increase this number 4 or 5 ' + 'times if you use weak hardware like RaspberryPi. (Default: 10240)') + + # parser.add_argument('-i', '--interface', dest='interface', nargs='?', + # help='If specified, measures speed from data for the whole network interface.') + + args = parser.parse_args() + + if args.use_proxy: + set_proxy(version=args.use_proxy, host=args.proxy_host, port=args.proxy_port) + + if args.list_servers: + args.store = True + + log = Log(suppress=args.suppress, store=args.store) + if not args.list_servers and args.server == '' and not args.store: + log.debug('Getting ready. Use parameter -h or --help to see available features.\n') + else: + log.debug('Getting ready\n') + try: + tespeed = TeSpeed(server=args.list_servers and 'list-servers' or args.server, num_servers=args.server_count, + num_top=args.list_servers, unit=args.unit, chunk_size=args.chunk_size, log=log) + tespeed.run_tests() + except (KeyboardInterrupt, SystemExit): + log.debug('\nSpeed test stopped.\n') diff --git a/tespeed/core.py b/tespeed/core.py new file mode 100755 index 0000000..a3a4d45 --- /dev/null +++ b/tespeed/core.py @@ -0,0 +1,343 @@ +# -*- coding: utf-8 -*- + +# Copyright: +# 2012-2013 Janis Jansons (janis.jansons@janhouse.lv) +# 2014 David Fischer (david.fischer.ch@gmail.com) + +from __future__ import absolute_import, division, print_function, unicode_literals + +import urllib, urllib2, time +from lxml import etree +from multiprocessing import Process, Pipe, Manager + +from .utils import CallbackStringIO, Log, closest, decompress_response, num_download_threads_for, num_upload_threads_for + +__all__ = ('TeSpeed', ) + + +class TeSpeed(object): + + DOWNLOAD_LIST = [ + '350x350', '350x350', '500x500', '500x500', '750x750', '750x750', '1000x1000', '1500x1500', '2000x2000', + '2500x2500', + + '3000x3000', '3500x3500', '4000x4000', '1000x1000', '1000x1000', '1000x1000', '1000x1000', '1000x1000', + '1000x1000', '1000x1000', '1000x1000', '1000x1000', '1000x1000', '1000x1000', '1000x1000', '1000x1000', + '1000x1000', '1000x1000', '1000x1000', '1000x1000', + + '2000x2000', '2000x2000', '2000x2000', '2000x2000', '2000x2000', '2000x2000', '2000x2000', '2000x2000', + '2000x2000', '2000x2000', '2000x2000', '2000x2000', '2000x2000', '2000x2000', '2000x2000', '2000x2000', + '2000x2000', '2000x2000', '2000x2000', '2000x2000', + + '4000x4000', '4000x4000', '4000x4000', '4000x4000', '4000x4000' + ] + + UPLOAD_SIZES = [ + 1024*256, 1024*256, 1024*512, 1024*512, 1024*1024, 1024*1024, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*512, + 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, + + 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, + 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, + + 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, 1024*256, + 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, 1024*512, + + 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, + 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2, + 1024*1024*2, 1024*1024*2, 1024*1024*2, 1024*1024*2 + ] + + HEADERS = { + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', + 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:11.0) Gecko/20100101 Firefox/11.0', + 'Accept-Language': 'en-us,en;q=0.5', + 'Connection': 'keep-alive', + 'Accept-Encoding': 'gzip, deflate', + #'Referer' : 'http://c.speedtest.net/flash/speedtest.swf?v=301256', + } + + def __init__(self, server=None, num_servers=3, num_top=0, unit=False, chunk_size=10240, log=None): + # FIXME having server, num_servers, ... looks like a hack that should be the responsibility of the caller + self.server = server + self.servers = [server] if server else [] + self.num_servers = num_servers + self.num_top = int(num_top) + if unit: + self.units = 'MiB' + self.unit = 1 + else: + self.units = 'Mbit' + self.unit = 0 + self.chunk_size = chunk_size + self.log = log + + if log.store: + log.debug('Printing CSV formated results to STDOUT.\n') + + self.best_servers = 5 + self.latency_count = 10 + self.post_data = '' + + self._config = None + self._server_list = None + + def convert_size(self, value): + return value / 1024 ** 2 * (1 if self.unit == 1 else 1.048576 * 8) + + def get_request(self, uri): + """Generate a GET request to be used with urlopen.""" + return urllib2.Request(uri, headers=self.HEADERS) + + def post_request(self, uri, stream): + """Generate a POST request to be used with urlopen.""" + return urllib2.Request(uri, stream, headers=self.HEADERS) + + def chunk_report(self, bytes_so_far, chunk_size, total_size, num, num_threads, d, w): + """Receive status update from download thread.""" + if w != 1: + d[num] = bytes_so_far + down = 0 + for i in xrange(num_threads): + down += d.get(i, 0) + if num == 0 or down >= total_size * num_threads: + percent = round(down / (total_size * num_threads) * 100, 2) + self.log.debug('Downloaded %d of %d bytes (%0.2f%%) in %d threads\r' % + (down, total_size * num_threads, percent, num_threads)) + + def chunk_read(self, response, num, num_threads, d, w=0, chunk_size=False, report_hook=None): + + chunk_size = chunk_size or self.chunk_size + + if w == 1: + return [0, 0, 0] + + total_size = int(response.info().getheader('Content-Length').strip()) + bytes_so_far = start_time = 0 + while True: + chunk = 0 + if not start_time: + chunk = response.read(1) + start_time = time.time() + else: + chunk = response.read(chunk_size) + if not chunk: + break + bytes_so_far += len(chunk) + if report_hook: + report_hook(bytes_so_far, chunk_size, total_size, num, num_threads, d, w) + + return [bytes_so_far, start_time, time.time()] + + def async_get(self, conn, uri, num, num_threads, d): + request = self.get_request(uri) + start_time = end_time = size = 0 + try: + response = urllib2.urlopen(request, timeout=30) + size, start_time, end_time = self.chunk_read(response, num, num_threads, d, report_hook=self.chunk_report) + except: + self.log.debug(Log.BLANK_LINE) + self.log.debug('Failed downloading.\n') + conn.send([0, 0, False]) + conn.close() + return + conn.send([size, start_time, end_time]) + conn.close() + + def async_post(self, conn, uri, num, num_threads, d): + postlen = len(self.post_data) + stream = CallbackStringIO(num, num_threads, d, self.post_data, log=self.log) + request = self.post_request(uri, stream) + start_time = end_time = 0 + try: + response = urllib2.urlopen(request, timeout=30) + size, start_time, end_time = self.chunk_read(response, num, num_threads, d, 1, + report_hook=self.chunk_report) + except: + self.log.debug(Log.BLANK_LINE) + self.log.debug('Failed uploading.\n') + conn.send([0, 0, False]) + conn.close() + return + conn.send([postlen, start_time, end_time]) + conn.close() + + def async_request(self, url, num, upload=0): + connections = [] + d = Manager().dict() + + start_time = time.time() + + for i in xrange(num): + full_url = self.servers[i % len(self.servers)] + url + connection = {} + connection['parent'], connection['child'] = Pipe() + connection['connection'] = Process(target=self.async_post if upload == 1 else self.async_get, + args=(connection['child'], full_url, i, num, d)) + connection['connection'].start() + connections.append(connection) + + for i in xrange(num): + connections[i]['size'], connections[i]['start'], connections[i]['end'] = connections[i]['parent'].recv() + connections[i]['connection'].join() + + end_time = time.time() + + self.log.debug(Log.BLANK_LINE) + + sizes = 0 + for i in xrange(num): + if connections[i]['end'] is not False: + sizes += connections[i]['size'] + + # Using more precise times for downloads + if upload == 0: + if i == 0: + start_time = connections[i]['start'] + end_time = connections[i]['end'] + else: + if connections[i]['start'] < start_time: + start_time = connections[i]['start'] + if connections[i]['end'] > end_time: + end_time = connections[i]['end'] + + return [sizes, end_time - start_time] + + def config(self, force_reload=False): + """Return the configuration loaded from the speedtest.net server API.""" + if not self._config or force_reload: + self.log.debug('Loading the configuration...\n') + uri = 'http://speedtest.net/speedtest-config.php?x=' + str(time.time()) + response = urllib2.urlopen(self.get_request(uri)) + client = etree.fromstring(decompress_response(response)).find('client') + self._config = { + 'ip': client.attrib['ip'], + 'isp': client.attrib['isp'], + 'lat': float(client.attrib['lat']), + 'lon': float(client.attrib['lon']) + } + self.log.debug('IP: {ip}; Lat: {lat}; Lon: {lon}; ISP: {isp}\n'.format(**self._config)) + return self._config + + def server_list(self, force_reload=False): + """Return the list of servers loaded from the speedtest.net server API.""" + if not self._server_list or force_reload: + self.log.debug('Loading the list of servers...\n') + uri = 'http://speedtest.net/speedtest-servers.php?x=' + str(time.time()) + response = urllib2.urlopen(self.get_request(uri)) + servers_xml = etree.fromstring(decompress_response(response)) + self._server_list = [ + { + 'lat': float(server.attrib['lat']), + 'lon': float(server.attrib['lon']), + 'url': server.attrib['url'].rsplit('/', 1)[0] + '/', + #'url2': server.attrib['url2'].rsplit('/', 1)[0] + '/', + 'name': server.attrib['name'], + 'country': server.attrib['country'], + 'sponsor': server.attrib['sponsor'], + 'id': server.attrib['id'], + } for server in servers_xml.find('servers').findall('server') + ] + return self._server_list + + def find_best_servers(self): + self.log.debug('Looking for closest and best servers...\n') + best_servers = self.test_latency(closest([self.config()['lat'], self.config()['lon']], self.server_list(), + self.best_servers)) + self.servers.extend(server['url'] for server in best_servers) + + def list_servers(self, num=0): + for i, server in enumerate(closest([self.config()['lat'], self.config()['lon']], self.server_list(), num), 1): + self.log.result('%s. %s (%s, %s, %s) [%0.2f km]\n' % (i, server['url'], server['sponsor'], server['name'], + server['country'], server['distance'])) + + def test_latency(self, servers): + """Find servers with lowest latency.""" + self.log.debug('Testing latency...\n') + po = [] + for server in servers: + latency = self.test_single_latency(server['url'] + 'latency.txt?x=' + str(time.time())) * 1000 + if not latency: + continue + self.log.debug('%0.0f ms latency for %s (%s, %s, %s) [%0.2f km]\n' % (latency, server['url'], + server['sponsor'], server['name'], server['country'], server['distance'])) + server['latency'] = latency + # Pick specified amount of servers with best latency for testing + if int(len(po)) < int(self.num_servers): + po.append(server) + else: + largest = -1 + for x in xrange(len(po)): + if largest < 0: + if latency < po[x]['latency']: + largest = x + elif po[largest]['latency'] < po[x]['latency']: + largest = x + if largest >= 0: + po[largest] = server + return po + + def test_single_latency(self, destination_address): + """Check latency for single server. Does that by loading latency.txt (empty page).""" + request = self.get_request(destination_address) + average_time = total = 0 + for i in xrange(self.latency_count): + start_time = time.time() + try: + urllib2.urlopen(request, timeout=5) + average_time += time.time() - start_time + total += 2 # Already "multiply" by 2 to return half of the round-trip (the latency) for free + except urllib2.URLError: + pass + return average_time / total if total else None + + def test_download(self): + """Test download speed.""" + max_speed = -1 + for counter, download in enumerate(self.DOWNLOAD_LIST): + url = 'random' + download + '.jpg?x=' + str(time.time()) + '&y=3' + sizes, took = self.async_request(url, num_download_threads_for(counter)) + if sizes == 0: + continue + + size = self.convert_size(sizes) # FIXME rename sizes to something more meaningful + speed = size / took + max_speed = max(speed, max_speed) + self.log.debug('Download size: %0.2f MiB; Downloaded in %0.2f s\n' % (size, took)) + self.log.debug('\033[91mDownload speed: %0.2f %s/s\033[0m\n' % (speed, self.units)) + + if took > 5: + break + return max_speed + + def test_upload(self): + """Test upload speed.""" + max_speed = -1 + data, url = '', 'upload.php?x=' + str(time.time()) + for i in xrange(len(self.UPLOAD_SIZES)): + if len(data) == 0 or self.UPLOAD_SIZES[i] != self.UPLOAD_SIZES[i-1]: + data = ''.join('1' for x in xrange(self.UPLOAD_SIZES[i])) + self.post_data = urllib.urlencode({'upload6': data}) + + sizes, took = self.async_request(url, num_upload_threads_for(i), 1) + if sizes == 0: + continue + + size = self.convert_size(sizes) # FIXME rename sizes to something more meaningful + speed = size / took + max_speed = max(speed, max_speed) + self.log.debug('Upload size: %0.2f MiB; Uploaded in %0.2f s\n' % (size, took)) + self.log.debug('\033[92mUpload speed: %0.2f %s/s\033[0m\n' % (speed, self.units)) + + if took > 5: + break + return max_speed + + def run_tests(self): + if self.server == 'list-servers': + self.list_servers(self.num_top) + else: + if not self.server: + self.find_best_servers() + download_speed = self.test_download() + upload_speed = self.test_upload() + self.log.result('%0.2f,%0.2f,"%s","%s"\n' % (download_speed, upload_speed, self.units, self.servers)) diff --git a/tespeed/utils.py b/tespeed/utils.py new file mode 100644 index 0000000..a4594c6 --- /dev/null +++ b/tespeed/utils.py @@ -0,0 +1,159 @@ +# -*- coding: utf-8 -*- + +# Copyright: +# 2012-2013 Janis Jansons (janis.jansons@janhouse.lv) +# 2014 David Fischer (david.fischer.ch@gmail.com) + +from __future__ import absolute_import, division, print_function, unicode_literals + +import gzip, socket, socks, sys +from math import radians, cos, sin, asin, sqrt +from StringIO import StringIO + +__all__ = ( + 'CallbackStringIO', 'Log', 'closest', 'decompress_response', 'distance', 'num_download_threads_for', + 'num_upload_threads_for', 'set_proxy' +) + + +# Magic! +def getaddrinfo(*args): + return [(socket.AF_INET, socket.SOCK_STREAM, 6, '', (args[0], args[1]))] +socket.getaddrinfo = getaddrinfo + + +class CallbackStringIO(StringIO): + """Using StringIO with callback to measure upload progress""" + + def __init__(self, num, th, d, buf='', log=None): + # Force self.buf to be a string or unicode + if not isinstance(buf, basestring): + buf = str(buf) + self.buf = buf + self.len = len(buf) + self.buflist = [] + self.pos = 0 + self.closed = False + self.softspace = 0 + self.th = th + self.num = num + self.d = d + self.total = self.len*self.th + self.log = log + + def read(self, n=10240): + next_chunk = StringIO.read(self, n) + #if 'done' in self.d: + # return + self.d[self.num] = self.pos + down = 0 + for i in xrange(self.th): + down = down + self.d.get(i, 0) + if self.num == 0: + percent = down / self.total + percent = round(percent * 100, 2) + if self.log: + self.log.debug('Uploaded %d of %d bytes (%0.2f%%) in %d threads\r' % + (down, self.total, percent, self.th)) + #if down >= self.total: + # if self.log: self.log.debug('\n') + # self.d['done']=1 + return next_chunk + + def __len__(self): + return self.len + + +class Log(object): + + BLANK_LINE = ' \r' + + def __init__(self, suppress=False, store=False): + self.suppress = suppress + self.store = store + + def debug(self, string): + if not self.suppress: + sys.stderr.write(string.encode('utf8')) + + def result(self, string): + if self.store: + sys.stdout.write(string.encode('utf8')) + + +def closest(center, points, num=5): + """Return object that is closest to center.""" + closest = {} + for p in xrange(len(points)): + p_distance = distance(center, [points[p]['lat'], points[p]['lon']]) + points[p]['distance'] = p_distance + while True: + if p_distance in closest: + p_distance = p_distance + 00.1 + else: + break + closest[p_distance] = points[p] + closest_objects = [] + # FIXME can be simplified + for key in sorted(closest): + closest_objects.append(closest[key]) + if len(closest_objects) >= num and num != 0: + break + return closest_objects + + +def decompress_response(response): + """Return decompressed gzip response.""" + data = StringIO(response.read()) + gzipper = gzip.GzipFile(fileobj=data) + return gzipper.read() + + +def distance(one, two): + """ + Compute the great circle distance between two points on the earth specified in decimal degrees `haversine + formula `_ convert decimal degrees to radians. + """ + lon1, lat1, lon2, lat2 = map(radians, [one[0], one[1], two[0], two[1]]) + a = sin((lat2 - lat1) / 2) ** 2 + cos(lat1) * cos(lat2) * sin((lon2 - lon1) / 2) ** 2 + c = 2 * asin(sqrt(a)) + km = 6367 * c + return km + + +def num_download_threads_for(num_downloads): + if num_downloads < 2: + return 1 + elif num_downloads < 11: + return 2 + elif num_downloads < 13: + return 4 + elif num_downloads < 25: + return 2 + elif num_downloads < 45: + return 3 + elif num_downloads < 65: + return 2 + return 2 + + +def num_upload_threads_for(num_uploads): + if num_uploads < 2: + return 1 + elif num_uploads < 7: + return 2 + elif num_uploads < 10: + return 3 + elif num_uploads < 25: + return 6 + elif num_uploads < 45: + return 4 + elif num_uploads < 65: + return 3 + return 2 + + +# Thanks to Ryan Sears for http://bit.ly/17HhSli +def set_proxy(version, host='127.0.0.1', port=9050): + socks.setdefaultproxy(getattr(socks, 'PROXY_TYPE_SOCKS' + version), host, port) + socket.socket = socks.socksocket