Request Analysis

edisonlz

用法：

python bencher.py threadcount duration hitfile


TEST SUMMARY
============
21 REQUESTS in 3s (7.00/s)
7.00 CYCLES in 3s (2.33/s)
  http://www.tl50.com/ status_count: 200=9 (avg 1338ms/req)
  http://www.google.com.hk/ status_count: 200=6 (avg 671ms/req)
  http://www.baidu.com/ status_count: 200=6 (avg 156ms/req)

编写test.txt 内容如下

http://www.google.com.hk/
http://www.baidu.com/




import socketimport threadimport httplibimport timeimport sysimport urlparseimport Queueimport osdef check_urls(urls):    for u in urls:        url_o = urlparse.urlparse(u)        conn = httplib.HTTPConnection(url_o.hostname, url_o.port)        try:            conn.request("GET", "%s%s?%s" % (url_o.path, url_o.params, url_o.query))        except socket.gaierror:            error("the url '%s' contains an unknown hostname" % u)        except socket.error:            error("the url '%s' contains an host/port combo that did not respond" % u)        resp = conn.getresponse()               if resp.status != 200:            error("the url '%s' did not return 200 is %s" % (u,resp.status))from itertools import cycledef test_thread(q, duration, urls):    looper = cycle(urls)    ind_bench = dict((u, []) for u in urls)    start = time.time()    count = 0    while time.time() - start < duration:        url = looper.next()        url_o = urlparse.urlparse(url)        m_s = time.time()        conn = httplib.HTTPConnection(url_o.hostname, url_o.port)        conn.request("GET", "%s%s?%s" % (url_o.path, url_o.params, url_o.query))        resp = conn.getresponse()        ind_bench[url].append((time.time() - m_s, resp.status))        count += 1    q.put((count, ind_bench))def thread_details(num, amt, urls, duration, details):    def out(s):        if 'VERBOSE' in os.environ:            print s    out('- Thread #%d -' % num)    cycles = amt / float(len(urls))    out('  %d total requests (%.2f/s), %.1f cycles (%.1f/s)' % (amt, float(amt) / duration,    cycles, cycles / duration))    url_data = {}    for url in urls:        stats = details[url]        status_acc = {}        tot_time = 0        for tm, status in stats:            tot_time += tm            try:                status_acc[status] += 1            except KeyError:                status_acc[status] = 1        avg_time = tot_time / float(len(stats))        out('  %s status_count: %s (avg %dms/req)' % (        url, ', '.join([('%s=%s' % (k, v)) for k, v in status_acc.iteritems()]),        avg_time * 1000))        url_data[url] = (avg_time, status_acc)    return cycles, url_datadef run_test(count, duration, urls):    check_urls(urls)    rec_q = Queue.Queue()    for x in xrange(count):        thread.start_new_thread(test_thread, (rec_q, duration, urls))    results = []    for x in xrange(count):        results.append(rec_q.get())    tot = 0    cycles = 0    total_acc = dict((u, {}) for u in urls)    times = dict((u, 0) for u in urls)    for x, res in enumerate(results):        amt, details = res        cyc, url_data = thread_details(x + 1, amt, urls, duration, details)        tot += amt        cycles += cyc        for url, (avg_time, status_acc) in url_data.iteritems():            times[url] += avg_time            for code, c_count in status_acc.iteritems():                try:                    total_acc[url] += c_count                except KeyError:                    total_acc[url] = c_count    print "TEST SUMMARY"    print "============"    print "%s REQUESTS in %ss (%.2f/s)" % (tot, duration, tot / float(duration))    print "%.2f CYCLES in %ss (%.2f/s)" % (cycles, duration, cycles / float(duration))    for url in urls:        print '  %s status_count: %s (avg %dms/req)' % (        url, ', '.join([('%s=%s' % (k, v)) for k, v in total_acc[url].iteritems()]),        (times[url] / float(count)) * 1000)   def error(msg):    sys.stderr.write('error: %s\n' % msg)    sys.stderr.write('usage: %s threadcount duration hitfile\n' % sys.argv[0])    raise SystemExit, 1def get_urls_from_file(fn):    return [line.strip() for line in open(fn) if line.strip()]def main_with_argv():    args = sys.argv[1:]    if len(args) != 3:        error("exactly three arguments are required")    count, duration, hitfile = args    try:        count = int(count)    except ValueError:        error("argument #1 (count) must be an int")    try:        duration = int(duration)    except ValueError:        error("argument #2 (duration) must be an int")    try:        urls = get_urls_from_file(hitfile)    except IOError:        error("could not retrieve URL list from file '%s'" % hitfile)    run_test(count, duration, urls)if __name__ == '__main__':    main_with_argv()