Request Analysis

用法：

python bencher.py threadcount duration hitfile


TEST SUMMARY
============
21 REQUESTS in 3s (7.00/s)
7.00 CYCLES in 3s (2.33/s)
  http://www.tl50.com/ status_count: 200=9 (avg 1338ms/req)
  http://www.google.com.hk/ status_count: 200=6 (avg 671ms/req)
  http://www.baidu.com/ status_count: 200=6 (avg 156ms/req)

编写test.txt 内容如下

http://www.google.com.hk/
http://www.baidu.com/

import socket
import thread
import httplib
import time
import sys
import urlparse
import Queue
import os

def check_urls(urls):
    for u in urls:
        url_o = urlparse.urlparse(u)
        conn = httplib.HTTPConnection(url_o.hostname, url_o.port)
        try:
            conn.request("GET", "%s%s?%s" % (url_o.path, url_o.params, url_o.query))
        except socket.gaierror:
            error("the url '%s' contains an unknown hostname" % u)
        except socket.error:
            error("the url '%s' contains an host/port combo that did not respond" % u)

        resp = conn.getresponse()
       
        if resp.status != 200:
            error("the url '%s' did not return 200 is %s" % (u,resp.status))

from itertools import cycle
def test_thread(q, duration, urls):
    looper = cycle(urls)
    ind_bench = dict((u, []) for u in urls)
    start = time.time()
    count = 0
    while time.time() - start < duration:
        url = looper.next()
        url_o = urlparse.urlparse(url)
        m_s = time.time()
        conn = httplib.HTTPConnection(url_o.hostname, url_o.port)
        conn.request("GET", "%s%s?%s" % (url_o.path, url_o.params, url_o.query))
        resp = conn.getresponse()
        ind_bench[url].append((time.time() - m_s, resp.status))
        count += 1
    q.put((count, ind_bench))

def thread_details(num, amt, urls, duration, details):
    def out(s):
        if 'VERBOSE' in os.environ:
            print s
    out('- Thread #%d -' % num)
    cycles = amt / float(len(urls))
    out('  %d total requests (%.2f/s), %.1f cycles (%.1f/s)' % (amt, float(amt) / duration,
    cycles, cycles / duration))
    url_data = {}
    for url in urls:
        stats = details[url]
        status_acc = {}
        tot_time = 0
        for tm, status in stats:
            tot_time += tm
            try:
                status_acc[status] += 1
            except KeyError:
                status_acc[status] = 1
        avg_time = tot_time / float(len(stats))
        out('  %s status_count: %s (avg %dms/req)' % (
        url, ', '.join([('%s=%s' % (k, v)) for k, v in status_acc.iteritems()]),
        avg_time * 1000))
        url_data[url] = (avg_time, status_acc)
    return cycles, url_data

def run_test(count, duration, urls):
    check_urls(urls)
    rec_q = Queue.Queue()
    for x in xrange(count):
        thread.start_new_thread(test_thread, (rec_q, duration, urls))

    results = []
    for x in xrange(count):
        results.append(rec_q.get())

    tot = 0
    cycles = 0
    total_acc = dict((u, {}) for u in urls)
    times = dict((u, 0) for u in urls)
    for x, res in enumerate(results):
        amt, details = res
        cyc, url_data = thread_details(x + 1, amt, urls, duration, details)
        tot += amt
        cycles += cyc
        for url, (avg_time, status_acc) in url_data.iteritems():
            times[url] += avg_time
            for code, c_count in status_acc.iteritems():
                try:
                    total_acc[url]
 += c_count
                except KeyError:
                    total_acc[url]
 = c_count

    print "TEST SUMMARY"
    print "============"
    print "%s REQUESTS in %ss (%.2f/s)" % (tot, duration, tot / float(duration))
    print "%.2f CYCLES in %ss (%.2f/s)" % (cycles, duration, cycles / float(duration))
    for url in urls:
        print '  %s status_count: %s (avg %dms/req)' % (
        url, ', '.join([('%s=%s' % (k, v)) for k, v in total_acc[url].iteritems()]),
        (times[url] / float(count)) * 1000)

   
def error(msg):
    sys.stderr.write('error: %s\n' % msg)
    sys.stderr.write('usage: %s threadcount duration hitfile\n' % sys.argv[0])
    raise SystemExit, 1

def get_urls_from_file(fn):
    return [line.strip() for line in open(fn) if line.strip()]

def main_with_argv():
    args = sys.argv[1:]
    if len(args) != 3:
        error("exactly three arguments are required")
    count, duration, hitfile = args
    try:
        count = int(count)
    except ValueError:
        error("argument #1 (count) must be an int")
    try:
        duration = int(duration)
    except ValueError:
        error("argument #2 (duration) must be an int")
    try:
        urls = get_urls_from_file(hitfile)
    except IOError:
        error("could not retrieve URL list from file '%s'" % hitfile)
    run_test(count, duration, urls)

if __name__ == '__main__':
    main_with_argv()



  


  

    
    
分享到：
      
      
    
  

  

    

      新闻抓取爬虫 PPT
      |
      构建于Rails之上的Hobo 1.0
    
  
  

    

      
2010-05-27 10:49
      
浏览 866
      
评论(0)
      
      
      
分类:编程语言
      
      
查看更多