从nginx日志读取URL来做性能测试

做性能测试的时候都希望能模拟用户的行为，ningx记录了用户的行为，比如nginx一般会记录如下：

 

222.210.49.198 - 209sdj3lufa7sf32 -c2b3lp - [14/Sep/2012:22:59:02 +0800]"GET /search/?page=33&kw=%E6%B5%B7%E8%B4%BC&type=feed&_type= HTTP/1.1" 200 14466 "http://www.duitang.com/search/?kw=%E6%B5%B7%E8%B4%BC&type=feed" "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.84 Safari/535.11 LBBROWSER"

 

写了一个简单的python脚本从nginx读取URL来做性能测试：

 

# -*- coding: utf-8 -*-
import re
import urllib2
import json
import threading
import Queue
import os

host="http://7199.t.duitang.com"
thread_count = 1
running = True

def check(uri):
   try:
	url = "%s%s"%(host,uri)
        content = urllib2.urlopen(url=url,timeout=2)
	#print '%s %s'%(content.code,uri)
	#print content.info()
	#print content.msg
        #print content.readlines()
	#json.loads(content.readlines())
	#print 'ok'
   except Exception,e:
        #print e
	print '%s %s'%(e,uri)
	#out = StringIO()
        #print_exc(file=out)
        #logger.info("check %s fail %s"%(server,out.getvalue()))

class Executor(threading.Thread):

        def __init__(self,queue):
                threading.Thread.__init__(self)
                self.queue = queue

	def run(self):

                while running:
			url = self.queue.get()
			print queue.qsize()
			check(url)
			self.queue.task_done()		

def analysis(readline,queue):
	r = re.compile("\d+")
	mc_url = re.compile("GET /[a-zA-Z/0-9_?=&]*")
	result = {}
	for read in readline:
		ip=url=http_code=http_time = None
		match =  mc_url.search(read)	
		if match:
			url = str(match.group())[4:].strip()
			queue.put(url)

def clean(a,b):
	print 'exit'
	running=False
	exit(1)

from signal import signal,SIGTERM,SIGINT,SIGQUIT
signal(SIGTERM,clean)
signal(SIGINT,clean)
signal(SIGQUIT,clean)

if __name__ == "__main__":
	queue = Queue.Queue()
	for i  in range(thread_count):
        	t = Executor(queue)
        	t.setDaemon(True)
        	t.start()

	f = open("by_category","r")
	analysis(f.readlines(),queue)
	print "total %s" %queue.qsize()
	queue.join()
	print 'exit'
	running = False