from urlparse import urlparse
from threading import Thread
import httplib, sys
from Queue import Queue
concurrent = 200
def doWork():
while True:
url=q.get()
status,url=getStatus(url)
doSomethingWithResult(status,url)
q.task_done()
def getStatus(ourl):
try:
url = urlparse(ourl)
conn = httplib.HTTPConnection(url.netloc)
conn.request("HEAD", url.path)
res = conn.getresponse()
return res.status, ourl
except:
return "error", ourl
def doSomethingWithResult(status, url):
print status, url
q=Queue(concurrent*2)
for i in range(concurrent):
t=Thread(target=doWork)
t.daemon=True
t.start()
try:
for url in open('urllist.txt'):
q.put(url.strip())
q.join()
except KeyboardInterrupt:
sys.exit(1)
this can be used to test broken link
change a little to crawl file
分享到:
相关推荐
重现步骤docker run --detach --name vespa-response --hostname vespa-container --volume ` pwd ` :/app --publish 8080:8080 vespaengine/vespadocker exec -it vespa-response bash -c ' /opt/vespa/bin/vespa-...
Codeigniter.php 是 CI 4 的核心所在,在这里接收并处理了 request 请求,安全检查,缓存处理, URL 解析以及路由匹配,执行过滤器,加载运行 Controller ,组装 repsonse 响应并发送等。
基于Udemy课程带有REST保证的REST API自动化的完整指南-课程的完整功能REST保证的概念包括:JSONPATH和XML PATH:在RestAssured中提取带有这些库的Repsonse信息添加根路径和多个断言以测试记录请求/响应到控制台的...
[NSURLConnection sendSynchronousRequest:request returningResponse:&repsonse error:nil]; NSString *mimeType = repsonse.MIMEType; NSLog(@"%@", repsonse.MIMEType); [[AFHTTPSessionManager manager] ...
很简单,您可以在正文中发送oauth设置,应用程序将调用github oauth并返回repsonse。本地运行确保您具有并且要在Heroku上部署已安装的 。 $ git clone ...
另一方面,STK App或STK Telephony层发出的命令则封装在`RIL_REQUEST_STK_SEND_TERMINAL_REPSONSE`,`RIL_REQUEST_SEND_ENVELOPE_COMMAND`或`RIL_REQUEST_HANDLE_CALL_SETUP_REQUESTED_FROM_SIM`中。 消息内容通常...