文本文件中获取http链接

#!/usr/bin/python2.7  
#coding=utf-8


import sys
import MySQLdb
import time
import hashlib
import datetime
import json
import urllib2
import httplib,urllib
import zim

reload(sys)
sys.setdefaultencoding('utf-8')
del sys.setdefaultencoding
print time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time()))

listtags = []
for i in range(97,123):

   listtags.append(chr(i))
   listtags.append(str(chr(i)).upper())

for i in range(0,10):

   listtags.append(str(i))

listtags.append(str('/'));
listtags.append(str('.'));
listtags.append(str(':'));


def visit_oapi(purl):


    response = urllib2.urlopen(purl)
    html = response.read()
    #print html

conn = MySQLdb.connect(host='10.13.81.11', user='portal',passwd='portal@sohu', db='smc_user', port=3306, charset='utf8')
cur =conn.cursor()
sql = "select passport from tbl_passport_bind_status WHERE  app_id =1 ORDER BY id DESC LIMIT 3000"
cur.execute(sql)
uids = cur.fetchall()

url = "http://internal.passport.sohu.com/openlogin/api/timeline/home";
appkey = 'f@JclHjuQ`DORG<f0,OqS/Q(Lpp4&G'
appid = '1106'

for id in uids:
    #print id[0]
        userid = str(id[0])
        ct = str(str(time.time()).split('.')[0])
        code = hashlib.md5(userid+appid+appkey+ct).hexdigest()   
        dicts={}
   
        dicts['userid'] = userid
        dicts['openid'] = userid
        dicts['ct'] = ct

        dicts['code'] = code
        dicts['appid'] = appid

  
        sjson = json.dumps(dicts)
        print sjson
        purl = url +'?json='+ sjson
        visit_oapi(url)
        req = urllib2.Request(url, sjson)
        response = urllib2.urlopen(req)
        the_page = response.read()   
        zim.getHttp(the_page,userid)
        #print the_page

--------------------------------------------------

#!/usr/bin/python
#coding=utf-8


import string

listtags = []
for i in range(97,123):

   listtags.append(chr(i))
   listtags.append(str(chr(i)).upper())

for i in range(0,10):

   listtags.append(str(i))

listtags.append(str('/'));
listtags.append(str('.'));
listtags.append(str(':'));
def writeFile(userid, listlink):
        fileWriteObj = open("output.txt", 'w')
        for i in listlink:
            fileWriteObj.write(userid+'\t'+i+'\n')
        fileWriteObj.close() 
           
def getHttp(content,userid):
        
        listlinks = []
        content =  string.lower(content)
        inx = string.find(content,'http',0,len(content))
        while inx>0:
                links =""
                hindex = inx+len('http')
                content = content[hindex:]
                i=0
                while i < len(content):
                     chars  =  str(content[i])
                     if chars in listtags:
                           links+=chars
                           i+=1
                     else:
                          print links
                          inx = string.find(content,'http',0,len(content))
                          if string.find(links,'www') < 0:
                                if(isHttpLink('http'+links)):
                                      listlinks.append('http'+links)
                          break
        writeFile(userid,listlinks)
import urllib2
import httplib,urllib
def isHttpLink(url):
  try:
    response = urllib2.urlopen(url)
    html = response.read()
    i = string.find(html,'<title>搜狐新闻客户端-首页</title>')
    j = string.find(html,'你访问的页面地址有误')
    if i > 0 or j >0:
          print ' you are is bad ...........',url
          return True
  except Exception, e:
          print ' you are is error! ...........',url
  return False

if __name__ == '__main__':

      stra = 'sssgsdgdfhttp://w353.5345中哦噶诺..http://sfjsdlfkjs77888)i'
      getHttp(stra,'werw')