python 爬虫小结1

python 爬虫小结1

1 正则匹配中注意的：
import re
a='<div>指数</div>'
word=re.findall('<div>(.*?)</div>',a)
print(word)

  其中（.*?)是能匹配基本所有的字符，但是对于跨行的例外
比如
import re
a='''<div>abc
</div>'''
word=re.findall('<div>(.*?)</div>',a,re.S)
print(word)
因为findall是逐行匹配的，当第一行没匹配的时候，从第2行匹配，所以最后参数用re.S,标识的是匹配包括换行在内的字符；
，在爬虫的时候，一般再进行换行清理下，使用
print(word[0].strip())

2 简单例子：
    import requests
import re

headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36'
}

info_lists = []

def judgment_sex(class_name):
  if class_name == 'womenIcon':
      return '女'
  else:
      return  '男'

def get_info(url):
    res = requests.get(url)
    ids = re.findall('<h2>(.*?)</h2>',res.text,re.S)
    levels = re.findall('<div class="articleGender \D+Icon">(.*?)</div>',res.text,re.S)
    sexs = re.findall('<div class="articleGender (.*?)">',res.text,re.S)
    contents = re.findall('<div class="content">.*?<span>(.*?)</span>',res.text,re.S)
    laughs = re.findall('<span class="stats-vote"><i class="number">(\d+)</i>',res.text,re.S)
    comments = re.findall('<i class="number">(\d+)</i> 评论',res.text,re.S)
    for id,level,sex,content,laugh,comment in zip(ids,levels,sexs,contents,laughs,comments):
        info = {
            'id':id,
            'level':level,
            'sex':judgment_sex(sex),
            'content':content,
            'laugh':laugh,
            'comment':comment
        }
        info_lists.append(info)

if __name__ == '__main__':
    urls = ['http://www.qiushibaike.com/text/page/{}/'.format(str(i)) for i in range(1,10)]
    for url in urls:
        get_info(url)
    for info_list in info_lists:
         f = open('d:/qiushi.txt','a+')
         try:
             f.write(info_list['id']+'\n')
             f.write(info_list['level'] + '\n')
             f.write(info_list['sex'] + '\n')
             f.write(info_list['content'] + '\n')
             f.write(info_list['laugh'] + '\n')
             f.write(info_list['comment'] + '\n\n')
             f.close()
         except UnicodeEncodeError:
          pass
        #print(info_list)


2   python中调用相关网站的API套路：
    import requests
import json
import pprint
address=input('请输入地点')
par = {'address': address, 'key': 'cb649a25c1f81c1451adbeca73623251'}
api = 'http://restapi.amap.com/v3/geocode/geo'
res = requests.get(api, par)
json_data = json.loads(res.text)
pprint.pprint(json_data)
   其中pprint是JSON格式化输出工具，使用JSON.LOAD来加载JSON结果
body > div.main > div.content > div.main-image > p > a > img


3 MYSQL配合抓取

比如抓取豆瓣TOP 250的电影
import requests
from lxml import etree
import re
import pymysql
import time

conn = pymysql.connect(host='localhost', user='root', passwd='38477000', db='python', port=3309, charset='utf8')
cursor = conn.cursor()

headers = {
    'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'
}

def get_movie_url(url):
    html = requests.get(url,headers=headers)
    selector = etree.HTML(html.text)
    movie_hrefs = selector.xpath('//div[@class="hd"]/a/@href')
    for movie_href in movie_hrefs:
        get_movie_info(movie_href)

def get_movie_info(url):
    html = requests.get(url,headers=headers)
    selector = etree.HTML(html.text)
    try:
        name = selector.xpath('//*[@id="content"]/h1/span[1]/text()')[0]
        director = selector.xpath('//*[@id="info"]/span[1]/span[2]/a/text()')[0]
        actors = selector.xpath('//*[@id="info"]/span[3]/span[2]')[0]
        actor = actors.xpath('string(.)')
        style = re.findall('<span property="v:genre">(.*?)</span>',html.text,re.S)[0]
        country = re.findall('<span class="pl">制片国家/地区:</span> (.*?)<br/>',html.text,re.S)[0]
        release_time = re.findall('上映日期:</span>.*?>(.*?)</span>',html.text,re.S)[0]
        time = re.findall('片长:</span>.*?>(.*?)</span>',html.text,re.S)[0]
        score = selector.xpath('//*[@id="interest_sectl"]/div[1]/div[2]/strong/text()')[0]
        cursor.execute(
            "insert into doubanmovie (name,director,actor,style,country,release_time,time,score) values(%s,%s,%s,%s,%s,%s,%s,%s)",
            (str(name), str(director), str(actor), str(style), str(country), str(release_time), str(time), str(score)))
         conn.execute()
    except IndexError:
        pass

if __name__ == '__main__':
    urls = ['https://movie.douban.com/top250?start={}'.format(str(i)) for i in range(0, 250, 25)]
    for url in urls:
        get_movie_url(url)
        time.sleep(5)
    conn.commit()

4 多线程+异步抓取简书网7日最热：
    from lxml import etree
import requests
import re
import json
from multiprocessing import Pool



header = {
    'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36'
}

def get_url(url):
    html = requests.get(url,headers=header)
    selector = etree.HTML(html.text)
    infos = selector.xpath('//ul[@class="note-list"]/li')
    for info in infos:
        article_url_part = info.xpath('div/a/@href')[0]
        get_info(article_url_part)

def get_info(url):
    article_url = 'http://www.jianshu.com/' + url
    html = requests.get(article_url,headers=header)
    selector = etree.HTML(html.text)
    author = selector.xpath('//span[@class="name"]/a/text()')[0]

    print(author)
    article = selector.xpath('//h1[@class="title"]/text()')[0]
    print(article)
    date = selector.xpath('//span[@class="publish-time"]/text()')[0]
    print(date)
    word = selector.xpath('//span[@class="wordage"]/text()')[0]
    print(word)
    view = re.findall('"views_count":(.*?),',html.text,re.S)[0]
    print(view)
    comment = re.findall('"comments_count":(.*?),',html.text,re.S)[0]
    print(comment)
    like = re.findall('"likes_count":(.*?),',html.text,re.S)[0]
    print(like)
    id = re.findall('{"id":(.*?),',html.text,re.S)[0]
    gain_url = 'http://www.jianshu.com/notes/{}/rewards?count=20'.format(id)
    wb_data = requests.get(gain_url,headers=header)
    json_data = json.loads(wb_data.text)
    gain = json_data['rewards_count']

    include_list = []
    include_urls = ['http://www.jianshu.com/notes/{}/included_collections?page={}'.format(id,str(i)) for i in range(1,10)]
    for include_url in include_urls:
        html = requests.get(include_url,headers=header)
        json_data = json.loads(html.text)
        includes = json_data['collections']
        if len(includes) == 0:
            pass
        else:
            for include in includes:
                include_title = include['title']
                include_list.append(include_title)
    info ={
        'author':author,
        'article':article,
        'date':date,
        'word':word,
        'view':view,
        'comment':comment,
        'like':like,
        'gain':gain,
        'include':include_list
    }


if __name__ == '__main__':
    urls = ['http://www.jianshu.com/trending/weekly?page={}'.format(str(i)) for i in range(0, 11)]
    pool = Pool(processes=4)
    pool.map(get_url,urls)


4 表单提交：
   使用FORM表单提交的套路，下面是抓取拉钩网的
    import requests
import json
import time


#client = pymongo.MongoClient('localhost', 27017)
#mydb = client['mydb']
#lagou = mydb['lagou']

headers = {
    'Cookie':'XXXXXX',
    'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36',
    'Connection':'keep-alive'
}

def get_page(url,params):
    html = requests.post(url, data=params, headers=headers)
    json_data = json.loads(html.text)
    print(json_data)
    total_Count = json_data['content']['positionResult']['totalCount']
    page_number = int(total_Count/15) if int(total_Count/15)<30 else 30
    get_info(url,page_number)

def get_info(url,page):
    for pn in range(1,page+1):
        params = {
            'first': 'true',
            'pn': str(pn),
            'kd': 'Python'
        }
        try:
            html = requests.post(url,data=params,headers=headers)
            json_data = json.loads(html.text)
            results = json_data['content']['positionResult']['result']
            for result in results:
                compangeName=result['companyFullName']
                print(compangeName)
                infos = {
                    'businessZones':result['businessZones'],
                    'city':result['city'],
                    'companyFullName':result['companyFullName'],
                    'companyLabelList':result['companyLabelList'],
                    'companySize':result['companySize'],
                    'district':result['district'],
                    'education':result['education'],
                    'explain':result['explain'],
                    'financeStage':result['financeStage'],
                    'firstType':result['firstType'],
                    'formatCreateTime':result['formatCreateTime'],
                    'gradeDescription':result['gradeDescription'],
                    'imState':result['imState'],
                    'industryField':result['industryField'],
                    'jobNature':result['jobNature'],
                    'positionAdvantage':result['positionAdvantage'],
                    'salary':result['salary'],
                    'secondType':result['secondType'],
                    'workYear':result['workYear']
                }
               # lagou.insert_one(infos)
                time.sleep(10)
        except requests.exceptions.ConnectionError:
            pass

if __name__ == '__main__':
    url = 'https://www.lagou.com/jobs/positionAjax.json'
    params = {
        'first': 'true',
        'pn': '1',
        'kd': 'Python'
    }
    get_page(url,params)

5 发现个老外的不错的，在线做词云的工具，样式比较多，推荐下　https://wordart.com/create，比如爬SINA微博好友圈

    import requests
import json

headers = {
    'Cookie':'XXXX'
}

f = open('d:/weibo.txt','a+',encoding='utf-8')

def get_info(url,page):
    html = requests.get(url,headers=headers)
    json_data = json.loads(html.text)
    card_groups = json_data[0]['card_group']
    for card_group in card_groups:
        f.write(card_group['mblog']['text'].split(' ')[0]+'\n')

    next_cursor = json_data[0]['next_cursor']

    if page<50:
        next_url = 'https://m.weibo.cn/index/friends?format=cards&next_cursor='+str(next_cursor)+'&page=1'
        page = page + 1
        get_info(next_url,page)
    else:
        pass
        f.close()

if __name__ == '__main__':
    url = 'https://m.weibo.cn/index/friends?format=cards'
    get_info(url,1)


然后分词：
    import jieba.analyse
path = 'd:\weibo.txt'
fp = open(path,'r',encoding='utf-8')
content = fp.read()
try:
    jieba.analyse.set_stop_words('G:\python学习相关\stop_words_zh.txt')
    tags = jieba.analyse.extract_tags(content, topK=100, withWeight=True)
    for item in tags:
        print(item[0]+'\t'+str(int(item[1]*1000)))
finally:
    fp.close()