Python制作糗事百科爬虫

# coding=utf8

import re
import requests
from lxml import etree
from multiprocessing.dummy import Pool as ThreadPool
import sys

reload(sys)
sys.setdefaultencoding('utf-8')


def getnewpage(url, total):
 nowpage = int(re.search('(d+)', url, re.S).group(1))
 urls = []

 for i in range(nowpage, total + 1):
 link = re.sub('(d+)', '%s' % i, url, re.S)
 urls.append(link)

 return urls


def spider(url):
 html = requests.get(url)
 selector = etree.HTML(html.text)

 author = selector.xpath('//*[@id="content-left"]/p/p[1]/a[2]/@title')
 content = selector.xpath('//*[@id="content-left"]/p/p[2]/text()')
 vote = selector.xpath('//*[@id="content-left"]/p/p[3]/span/i/text()')

 length = len(author)
 for i in range(0, length):
 f.writelines('作者 : ' + author[i] + '
')
 f.writelines('内容 ：' + str(content[i]).replace('
','') + '
')
 f.writelines('支持 ： ' + vote[i] + '

')


if __name__ == '__main__':

 f = open('info.txt', 'a')
 url = 'http://www.qiushibaike.com/text/page/1/'
 urls = getnewpage(url, 20)

 pool = ThreadPool(4)
 pool.map(spider,urls)
 f.close()