python中五个模块的详细介绍

# coding:utf-8
from com.wenhy.crawler_baidu_baike import url_manager, html_downloader, html_parser, html_outputer

print "爬虫百度百科调度入口"

# 创建爬虫类
class SpiderMain(object):

 # 初始化 url管理器 html下载器 解析器 

# coding:utf-8

print "URL管理器"


class UrlManager(object):

 # 初始化url容器（set集合）
 def __init__(self):
 self.new_urls = set()
 self.old_urls = set()

 # 添加一个新的待爬取url
 def add_new_url(self, url):

 if url is None:
 return

 # 判断url不在新的url集合中也不再旧的url集合中 说明是一个全新的url
 if url not in self.new_urls and url not in self.old_urls:
 self.new_urls.add(url)

 # 批量添加解析页面的URL
 def add_new_urls(self, urls):

 if urls is None or len(urls) == 0:
 return
 for url in urls:
 self.add_new_url(url)

 # 判断是否有新的待爬取的url
 def has_new_url(self):
 # 如果新的url集合里面len不等于0 说明有待爬取的url
 return len(self.new_urls) != 0

 # 获得url地址
 def get_new_url(self):
 # 获取url并移除当前url
 new_url = self.new_urls.pop()
 self.old_urls.add(new_url)
 return new_url

3、HTML下载器（html_downloader.py）

# coding:utf-8

import urllib2
print "下载HTML"


class HtmlDownloader(object):

 def download(self, url):
 # 判断是否为空
 if url is None:
 return None
 # 下载url
 response = urllib2.urlopen(url)
 # 判断返回结果是否为200
 if response.getcode() != 200:
 return None
 # 成功 返回页面内容
 return response.read()

4、HTML 解析器(html_parser.py)

# coding:utf-8

from bs4 import BeautifulSoup
import re
import urlparse

print "HTML 解析器"


class HtmlParser(object):

 def _get_new_urls(self, page_url, soup):

 # 创建一个集合保存页面解析出来的所有url
 new_urls = set()
 # /item/ 词条url
 links = soup.find_all('a', href=re.compile(r'/item/'))
 for link in links:
 # 获取连接
 new_url = link['href']
 # url拼接
 new_full_url = urlparse.urljoin(page_url, new_url)
 # print 'new_full_url', new_full_url
 # 添加到集合中url
 new_urls.add(new_full_url)
 return new_urls


 def _get_new_data(self, page_url, soup):
 # 定义一个dict 集合
 res_data = {}

 res_data['url'] = page_url
 # < dd class ="lemmaWgt-lemmaTitle-title" >< h1 > 自由软件 < / h1 >
 title_node = soup.find('dd', class_='lemmaWgt-lemmaTitle-title').find('h1')
 res_data['title'] = title_node.get_text()
 # <div class="lemma-summary" label-module="lemmaSummary">
 summary_node = soup.find('div', class_='lemma-summary')
 res_data['summary'] = summary_node.get_text()

 return res_data


 # 解析器
 def parser(self, page_url, html_content):

 if page_url is None or html_content is None:
 return

 # 使用BeautifulSoup 解析html页面
 soup = BeautifulSoup(html_content, 'html.parser', from_encoding='utf-8')
 new_urls = self._get_new_urls(page_url, soup)
 new_data = self._get_new_data(page_url, soup)
 return new_urls, new_data

5、HTML 

# coding:utf-8

print "HTML