#環境是eclipse，python3.5

#spider_main
#?coding:?utf-8?

from?baike_python?import?html_downloader,?url_manager,?html_parser,?html_outputer

class?SpiderMain(object):
????def?__init__(self):#構造方法
????????self.urls?=?url_manager.UrlManager()#初始化URL管理器
????????self.downloader?=?html_downloader.HtmlDownloader()#初始化網頁下載器
????????self.parser?=?html_parser.HtmlParser()#初始化網頁解析器
????????self.outputer?=?html_outputer.HtmlOutputer()#初始化輸出器
????????????
????def?craw(self,?root_url):#開始執行爬蟲的方法
????????count?=?1#計數器，計數爬取頁面的總數量
????????count2?=?0#計數器，計數爬取失敗的網頁個數
????????self.urls.add_new_url(root_url)#傳入網頁入口
????????while?self.urls.has_new_url():#對網頁內包括的連接網頁循環抓取，先判斷URL管理器不空
????????????try:#有些頁面可能失效了，要有異常處理
????????????????new_url?=?self.urls.get_new_url()#獲取URL管理器中的一個URL
????????????????print?("craw?%d?:?%s?\n"%(count,new_url)#打印當前爬去的頁面
????????????????html_cont?=?self.downloader.download(new_url)
????????????????new_urls,?new_data?=?self.parser.parse(new_url,html_cont)#把頁面解析為新連接和網頁數據兩部分，其中new_data?中含有當頁的鏈接、title和summary，new_url是當前頁面中的所有鏈接的集合
????????????????print?(new_data["title"]+"\n",?new_data["summary"])??
????????????????self.urls.add_new_urls(new_urls)#新鏈接存入URL管理器
????????????????self.outputer.collect_data(new_data)#網頁數據收集
????????????????if?count?==?10:#控制打印頁面的數量
????????????????????break
????????????????count?=?count+1???
????????????except?Exception,e:
????????????????count2?=?count2+1
????????????????print('e')
????????????????print?("craw?failed")
????????
????????self.outputer.output_html()
????????print?(str(count-count2)+"?successful,","?while?"+str(count2)+"?failed?")

if?__name__=="__main__":?#主函數
????root_url?=?"http://baike.baidu.com/view/21087.htm"?#入口頁
????obj_spider?=?SpiderMain()#創建對象
????obj_spider.craw(root_url)#啟動爬蟲

#html_downloader

import?urllib

class?HtmlDownloader(object):
????
????
????def?download(self,url):
????????if?url?is?None:
????????????return?None?
????????#request?=?urllib2.Request(url)
????????response?=?urllib.request.urlopen(url)
????????
????????
????????if?response.getcode()?!=?200:
????????????return?None?
????????return?response.read().decode('utf-8')

感謝查看，找bug真的煩= =