主程序：from baidubaike import url_manger, html_download, html_parser, html_outputerclass SpiderMain(object):? ? def __init__(self):? ? ? ? self.urls = url_manger.UrlManager()? ? ? ? self.downloader = html_download.HtmlDownloader()? ? ? ? self.parser = html_parser.HtmlParser()? ? ? ? self.outputer = html_outputer.HtmlOutputer? ?? ? def craw(self,root_url):? ? ? ? count = 1? ? ? ? self.urls.add_new_url(root_url)? ? ? ? while self.urls.has_new_url():? ? ? ? ? ? try:? ? ? ? ? ? ? ? new_url = self.urls.get_new_url()? ? ? ? ? ? ? ? print('craw %d:%s'%(count,new_url))? ? ? ? ? ? ? ? html_cont = self.downloader.download(new_url)? ? ? ? ? ? ? ? new_urls,new_data = self.parser.parse(new_url,html_cont)? ? ? ? ? ? ? ? self.urls.add_new_urls(new_urls)? ? ? ? ? ? ? ? self.outputer.collect_data(new_data)? ? ? ? ? ? ? ??? ? ? ? ? ? ? ? if count == 1000:? ? ? ? ? ? ? ? ? ? break? ? ? ? ? ? ? ? count +=1? ? ? ? ? ? except:? ? ? ? ? ? ? ? print('craw failed')? ? ? ? self.outputer.output_html()? ??? ??if __name__ =="main":? ? root_url = "http://baike.baidu.com/view/21087.htm"? ? obj_spider = SpiderMain()? ? obj_spider.craw(root_url)parser程序：from bs4 import BeautifulSoupimport refrom urllib.parse import urljoinclass HtmlParser(object):? ?? ?def _get_new_urls(self, page_url, soup):? ? ? ?res_data = {}? ? ? ?? ? ? ?res_data['url'] = page_url? ? ? ?#<dd class="lemmaWgt-lemmaTitle-title"><h1>Python</h1>? ? ? ?title_node = soup.find('dd',class_="lemmaWgt-lemmaTitle-title").find("h1")? ? ? ?res_data['title'] = title_node.get_text()? ? ? ?#<div class="lemma-summary" label-module="lemmaSummary">? ? ? ?summary_node = soup.find('div',class_="lemma-summary")? ? ? ?res_data['summary'] = summary_node.get_text()? ? ? ?return res_data? ?? ?def _get_new_data(self, page_url, soup):? ? ? ?new_urls = set()? ? ? ?links = soup.find_all('a',re.compile(r'/view/\d+\.htm'))? ? ? ?for link in links:? ? ? ? ? ?new_url = link['href']? ? ? ? ? ?new_full_url = urljoin(page_url,new_url)? ? ? ? ? ?new_urls.add(new_full_url)? ? ? ?return new_urls? ?def parse(self,page_url,html_cont):? ? ? ?if page_url is None or html_cont is None:? ? ? ? ? ?return? ? ? ? ??? ? ? ?soup = BeautifulSoup(html_cont,'html.parser',from_encoding = 'utf-8')? ? ? ?new_urls = self._get_new_urls(page_url,soup)? ? ? ?new_data = self._get_new_data(page_url,soup)? ? ? ?return new_urls,new_dataoutputer程序：class HtmlOutputer(object):? ?def __init__(self):? ? ? ?self.datas = []? ? ? ?? ?def collect_data(self,data):? ? ? ?if data is None:? ? ? ? ? ?return? ? ? ?self.datas.append(data)? ? ??? ?def output_html(self):? ? ? ?fout = open('output.html','w')? ? ? ?? ? ? ?fout.write('<html>')? ? ? ?fout.write('<body>')? ? ? ?fout.write('<table>')? ? ? ?for data in self.datas:? ? ? ? ? ?fout.write('<tr>')? ? ? ? ? ?fout.write('<td>%s</td>'% data['url'].encode('utf-8'))? ? ? ? ? ?fout.write('<td>%s</td>'% data['title'].encode('utf-8'))? ? ? ? ? ?fout.write('<td>%s</td>'% data['summary'].encode('utf-8'))? ? ? ?fout.write('</table>')? ? ? ?fout.write('</body>')? ? ? ?fout.write('</html>')
添加回答
舉報
0/150
提交
取消