from?bs4?import?BeautifulSoup
import?re
from?urllib.parse?import?urlparse

class?HtmlParser(object):
????def?_get_new_urls(self,?url,?soup):
????????new_urls?=?set()
????????links?=?soup.find_all('a',href=re.compile(r"item/\d"))
????????for?link?in?links:
????????????new_url?=?link['href']
????????????new_full_url?=?urlparse.urljoin(url,new_url)
????????????new_urls.add(new_full_url)
????????return?new_urls

????def?_get_new_data(self,?url,?soup):
????????res_data?=?{}

????????res_data['url']?=?url

????????title_node?=?soup.find('dd',class_="lemmaWgt-lemmaTitle-title").find('h1')
????????res_data['title']?=?title_node.get_text()

????????summary_node?=?soup.find('dev',class_="lemma-summary")
????????res_data['summary']?=?summary_node.get_text()

????????return?res_data


????def?parse(self,?url,?html_cont):
????????if?url?is?None?or?html_cont?is?None:
????????????return
????????soup?=?BeautifulSoup(html_cont,'html.parser')
????????new_urls?=?self._get_new_urls(url,soup)
????????new_data?=?self._get_new_data(url,soup)
????????return?new_urls,new_data