#coding=utf-8

import urllib

from bs4 import BeautifulSoup

import re

import pymysql.cursors

resp = urllib.urlopen('https://en.wikipedia.org/wiki/Main_Page').read().decode('utf-8')

soup = BeautifulSoup(resp, "html.parser")

# 獲取所有以/wiki/開頭的a標簽的href屬性

listUrls = soup.findAll("a", href=re.compile("^/wiki/"))

# 輸出所有的詞條對應的名稱和URL

for url in listUrls:

# 過濾以。jpg或JPG結尾的url

if not re.search("\.(jpg|JPG)$", url['href']):

print url.get_text(), '<======>','https://en.wikipedia.org'+url['href']

# 獲取數據庫連接

connection = pymysql.connect(host = 'localhost',

user = 'root',

password = '6322004',

db = 'wikiurl',

charset = 'utf8mb4')

try:

# 獲取會話指針

with connection.cursor() as cursor:

sql = "insert into `urls`(`urlname`, `urlhref`) values(%s, %s)"

# 執行sql語句

cursor.execute(sql,(url.get_text(),'https://en.wikipedia.org'+url['href']))

connection.commit()

finally:

connection.close()