通过正则表达式找到当前页面中的所有URL，储存在set中（剔除重复），用类似图数据结构的深度优先遍历算法遍历set，实现全站爬虫。
from urllib import request
from bs4 import BeautifulSoup as bs
import re
import time
url = “http://xxxxx.jinan.cn/”
visited = set()
def get_local_pages(url):
try:
time.sleep(1)
web = request.urlopen(url=url)
except:
print(“Open url”,url, “failed error!”)
return
soup = bs(web.read(), ‘html.parser’)
tags = soup.find_all(name=”a”) #a标签储存URL，找到所有a标签
r = re.compile(r’href=”/.+?\.html”‘) #使用非贪婪模式
pages_temp = set()
pages = set()
for tag in tags:
tag = str(tag)
urls = r.findall(tag)
if urls:
pages_temp.add(urls[0])
for page in pages_temp:
if page[7:11] != “jnyzh”:
page = “http://xxxxx.jinan.cn/” + page[7:-1]
pages.add(page)
print(pages)
return pages
def dfs(pages): #使用类似图的深度优先遍历方法递归遍历URL
global visited
if pages is set():
return
for page in pages:
if page not in visited:
print(“Visiting”,page)
visited.add(page)
url = page
print(“A”)
pages = get_local_pages(url)
dfs(pages)
pages = get_local_pages(url)
dfs(pages)
————————————————
版权声明：本文为CSDN博主「河北一帆」的原创文章，遵循 CC 4.0 BY-SA 版权协议，转载请附上原文出处链接及本声明。
原文链接：https://blog.csdn.net/qq_41816368/article/details/81120998