爬取去哪网的景点
# -*- codeing = utf-8 -*-
from bs4 import BeautifulSoup # 网页解析,获取数据
import re # 正则表达式,进行文字匹配`
import urllib.request, urllib.error # 制定URL,获取网页数据
import xlwt # 进行excel操作
findTitle = re.compile(r'<span class="cn_tit">(.*?)</span>')
findRating = re.compile(r'<span class="ranking_sum">(.*?)</span>')
findInq = re.compile(r'<div class="desbox">(.*?)</div>')
def main():
baseurl = "https://travel.qunar.com/p-cs299914-beijing-jingdian-1-" # 要爬取的网页链接
datalist = getData(baseurl)
savepath = "北京景点.xls" # 当前目录新建XLS,存储进去
saveData(datalist, savepath)
# 爬取网页
def getData(baseurl):
datalist = [] # 用来存储爬取的网页信息
for i in range(0, 20): # 调用获取页面信息的函数,10次
url = baseurl + str(i + 1)
html = askURL(url)
soup = BeautifulSoup(html, "html.parser")
for item in soup.find_all('li', class_="item"): # 查找符合要求的字符串
data = [] # 保存一部电影所有信息
item = str(item)
titles = re.findall(findTitle, item)
data.append(titles)
rating = re.findall(findRating, item)
data.append(rating)
inq = re.findall(findInq, item)
data.append(inq)
datalist.append(data)
return datalist
# 得到指定一个URL的网页内容
def askURL(url):
head = { # 模拟浏览器头部信息,向豆瓣服务器发送消息
"User-Agent": "Mozilla / 5.0(Windows NT 10.0; Win64; x64) AppleWebKit / 537.36(KHTML, like Gecko) Chrome / 80.0.3987.122 Safari / 537.36"
}
request = urllib.request.Request(url, headers=head)
html = ""
try:
response = urllib.request.urlopen(request)
html = response.read().decode("utf-8")
except urllib.error.URLError as e:
if hasattr(e, "code"):
print(e.code)
if hasattr(e, "reason"):
print(e.reason)
return html
# 保存数据到表格
def saveData(datalist, savepath):
print("save.......")
book = xlwt.Workbook(encoding="utf-8", style_compression=0) # 创建workbook对象
sheet = book.add_sheet('北京景点', cell_overwrite_ok=True) # 创建工作表
col = ("景点名称", "排名", "介绍")
for i in range(0, 3):
sheet.write(0, i, col[i]) # 列名
for i in range(0, 200):
# print("第%d条" %(i+1)) #输出语句,用来测试
data = datalist[i]
for j in range(0, 3):
sheet.write(i + 1, j, data[j]) # 数据
book.save(savepath) # 保存
if __name__ == "__main__": # 当程序执行时
main()
print("爬取完毕!")
爬取出的数据老是会有多出的标签
还有为什么爬出的数据不是200个?