今天测试自己写的模块,在大部分地方成功运行,可是到了壁纸的网站就失败了
url='https://wallhaven.cc/toplist?page= 2‘
模块主程序:
import requests
from lxml import etree
import os
import time
def coffee_dou_requests(m,u,i,p,j):
start = time.time()
for mima in range(5):
if m == 'qazplm75124':
break
else:
print("密码错误")
exit('')
for c in ['1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12', '13', '14', '15', '16', '17', '18', '19',
'20', '21', '22', '23', '24', '25', '26', '27', '28', '29', '30', '31', '32', '33']:
url = u
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.71 Safari/537.36'
}
b = '.jpg'
a = int(0)
response = requests.get(url=url, headers=headers)
page_text = response.text
tree = etree.HTML(page_text)
li_list = tree.xpath(i)
if not os.path.exists(f'./{p}'):
os.mkdir(f'./{p}')
for li in li_list:
a = a+1
u = li.xpath(f'{j}')[0]
img_name = (f'{a}+{b}')
# img_name = img_name.encode('iso-8859-1').decode('gbk')
img_data = requests.get(url=u, headers=headers).content
img_path = f'{p}/' + img_name
with open(img_path, 'wb') as fp:
fp.write(img_data)
print(img_name, u, '爬取完毕!!')
end = time.time()
print('总耗时:', end - start)
print("爬取结束!!!")
return a
#u为url
#i为宏观位置
#p为文件名称
#j为具体位置
程序部分:
from coffee_dou_requests import coffee_dou_requests
#import asyncio
#u=input("请输入url或url格式:")
#i=input("请输入宏观位置(类型为xpath):")
#p=input("请输入文件名称:")
#j=input("请输入爬取内容具体位置(类型为xpath):")
m=input("请输密码:")
u = 'https://wallhaven.cc/toplist?' + 'page=' + "2"
i = '//*[@id="thumbs"]/section[1]/ul/li//a/@href'
p = "壁纸"
j = '//img[@id="wallpaper"]/@src'
u = coffee_dou_requests(m, u, i, p, j)
print(u)
#密码为:qazplm75124
报错为:
Traceback (most recent call last):
File "C:/Users/16662/Desktop/555.py", line 15, in <module>
u = coffee_dou_requests(m, u, i, p, j)
File "C:\Users\16662\Desktop\coffee_dou_requests.py", line 30, in coffee_dou_requests
u = li.xpath(f'{j}')[0]
AttributeError: 'lxml.etree._ElementUnicodeResult' object has no attribute 'xpath'
在测试其他的网站时,程序成功运行,可到了这个就报错
相关模块代码禁止转载,如需,请联系作者
这个网站可能是动态传输,所以爬不到数据