def getHTMLText(url_):
try:
r = requests.get(url_, timeout=30)
r.raise_for_status() # 如果状态不是200,引发HTTPError异常
r.encoding = r.apparent_encoding
return r.text
except:
return "产生异常"
def getSoup(url_):
html_ = getHTMLText(url_)
soup_ = BeautifulSoup(html_, 'lxml')
return soup_
if __name__ == "__main__":
year = '23'
month = '01'
day = '04'
page = '01'
for page in range(1, 9): # 注释掉本行则可正常运行
target = f"http://paper.people.com.cn/rmrb/html/20{year}-{month}/{day}/nbs.D110000renmrb_{page}.htm"
soup = getSoup(target)
title_list = soup.find('ul',class_='news-list')
title_list = title_list.find_all('a')
for title in title_list:
print(title.string)
运行结果及详细报错内容Traceback (most recent call last):
File "C:\Users\songz\PycharmProjects\pythonProject1\spiders\spiderNews.py", line 47, in
title_list = title_list.find_all('a')
AttributeError: 'NoneType' object has no attribute 'find_all'
01.html和1.html很显然不是同一个网址,你在错误的网址下当然抓不到东西,是空的
改成
target = f"http://paper.people.com.cn/rmrb/html/20{year}-{month}/{day}/nbs.D110000renmrb_{page:02d}.htm"
你的page是数字1,网页的是01,要这样写
url=f"http://paper.people.com.cn/rmrb/html/20{year}-{month}/{day}/nbs.D110000renmrb_0{page}.htm"