selenium只能爬取只能重复第一页数据
from selenium import webdriver
import json
import time, random
import re
driver = webdriver.Chrome()
url = ''
driver.get(url)
def parse_data():
response = driver.page_source
json_str = re.findall('g_page_config = (.*);', response)[0]
json_dict = json.loads(json_str)
print(json_dict)
for page in range(0, 11):
print(f'---正在打印第{page}页---')
parse_data()
driver.find_element(By.XPATH, '//*[text()="下一页"]').click()
time.sleep(random.randint(3, 8))
打印出来只有第一页数据, 跳转下一页是正常的
首先检查一下response会不会发生变化