爬菜价,能爬到网页但是tr爬不到显示0条,不知道哪里不对
url = http://www.xinfadi.com.cn/priceDetail.html
import requests
from lxml import etree
from concurrent.futures import ThreadPoolExecutor
import csv
f = open("菜价.csv",mode="w",encoding="utf-8")
csvwriter = csv.writer(f)
def download_one_page(url):
resp = requests.get(url) #拿到源代码
resp.encoding = "utf-8"
# print(resp.text)
html = etree.HTML(resp.text)
table = html.xpath("/html/body/div[2]/div/div/div/div[4]/div[1]/div/table")[0]
# print(table)
trs = table.xpath("./tr")[1:]
print(len(trs))
limit后面数字你自己改,表示你想一次爬多少条数据
url='http://www.xinfadi.com.cn/getPriceData.html?&limit=2000'
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36'
}
req = requests.post(url=url, headers=headers).json()
data=pd.DataFrame(req['list'])
data.to_csv('e:/菜价.csv',encoding='gbk')