这里的请求发送不出去
到这就停了
有没有大佬帮忙看看啊,不胜感激
# -*- coding: utf-8 -*-
import scrapy,json
from ..items import goodsItem
class JdSpiderSpider(scrapy.Spider):
name = 'jd_spider'
allowed_domains = ['jd.com']
start_urls = ['https://search.jd.com/Search?keyword=%E7%AC%94%E8%AE%B0%E6%9C%AC']
def parse(self, response):
goods = response.xpath(r'//li[@data-sku]')
for good in goods:
item1 = goodsItem()
item1['ID'] = good.xpath(r'./@data-sku').get()
item1['name'] = good.xpath('.//div[@class="p-name p-name-type-2"]//em/text()').getall()
item1['name'] = ' '.join(item1['name'])
item1['shop_name'] = good.xpath('.//a[@class="curr-shop hd-shopname"]/@title').get()
item1['link'] = good.xpath('.//div[@class="p-img"]/a/@href').get()
item1['shop_id'] = good.xpath('.//div/@data-venid').get()
# url = 'http:' + item1['link']
url = 'https://club.jd.com/comment/productCommentSummaries.action?referenceIds=' + str(item1['ID'])
yield scrapy.Request(url,meta={'item':item1},callback=self.parse_getCommentnum)# 请求评论文件
def parse_getCommentnum(self,response):
item1 = response.meta['item']
js = json.loads(response.text)
item1['GoodCount'] = js['CommentsCount'][0]['GoodCount']
print(item1['GoodCount'])
print('='*30)
item1['GeneralCount'] = js['CommentsCount'][0]['GeneralCount']
item1['PoorCount'] = js['CommentsCount'][0]['PoorCount']
num = item1['ID']
shop_id = item1['shop_id']
# https://c0.3.cn/stock?skuId=100004563443&area=13_1007_37916_0&venderId=1000000904&cat=670,671,672
url = 'https://c0.3.cn/stock?skuId='+str(num)+'&area=13_1007_37916_0&venderId='+str(shop_id)+'&cat=670,671,672'
print(url)
print('='*30)
# https://c0.3.cn/stock?skuId=100010816812&area=13_1007_37916_0&venderId=1000000904&buyNum=1&choseSuitSkuIds=&cat=670,671,672&fqsp=0&pdpin=&pduid=1919946493&ch=1
# https://c0.3.cn/stock?skuId=100005171461&area=13_1007_37916_0&venderId=1000000157&buyNum=1&choseSuitSkuIds=&cat=670,671,672&extraParam={%22originid%22:%221%22}&fqsp=0&pdpin=&pduid=1919946493&ch=1&callback=jQuery3127510
yield scrapy.Request(url, meta={'item':item1}, callback=self.parse_price)# 请求价格文件,为啥发不出去请求啊
def parse_price(self,response): # 这里运行不了啊
item1 = response.meta['item']
js = json.loads(response.text)
item1['price'] = js['stock']['jdPrice']['p']
print(item1)
print('*'*50)
return item1