B站链接发出来看看
SPIDER_MIDDLEWARES = {
'scrapy_splash.SplashDeduplicateArgsMiddleware':100
}
DOWNLOADER_MIDDLEWARES = {
'scrapy_splash.SplashCookiesMiddleware': 723,
'scrapy_splash.SplashMiddleware': 725,
'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 810,
# 'BL.middlewares.BlDownloaderMiddleware': 812, # 爬虫的下载中间件
}
DUPEFILTER_CLASS = 'scrapy_splash.SplashAwareDupeFilter'
HTTPCACHE_STORAGE = 'scrapy_splash.SplashAwareFSCacheStorage'
SPLASH_URL = 'http://127.0.0.1:8050'
USER_AGENT_LIST = [
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/537.13+ (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/534.55.3 (KHTML, like Gecko) Version/5.1.3 Safari/534.53.10",
"Mozilla/5.0 (iPad; CPU OS 5_1 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko ) Version/5.1 Mobile/9B176 Safari/7534.48.3",
"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; de-at) AppleWebKit/533.21.1 (KHTML, like Gecko) Version/5.0.5 Safari/533.21.1",
"Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_7; da-dk) AppleWebKit/533.21.1 (KHTML, like Gecko) Version/5.0.5 Safari/533.21.1",
"Mozilla/5.0 (Windows; U; Windows NT 6.1; tr-TR) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27",
"Mozilla/5.0 (Windows; U; Windows NT 6.1; ko-KR) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27",
"Mozilla/5.0 (Windows; U; Windows NT 6.1; fr-FR) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27",
"Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27",
"Mozilla/5.0 (Windows; U; Windows NT 6.1; cs-CZ) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27",
"Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27",
]
PROXY_LIST = [
{'ip_port': 'http://116.8.108.53:16816', 'user_passwd': '***:***'},
{'ip_port': 'http://39.99.158.153:16817', 'user_passwd': '***:***'},
# {'ip_port': 'http://114.117.164.198:16817', 'user_passwd': ***:***'},
# {'ip_port':'http://220.249.149. 52:9999'},
# {'ip_port':'http://175.42.129.2 15:9999'},
# {'ip_port':'http://175.42.129.1 20:9999'},
# {'ip_port':'http://36.250.156.135:9999'},
# {'ip_port':'http://163.204.242.238:9999'},
# {'ip_port':'http://60.174.190.15:9999'},
# {'ip_port':'http://117.95.198.132:9999'},
# {'ip_port':'http://114.239.151.229:9999'},
# {'ip_port':'http://36.249.48.47:9999'},
# {'ip_port':'http://175.44.109.205:9999'},
# {'ip_port':'http://58.22.177.224:9999'},
# {'ip_port':'http://60.169.133.225:9999'},
# {'ip_port':'http://1.198.42.177:9999'},
# {'ip_port':'http://49.70.94.154:9999'},
# {'ip_port':'http://42.238.87.240:9999'},
]
import base64
import random
from BL.settings import USER_AGENT_LIST, PROXY_LIST
class BlSpider(scrapy.Spider):
name = 'bl'
allowed_domains = ['目标网站.com']
# 进入主页
start_urls = ['目标网站']
def start_requests(self):
# for url in self.start_urls:
proxy = random.choice(PROXY_LIST)
print('使用的IP是', proxy)
b64_up = base64.b64encode(proxy['user_passwd'].encode())
# 设置认证
# Basic 有一个空格
ProxyAuthorization = 'Basic ' + b64_up.decode()
# 设置代理
yield SplashRequest(self.start_urls[0],
callback=self.parse_splash,
args={'wait': 5}, # 最大超时时间
endpoint='render.html', # 使用splash服务的固定参数
dont_process_response=False,
splash_headers={
"referer": 'https://www.bilibili.com/v/technology/science#/',
"Proxy-Authorization": ProxyAuthorization
},
dont_send_headers=False,
magic_response=True,
session_id='default',
http_status_from_error_code=True,
cache_args=None,
meta=None,
)
安装的时候把杀毒软件退出,有时候杀毒软件会拦截安装
Visendo SmtpExtender这个东东是搭建邮件服务器安装的一个软件。如果你的操作和up主的一样,但是你的显示指定的服务未安装错误,那么很大可能就是你的这个SmtpExtender没有安装好,可能是被你的杀毒软件误杀了导致某些程序软件被删除。建议你重新安装该软件,使用修复模式安装。或者完全重新卸载掉再安装,安装的时候关闭掉杀毒软件,同时右键安装包,以管理员身份运行安装看看。应该就是这个原因
好好看看他的视频 在你跟着他做的前提 他是不是安装了什么东西
你这么说很笼统