突然就提取不到数据。。不知道怎么回事,不用线程池还能正常提取数据,不过就是很慢罢了
import csv
from concurrent.futures import ThreadPoolExecutor
import requests
result = []
f = open('动作片1w2.csv', mode="w", encoding="ANSI", newline="")
csvwriter = csv.writer(f)
def download_page(url):
headers = {
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/97.0.4692.99 Safari/537.36 Edg/97.0.1072.69"
}
resp = requests.get(url=url, headers=headers,)
resp.encoding = 'utf-8'
dic = resp.json()
for item in dic['data']:
name = item['title']
rate = item['rate']
people = item['casts']
resp.close()
one_result = [name, rate, people]
result.append(one_result)
csvwriter.writerow([name, rate, people])
if __name__ == "__main__":
with ThreadPoolExecutor(50) as t:
for i in range(0, 801, 20):
t.submit(download_page,
f"https://movie.douban.com/j/new_search_subjects?sort=U&range=0,100&tags=电影&start={i}&genres=动作")
print("over")
f.close()
我debug了一下,这个程序值运行了最后的if__name__=='main':后面的内容,没有运行我定义的函数,不知道为啥