import requests
import logging
import re
from urllib.parse import urljoin
logging.basicConfig(level=logging.INFO, format ='%(asctime)s - %(levelname)s: %(message)s')
BASE_URL='https://ssr1.scrape.center'
TOTAL_PAGE = 10
def scrape_page(url):
logging.info('scraping %s...', url)
try:
response=requests.get(url)
if response.status_code == 200:
return response.text
logging.error('get invalid status code %s while scraping %s',response.status_code,url)
except requests.RequestException:
logging.error('error occurred while scraping %s',url,exc_info=True)
def scrape_index(page):
index_url = f'{BASE_URL}/page/{page}'
return scrape_page(index_url)
def parse_index(html):
pattern = re.compile('<a.*?href="(.*?)".*?class="name">')
items = re.findall(pattern,html)
if not items:
return []
for item in items:
detail_url = urljoin(BASE_URL,item)
logging.info('get detail url %s',detail_url)
yield detail_url
def scrape_detail(url):
return scrape_page(url)
# noinspection PyTypeChecker
def parse_detail(html):
cover_pattern = re.compile('class="item.*?<img.*?scr="(.*?)".*?class="cover"',re.S)
name_pattern = re.compile('<h2.*?>(.*?)</h2>')
categories_pattern = re.compile('<button.*?category.*?<span>(.*?)</span>.*?</botton>',re.S)
published_at_pattern = re.compile('(\d{4}-\d{2}-\d{2})\s?上映')
drama_pattern = re.compile('<div.*?drama.*?>.*?<P*?>(>*?)</p>',re.S)
score_pattern = ('<p.*?score.*?>(.*?)</p>',re.S)
cover = re.compile(cover_pattern,html).group(1).strip() if re.search(cover_pattern,html) else None
name = re.search(name_pattern,html).group(1).strip() if re.search(name_pattern,html) else None
categories = re.findall(categories_pattern,html) if re.findall(categories_pattern,html) else []
published_at = re.search(published_at_pattern,html).group(1) if re.search(published_at_pattern,html) else None
drama = re.search(drama_pattern,html).group(1).strip() if re.search(drama_pattern,html) else None
score = float(re.search(score_pattern,html).group(1).strip()) if re.search(score_pattern,html) else None
return{
'cover':cover,
'name':name,
'categories':categories,
'published_at':published_at,
'drama':drama,
'score':score
}
def main():
"""global detail_urls"""
for page in range (1, TOTAL_PAGE + 1):
index_html = scrape_index(page)
detail_urls = parse_index(index_html)
for detail_url in detail_urls:
detail_html = scrape_detail(detail_url)
data = parse_detail(detail_html)
logging.info('get detail data %s ',data)
if __name__ == '__main__':
main ()
报错
```python
Traceback (most recent call last):
File "C:\Users\86137\PycharmProjects\pythonProject\main.py", line 70, in <module>
main ()
File "C:\Users\86137\PycharmProjects\pythonProject\main.py", line 66, in main
data = parse_detail(detail_html)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\86137\PycharmProjects\pythonProject\main.py", line 49, in parse_detail
score = float(re.search(score_pattern,html).group(1).strip()) if re.search(score_pattern,html) else None
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\86137\AppData\Local\Programs\Python\Python311\Lib\re\__init__.py", line 176, in search
return _compile(pattern, flags).search(string)
^^^^^^^^^^^^^^^^^^^^^^^^
File "C:\Users\86137\AppData\Local\Programs\Python\Python311\Lib\re\__init__.py", line 286, in _compile
raise TypeError("first argument must be string or compiled pattern")
TypeError: first argument must be string or compiled pattern
应该如何修改参数类型,使程序可以正常运行
截图的这一行也就是43行改成:score_pattern =re.compile ('<p.?score.*?>(.?)
',re.S)有帮助的话,请点采纳该答案~
【以下回答由 GPT 生成】
抱歉,但是在您的问题中似乎没有给出具体的爬虫报错信息和数据类型错误的代码行号。无法根据这些信息提供具体的解决方案。请提供更多细节和错误信息,我会尽力帮助您解决问题。
【相关推荐】