"""
将爬取的数据写入ElasticSearch搜索引擎
"""
import json
import random
import re
import time

import bs4
import requests


def handle_white_space(content):
    return re.sub(r'\n|\s{2,}', ' ', content.strip())


counter = 0
for page in range(1, 11):
    resp = requests.get(
        url=f'https://movie.douban.com/top250?start={(page - 1) * 25}',
        headers={
            'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_6) '
                          'AppleWebKit/537.36 (KHTML, like Gecko) '
                          'Chrome/83.0.4103.97 Safari/537.36',
            'Accept': 'text/html,application/xhtml+xml,application/xml;'
                      'q=0.9,image/webp,image/apng,*/*;'
                      'q=0.8,application/signed-exchange;v=b3;q=0.9',
            'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
        },
    )
    soup = bs4.BeautifulSoup(resp.text, features='lxml')
    elements = soup.select('.info')
    for element in elements:
        counter += 1
        title = element.select_one('.title').text
        info = element.select_one('.bd>p:first-child').text
        info = handle_white_space(info)
        star = element.select_one('.rating_num').text
        quote = element.select_one('.inq').text
        resp = requests.post(
            url=f'http://47.104.31.138:9200/douban/movie/{counter}/',
            headers={
                'content-type': 'application/json'
            },
            data=json.dumps({
                'title': title,
                'info': info,
                'star': star,
                'quote': quote
            })
        )
    # 随机休眠一段时间避免爬取过于频繁
    time.sleep(random.random() * 5)