"""
Python爬取豆瓣读书全部信息
"""
# -*- coding: utf-8 -*-
# @Author: yqbao
# @GiteeURL: https://gitee.com/yqbao
# @Date: 2019/9/11 23:14
# @Version: v.0.0

import redis
import requests
from pyquery import PyQuery as Pq
import json


def query():
    """获取数据"""
    url = 'http://webapi.http.zhimacangku.com/getip?num=1&type=2&pro=0&city=0&yys=0&port=1&pack=64167&ts=0&ys=0&cs=0&lb=1&sb=0&pb=45&mr=1&regions='
    req = requests.get(url=url)
    response = req.content.decode('utf-8')
    data = json.loads(response)
    ip_port = str(data['data'][0]['ip']) + ':' + str(data['data'][0]['port'])
    print('代理IP：', ip_port)
    proxies = {
        'http': 'http://' + ip_port,
        'https': 'https://' + ip_port,
    }
    tag_url = 'https://book.douban.com/tag/?view=type&icn=index-sorttags-all'
    headers = {
        'Host': 'book.douban.com',
        'Upgrade-Insecure-Requests': '1',
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36',
    }
    try:
        req = requests.get(url=tag_url, headers=headers, proxies=proxies)  # 封装请求头并请求网页
        if req.status_code == 200:  # 成功，则获取响应的html内容，并返回
            response = req.content.decode('utf-8')
            return response
        else:
            return None
    except requests.exceptions.RequestException:
        return None


def analysis(content):
    """解析数据"""
    base_url = 'https://book.douban.com'
    doc = Pq(content)  # 初始化PyQuery
    items = doc('.tagCol tbody td a')
    for item in items.items():  # 在信息列表中解析需要的数据
        tags_url = base_url + item.attr('href')
        print('书籍标签url：', tags_url)
        yield tags_url


def main():
    """调度函数"""
    data = query()
    link = redis.Redis(host='localhost', port=6379, db=0)
    for item in analysis(data):
        link.lpush('book:tags_url', item)


if __name__ == '__main__':
    main()