import ast
import os

import requests
from requests.utils import add_dict_to_cookiejar


class SetSpiderConfiguration(object):
    def __init__(self):
        self.session = requests.session()
        self.session.headers = {
            'Accept': 'application/json, text/plain, */*',
            'Accept-Language': 'zh-CN,zh;q=0.9',
            'Authorization': 'token 1f3e43d23fa1990ffa1890683ab4601ddda299b8',
            'Connection': 'keep-alive',
            'Referer': 'http://192.168.2.107:8000/spider_data/spider_config',
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36',
        }
        add_dict_to_cookiejar(self.session.cookies, {
            'csrftoken': '2HErczWBgLGv0Bq9GYlxkDu0fbAgw1bVFyoRm83tGWAP2MQbnkcFAUzHoxeX1JLE',
            'kyls_token': '1f3e43d23fa1990ffa1890683ab4601ddda299b8',
        })

    def get_all_dict(self, num):
        '''
        获取num个数的dict列表
        '''
        url = 'http://192.168.2.107:8000/api/spider_data_config/'
        params = (
            ('offset', '0'),
            ('limit', str(num)),
            ('search', ''),
            ('ordering', '-run_time'),
            ('site_admin', ''),
            ('run_status', ''),
            ('script_status', ''),
            ('province', ''),
            ('site_type', ''),
        )
        return self.session.get(url=url, params=params).json()['results']

    def get_id_dict(self, id):
        '''
        根据id单个返回dict
        '''
        url = 'http://192.168.2.107:8000/api/spider_data_config/'
        params = (
            ('offset', '0'),
            ('limit', '50'),
            ('search', str(id)),
            ('ordering', '-run_time'),
            ('site_admin', ''),
            ('run_status', ''),
            ('script_status', ''),
            ('province', ''),
            ('site_type', ''),
        )
        return self.session.get(url=url, params=params).json()['results'][0]

    def get_id_script_config(self,id):
        '''
        输入id
        返回代码信息
        '''
        return self.get_id_dict(id)['script_config']

    def upload(self, data):
        '''
        上传数据
        '''
        response = self.session.put(url=f'http://192.168.2.107:8000/api/spider_data_config/{data["id"]}/', json=data,
                                    verify=False)
        return response.status_code

    def loading_file(self, path):
        '''
        输入path地址
        返回
        '''
        with open(path, mode='r', encoding='utf-8') as f:
            return ast.literal_eval(f.read())

    def folder_read(self,path):
        '''
        文件夹数据读取读取
        '''
        file_list = []
        for root, dirs, files in os.walk(path, topdown=True):
            for li in files:
                file_list.append(root+'\\'+li)
        return file_list

    def check_script_name(self, name='', num=9000):
        '''
        检测查看脚本名称对应的数量
        '''
        data_list = []
        for data in self.get_all_dict(num=num):
            if name == data['script_name'] and data['script_status'] == '成功':
                data_list.append(data)
        return data_list

    def main(self, num, script_name='', file_list=None):
        if file_list is None:
            file_list = [{
                # 列表页
                'site_path_url': 'https://zwgk.hefei.gov.cn/public/column/13751?type=4&catId=7003771&action=list',
                # 网站名称
                'site_name': '蜀山区人民政府',
                # 网站类型
                'title_type': '国家部委',
                # 网站目录
                'site_path_name': '首页 > 信息公开 > 蜀山区人民政府（政府办公室） > 重大建设项目批准和实施 > 批准结果信息',
                ###############################################
                # 网站列表数据xpath/css
                'ListQuery': '.clearfix.xxgk_nav_list li',
                # 网站的url 数据 包括 title_name 和 title_url
                "title_url_query": "a",
                # 网站的url 数据 包括 title_name 和 title_url
                "title_name_query": "a",
                # 网站列表下面的date 数据
                'title_date_query': 'span',
                # 详情页的css选择器
                'content_selector': '.gk_container.gkwz_container',
                # 下一页 css选择器
                'next_page': '#page_public_info > a:nth-child(8)',
                # site_id
                'site_id': '5F63ED782E',
                'stop_time': '3'
            },
                {
                    # 列表页
                    'site_path_url': 'http://www.wlthq.gov.cn/sites/main/textlist.jsp?ColumnID=398&SiteID=main',
                    # 网站名称
                    'site_name': '乌拉特后旗人民政府',
                    # 网站类型
                    'title_type': '国家部委',
                    # 网站目录
                    'site_path_name': '首页>政务公开>执行公开>环境保护',
                    ###############################################
                    # 网站列表数据xpath/css
                    'ListQuery': '.text-list a',
                    # 网站的url 数据 包括 title_name 和 title_url
                    "title_url_query": "a",
                    # 网站的url 数据 包括 title_name 和 title_url
                    "title_name_query": "a",
                    # 网站列表下面的date 数据
                    'title_date_query': '.rt',
                    # 详情页的css选择器
                    'content_selector': '.detail',
                    # 下一页 css选择器
                    'next_page': '.list-page > table:nth-child(1) > tbody:nth-child(1) > tr:nth-child(1) > td:nth-child(2) > font:nth-child(1) > a:nth-child(6)',
                    # site_id
                    'site_id': 'DC39472268',
                    'stop_time': '1'
                },
                # # 网站加载不了
                {
                    # 列表页
                    'site_path_url': 'http://yjgl.hebei.gov.cn/portal/index/toInfoNewsList?categoryid=63eedd10-e7a5-4c8b-867c-15a2065433a3&parentid=8d46b559-337f-49ab-89f7-50c6c7840065',
                    # 网站名称
                    'site_name': '河北省应急管理厅',
                    # 网站类型
                    'title_type': '国家部委',
                    # 网站目录
                    'site_path_name': '首页>工作动态>安全生产',
                    ###############################################
                    # 网站列表数据xpath/css
                    'ListQuery': '.safe-con.fl li',
                    # 网站的url 数据 包括 title_name 和 title_url
                    "title_url_query": "a",
                    # 网站的url 数据 包括 title_name 和 title_url
                    "title_name_query": "a",
                    # 网站列表下面的date 数据
                    'title_date_query': 'span',
                    # 详情页的css选择器
                    'content_selector': '.sentiment.clearfix.boxSizing',
                    # 下一页 css选择器
                    'next_page': '.bwl-ym2 > a:nth-child(3)',
                    # site_id
                    'site_id': 'C7CC47E5F7',
                    'stop_time': '3',
                    'load_time': 0
                }]
        for file in file_list:
                if 'site_id' not in file.keys():
                    with open(r'D:\KylsDemo\ProposedBuilding\ceuwu.txt', mode='a+', encoding='utf-8') as f:
                        f.write(str(file) + ',')

        for data in self.get_all_dict(num=num):
            for file in file_list:
                try:
                    if file['site_id'] == data['site_id']:
                        data['script_name'] = script_name
                        data['script_config'] = str(file)
                        print(data['site_id'] )
                        print(self.upload(data))
                except Exception as r:
                    print(r)




if __name__ == '__main__':
    a = SetSpiderConfiguration()
    # 获取数据库里的代码
    # print(a.get_id_script_config(id='DC39472268'))
    # data = a.check_script_name('xpathRequestSpider')
    # data = a.check_script_name('news_aio_spiders')
    data = a.check_script_name('gjbw_com_spider')
    print(data,len(data))


    # # 将代码导入数据库
    # # 返回值为文件内容的eval,取出里面的list, script_name脚本名称
    # path = r'C:\Users\Kyls\Desktop\上传spider配置'
    # for file in a.folder_read(path=path):
    #         file_list = a.loading_file(path=file)
    #         script_name = file.split('\\')[-1].split('.')[0]
    #         # print(script_name)
    #         a.main(num=9000, script_name=script_name, file_list=file_list)