from lxml import etree
from selenium import webdriver
import time
import random
import pandas as pd
from selenium.webdriver.chrome.options import Options as ChromeOptions


class Netease_spider:
    # 初始化数据
    def __init__(self):
        # 无头启动 selenium
        opt = webdriver.chrome.options.Options()
        opt.add_argument('--headless')
        self.browser = webdriver.Chrome(options=opt)
        self.originURL = 'https://music.163.com/#/discover/playlist/?cat=%E5%8D%8E%E8%AF%AD&order=hot'
        self.data = list()

    # 获取网页源代码
    def get_page(self,url):
        self.browser.get(url)
        self.browser.switch_to.frame('g_iframe')
        html = self.browser.page_source
        return html

    # 解析网页源代码，获取数据
    def parse4data(self,html):
        html_elem = etree.HTML(html)
        song_title = html_elem.xpath('//ul[@id="m-pl-container"]/li/p[1]/a/@title')
        song_href = html_elem.xpath('//ul[@id="m-pl-container"]/li/p[1]/a/@href')
        song_link = ['https://music.163.com/#'+item for item in song_href]
        data = list(map(lambda a,b:{'name':a,'id':b},song_title,song_link))
        return data

    # 解析网页源代码，获取下一页链接
    def parse4link(self,html):
        html_elem = etree.HTML(html)
        href = html_elem.xpath('//div[@id="m-pl-pager"]/div[@class="u-page"]/a[@class="zbtn znxt"]/@href')
        if not href:
            return None
        else:
            return 'https://music.163.com/#' + href[0]

    # 开始爬取网页
    def crawl(self):
        # 爬取数据
        print('爬取数据')
        html = self.get_page(self.originURL)
        data = self.parse4data(html)
        self.data.extend(data)
        link = self.parse4link(html)
        #循环爬取
        while(link):
            html = self.get_page(link)
            data = self.parse4data(html)
            self.data.extend(data)
            link = self.parse4link(html)
            time.sleep(random.random())
        # 写入文件
        print('写入文件')
        print(self.data)
        save_date(self.data)
    
    #保存数据
def save_date(list):
        df1 = pd.DataFrame(list)
        df3 = df1.drop_duplicates(subset=None, keep='first',inplace = False) 
        df3.to_csv('music_list.csv',index_label="index_label", encoding='utf-8-sig')  # index_label索引列的列标签

if __name__ == '__main__':
    spider = Netease_spider()
    spider.crawl()
    print('Finished')