一、需求

Python爬取网易云音乐所有歌手的名称和链接

二、代码实现

import requests
from lxml import etree
from urllib import request
import time


class WangYiYun:
    def __init__(self, base_url):
        # 初始化tree
        self.html = self.request_url(base_url)
        self.parse_html()

    def request_url(self, base_url):
        # 请求访问 生成树结构
        headers = { 
            'user-agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'
        }
        self.html = requests.get(url=base_url, headers=headers).content.decode('utf-8')
        self.tree = etree.HTML(self.html)
        return self.tree

    def parse_html(self):
        # 解析各类歌手url
        group_list = self.html.xpath('//div[@class = "blk"]')

        for group in group_list:
            # 连接分类
            href_list = group.xpath('.//a/@href')
            # 歌手分类
            group_name_list = group.xpath('.//a/text()')
            # print(group_name_list)
            for href, group_name in zip(href_list, group_name_list):
                full_href = request.urljoin('https://music.163.com', href)
                # print(full_href,group_name)
                html1 = requests.get(full_href).content.decode('utf-8')
                tree1 = etree.HTML(html1)
                # print(tree1)
                # 歌手分类对应的歌手列表页链接
                url_list = tree1.xpath('//ul[@class="n-ltlst f-cb"]/li[position()>1]/a/@href')
                # print(url_list)
                # 歌手列表页链接遍历
                for url in url_list:
                    full_url = request.urljoin('https://music.163.com', url)
                    single_letter = requests.get(full_url).content.decode('utf-8')
                    #
                    single_letter_tree = etree.HTML(single_letter)
                    # 歌手列表页获取歌手信息
                    li_list = single_letter_tree.xpath('//ul[@class="m-cvrlst m-cvrlst-5 f-cb"]/li')
                    for li in li_list:
                        # 从歌手列表页中遍历获取名字列表及对应的链接列表
                        name_list = li.xpath('//a[@class="nm nm-icn f-thide s-fc0"]/text()')
                        link_list = li.xpath('//a[@class="nm nm-icn f-thide s-fc0"]/@href')
                        for name, link in zip(name_list, link_list):
                            full_link = 'https://music.163.com' + link
                            print(name, full_link)
                            time.sleep(0.5)

if __name__ == '__main__':
    base_url = 'https://music.163.com/discover/artist'
    WangYiYun(base_url)

三、程序运行结果