手把手教你爬取某果TV彈幕數(shù)據(jù)，并進(jìn)行數(shù)據(jù)可視化分析~

回復(fù)“書籍”即可獲贈(zèng)Python從入門到進(jìn)階共10本電子書

本期是對(duì)芒果TV視頻評(píng)論的一次爬蟲與數(shù)據(jù)分析。

爬蟲方面：由于芒果的評(píng)論數(shù)據(jù)是封裝在json里面，所以只需要找到j(luò)son文件，對(duì)需要的數(shù)據(jù)進(jìn)行提取保存即可。

• 視頻網(wǎng)址：https://www.mgtv.com/b/44793/11017269.html?fpa=se&lastp=so_result
• 評(píng)論json數(shù)據(jù)網(wǎng)址：https://comment.mgtv.com/v4/comment/getCommentList?page=1&subjectType=hunantv2014&subjectId=11017269
• 注：只要替換subjectId的值，即可爬取其他視頻的評(píng)論

數(shù)據(jù)分析方面：涉及到了詞云圖，條形，折線，餅圖，后三者是對(duì)評(píng)論時(shí)間的分析，然而芒果TV的評(píng)論時(shí)間是以時(shí)間戳的形式顯示，所以要進(jìn)行轉(zhuǎn)換，再去統(tǒng)計(jì)出現(xiàn)次數(shù)。

項(xiàng)目結(jié)構(gòu)：一.  爬蟲部分：1.爬蟲代碼：spiders.py

# coding=gbk
import csv
import os
import sys
import time

import rdata as rdata
import requests
import json
import pandas as pd


# 封裝數(shù)據(jù)的網(wǎng)站
from python_helper.api.src.service.LogHelper import setting

headers = {
        'cookie': 'cna=J7K2Fok5AXECARu7QWn6+cxu; isg=BCcnDiP-NfKV5bF-OctWuXuatl3xrPuOyBVJJfmQLrZn6ESqAX0y3jrhCuj2ANMG; l=eBSmWoPRQeT6Zn3iBO5whurza77O1CAf1sPzaNbMiIncC6BR1AvOCJxQLtyCvptRR8XcGLLB4nU7C5eTae7_7CDmndLHuI50MbkyCef..',
        'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.138 Safari/537.36'
    }
for i in range(1,99):
    #url = f'https://search.damai.cn/searchajax.html?keyword=&cty=&ctl=%E6%BC%94%E5%94%B1%E4%BC%9A&sctl=&tsg=0&st=&et=&order=1&pageSize=30&currPage={i}&tn='
    url = f'https://comment.mgtv.com/v4/comment/getCommentList?page={i}&subjectType=hunantv2014&subjectId=11017269'
    print(url)

    response = requests.get(url, headers=headers)
    json_text = json.loads(response.text)

    # print(json_text.keys())
    for t in range(1, 14):
        rdata1 = json_text['data']['list'][t]['content']
        rdata2 = int(json_text['data']['list'][t]['createTime'])

        # 轉(zhuǎn)換為其他日期格式,如:"%Y-%m-%d %H:%M:%S"
        timeArray = time.localtime(rdata2)
        rdata2 = time.strftime("%Y-%m-%d %H:%M:%S", timeArray)
        #print(rdata2)

        print(rdata1,rdata2)

        f = open('百變大咖秀.txt', 'a+',encoding="utf-8")
        print(rdata1,file = f)

        f = open('時(shí)間.txt', 'a+', encoding="utf-8")
        print(rdata2, file=f)


        f.close()

2.將評(píng)論時(shí)間的txt文件讀入csv文件 CD.py

# coding=gbk
import csv
csvFile = open("data.csv",'w',newline='',encoding='utf-8')
writer = csv.writer(csvFile)
csvRow = []

f = open("時(shí)間.txt",'r',encoding='GB2312')
for line in f:
    csvRow = line.split()
    writer.writerow(csvRow)

f.close()
csvFile.close()

二. 數(shù)據(jù)分析 1.制作詞云圖 wc.py

import numpy as np
import jieba
from wordcloud import WordCloud
from matplotlib import pyplot as plt
from PIL import Image

# 上面的包自己安裝，不會(huì)的就百度

f = open('../Spiders/百變大咖秀.txt', 'r', encoding='utf-8')  # 這是數(shù)據(jù)源，也就是想生成詞云的數(shù)據(jù)
txt = f.read()  # 讀取文件
f.close()  # 關(guān)閉文件，其實(shí)用with就好，但是懶得改了
# 如果是文章的話，需要用到j(luò)ieba分詞，分完之后也可以自己處理下再生成詞云
words = jieba.lcut(txt)
newtxt = ' '.join(words)
img = Image.open(r'wc.jpg')  # 想要搞得形狀
img_array = np.array(img)

# 相關(guān)配置，里面這個(gè)collocations配置可以避免重復(fù)
wordcloud = WordCloud(
    background_color="white",
    width=1080,
    height=960,
    font_path="../文悅新青年.otf",
    max_words=150,
    scale=7,#清晰度
    max_font_size=100,
    mask=img_array,
    collocations=False).generate(txt)

plt.imshow(wordcloud)
plt.axis('off')
plt.show()
wordcloud.to_file('../Photo/result.png')

輪廓圖：wc.jpg詞云圖：result.png （注：停用詞自己加，這里沒有放）2.可視化分析

（1）時(shí)間數(shù)據(jù)處理

py.py   (統(tǒng)計(jì)一天各個(gè)時(shí)間段內(nèi)的評(píng)論數(shù)）

# coding=gbk
import csv

from pyecharts import options as opts
from sympy.combinatorics import Subset
from wordcloud import WordCloud

with open('../Spiders/data.csv') as csvfile:
    reader = csv.reader(csvfile)

    data1 = [str(row[1])[0:2] for row in reader]

    print(data1)
print(type(data1))


#先變成集合得到seq中的所有元素，避免重復(fù)遍歷
set_seq = set(data1)
rst = []
for item in set_seq:
    rst.append((item,data1.count(item)))  #添加元素及出現(xiàn)個(gè)數(shù)
rst.sort()
print(type(rst))
print(rst)

with open("time2.csv", "w+", newline='', encoding='utf-8') as f:
    writer = csv.writer(f, delimiter=',')
    for i in rst:                # 對(duì)于每一行的，將這一行的每個(gè)元素分別寫在對(duì)應(yīng)的列中
        writer.writerow(i)

with open('time2.csv') as csvfile:
     reader = csv.reader(csvfile)
     x = [str(row[0]) for row in reader]
     print(x)
with open('time2.csv') as csvfile:
    reader = csv.reader(csvfile)
    y1 = [float(row[1]) for row in reader]
    print(y1)

處理結(jié)果（評(píng)論時(shí)間，評(píng)論數(shù)）py1.py （統(tǒng)計(jì)最近評(píng)論數(shù)）

# coding=gbk
import csv

from pyecharts import options as opts
from sympy.combinatorics import Subset
from wordcloud import WordCloud

with open('../Spiders/data.csv') as csvfile:
    reader = csv.reader(csvfile)

    data1 = [str(row[0]) for row in reader]
    #print(data1)
print(type(data1))


#先變成集合得到seq中的所有元素，避免重復(fù)遍歷
set_seq = set(data1)
rst = []
for item in set_seq:
    rst.append((item,data1.count(item)))  #添加元素及出現(xiàn)個(gè)數(shù)
rst.sort()
print(type(rst))
print(rst)



with open("time1.csv", "w+", newline='', encoding='utf-8') as f:
    writer = csv.writer(f, delimiter=',')
    for i in rst:                # 對(duì)于每一行的，將這一行的每個(gè)元素分別寫在對(duì)應(yīng)的列中
        writer.writerow(i)

with open('time1.csv') as csvfile:
     reader = csv.reader(csvfile)
     x = [str(row[0]) for row in reader]
     print(x)
with open('time1.csv') as csvfile:
    reader = csv.reader(csvfile)
    y1 = [float(row[1]) for row in reader]

    print(y1)

處理結(jié)果（評(píng)論時(shí)間，評(píng)論數(shù)）（2）制作最近評(píng)論數(shù)條形圖與折線圖 DrawBar.py

# encoding: utf-8
import csv
import pyecharts.options as opts
from pyecharts.charts import Bar
from pyecharts.globals import ThemeType


class DrawBar(object):

    """繪制柱形圖類"""
    def __init__(self):
        """創(chuàng)建柱狀圖實(shí)例，并設(shè)置寬高和風(fēng)格"""
        self.bar = Bar(init_opts=opts.InitOpts(width='1500px', height='700px', theme=ThemeType.LIGHT))

    def add_x(self):
        """為圖形添加X軸數(shù)據(jù)"""
        with open('time1.csv') as csvfile:
            reader = csv.reader(csvfile)
            x = [str(row[0]) for row in reader]
            print(x)


        self.bar.add_xaxis(
            xaxis_data=x,

        )

    def add_y(self):
        with open('time1.csv') as csvfile:
            reader = csv.reader(csvfile)
            y1 = [float(row[1]) for row in reader]

            print(y1)



        """為圖形添加Y軸數(shù)據(jù)，可添加多條"""
        self.bar.add_yaxis(  # 第一個(gè)Y軸數(shù)據(jù)
            series_name="評(píng)論數(shù)",  # Y軸數(shù)據(jù)名稱
            y_axis=y1,  # Y軸數(shù)據(jù)
            label_opts=opts.LabelOpts(is_show=False),  # 設(shè)置標(biāo)簽
            bar_max_width='70px',  # 設(shè)置柱子最大寬度
        )


    def set_global(self):
        """設(shè)置圖形的全局屬性"""
        #self.bar(width=2000,height=1000)
        self.bar.set_global_opts(
            title_opts=opts.TitleOpts(  # 設(shè)置標(biāo)題
                title='百變大咖秀近日評(píng)論統(tǒng)計(jì)',title_textstyle_opts=opts.TextStyleOpts(font_size=35)

            ),
            tooltip_opts=opts.TooltipOpts(  # 提示框配置項(xiàng)（鼠標(biāo)移到圖形上時(shí)顯示的東西）
                is_show=True,  # 是否顯示提示框
                trigger="axis",  # 觸發(fā)類型（axis坐標(biāo)軸觸發(fā)，鼠標(biāo)移到時(shí)會(huì)有一條垂直于X軸的實(shí)線跟隨鼠標(biāo)移動(dòng)，并顯示提示信息）
                axis_pointer_type="cross"  # 指示器類型（cross將會(huì)生成兩條分別垂直于X軸和Y軸的虛線，不啟用trigger才會(huì)顯示完全）
            ),
            toolbox_opts=opts.ToolboxOpts(),  # 工具箱配置項(xiàng)(什么都不填默認(rèn)開啟所有工具)

        )

    def draw(self):
        """繪制圖形"""

        self.add_x()
        self.add_y()
        self.set_global()
        self.bar.render('../Html/DrawBar.html')  # 將圖繪制到 test.html 文件內(nèi)，可在瀏覽器打開
    def run(self):
        """執(zhí)行函數(shù)"""
        self.draw()



if __name__ == '__main__':
    app = DrawBar()

app.run()

效果圖：DrawBar.html（3）制作每小時(shí)評(píng)論條形圖與折線圖 DrawBar2.py

# encoding: utf-8
import csv
import pyecharts.options as opts
from pyecharts.charts import Bar
from pyecharts.globals import ThemeType


class DrawBar(object):

    """繪制柱形圖類"""
    def __init__(self):
        """創(chuàng)建柱狀圖實(shí)例，并設(shè)置寬高和風(fēng)格"""
        self.bar = Bar(init_opts=opts.InitOpts(width='1500px', height='700px', theme=ThemeType.MACARONS))

    def add_x(self):
        """為圖形添加X軸數(shù)據(jù)"""
        str_name1 = '點(diǎn)'

        with open('time2.csv') as csvfile:
            reader = csv.reader(csvfile)
            x = [str(row[0] + str_name1) for row in reader]
            print(x)


        self.bar.add_xaxis(
            xaxis_data=x
        )

    def add_y(self):
        with open('time2.csv') as csvfile:
            reader = csv.reader(csvfile)
            y1 = [int(row[1]) for row in reader]

            print(y1)



        """為圖形添加Y軸數(shù)據(jù)，可添加多條"""
        self.bar.add_yaxis(  # 第一個(gè)Y軸數(shù)據(jù)
            series_name="評(píng)論數(shù)",  # Y軸數(shù)據(jù)名稱
            y_axis=y1,  # Y軸數(shù)據(jù)
            label_opts=opts.LabelOpts(is_show=False),  # 設(shè)置標(biāo)簽
            bar_max_width='50px',  # 設(shè)置柱子最大寬度

        )


    def set_global(self):
        """設(shè)置圖形的全局屬性"""
        #self.bar(width=2000,height=1000)
        self.bar.set_global_opts(
            title_opts=opts.TitleOpts(  # 設(shè)置標(biāo)題
                title='百變大咖秀各時(shí)間段評(píng)論統(tǒng)計(jì)',title_textstyle_opts=opts.TextStyleOpts(font_size=35)

            ),
            tooltip_opts=opts.TooltipOpts(  # 提示框配置項(xiàng)（鼠標(biāo)移到圖形上時(shí)顯示的東西）
                is_show=True,  # 是否顯示提示框
                trigger="axis",  # 觸發(fā)類型（axis坐標(biāo)軸觸發(fā)，鼠標(biāo)移到時(shí)會(huì)有一條垂直于X軸的實(shí)線跟隨鼠標(biāo)移動(dòng)，并顯示提示信息）
                axis_pointer_type="cross"  # 指示器類型（cross將會(huì)生成兩條分別垂直于X軸和Y軸的虛線，不啟用trigger才會(huì)顯示完全）
            ),
            toolbox_opts=opts.ToolboxOpts(),  # 工具箱配置項(xiàng)(什么都不填默認(rèn)開啟所有工具)

        )

    def draw(self):
        """繪制圖形"""

        self.add_x()
        self.add_y()
        self.set_global()
        self.bar.render('../Html/DrawBar2.html')  # 將圖繪制到 test.html 文件內(nèi)，可在瀏覽器打開
    def run(self):
        """執(zhí)行函數(shù)"""
        self.draw()



if __name__ == '__main__':
    app = DrawBar()

app.run()

效果圖：DrawBar2.html（4）制作各類餅圖

• pie_pyecharts.py

import csv

from pyecharts import options as opts
from pyecharts.charts import Pie
from random import randint

from pyecharts.globals import ThemeType

with open('time1.csv') as csvfile:
    reader = csv.reader(csvfile)
    x = [str(row[0]) for row in reader]
    print(x)
with open('time1.csv') as csvfile:
    reader = csv.reader(csvfile)
    y1 = [float(row[1]) for row in reader]

    print(y1)



num = y1
lab = x
(
    Pie(init_opts=opts.InitOpts(width='1500px',height='500px',theme=ThemeType.LIGHT))#默認(rèn)900，600
    .set_global_opts(
        title_opts=opts.TitleOpts(title="百變大咖秀近日評(píng)論統(tǒng)計(jì)",
                                               title_textstyle_opts=opts.TextStyleOpts(font_size=27)),legend_opts=opts.LegendOpts(

            pos_top="8%",# 圖例位置調(diào)整
            ),)
    .add(series_name='',center=[400, 300], data_pair=[(j, i) for i, j in zip(num, lab)])#餅圖
   #.add(series_name='',center=[750, 300],data_pair=[(j,i) for i,j in zip(num,lab)],radius=['40%','75%'])#環(huán)圖
    .add(series_name='', center=[1100, 300],data_pair=[(j, i) for i, j in zip(num, lab)], rosetype='radius')#南丁格爾圖
).render('../Html/pie_pyecharts.html')

效果圖

• pie_pyecharts2.py

import csv

from pyecharts import options as opts
from pyecharts.charts import Pie
from random import randint

from pyecharts.globals import ThemeType

str_name1 = '點(diǎn)'

with open('time2.csv') as csvfile:
    reader = csv.reader(csvfile)
    x = [str(row[0]+str_name1) for row in reader]
    print(x)
with open('time2.csv') as csvfile:
    reader = csv.reader(csvfile)
    y1 = [int(row[1]) for row in reader]

    print(y1)



num = y1
lab = x
(
    Pie(init_opts=opts.InitOpts(width='1520px',height='520px',theme=ThemeType.LIGHT,))#默認(rèn)900，600
     .set_global_opts(
        title_opts=opts.TitleOpts(title="百變大咖秀每小時(shí)評(píng)論統(tǒng)計(jì)"
                                  ,title_textstyle_opts=opts.TextStyleOpts(font_size=27)),
        legend_opts=opts.LegendOpts(

            pos_top="8%",# 圖例位置調(diào)整
            ),
    )
    .add(series_name='',center=[250, 320], data_pair=[(j, i) for i, j in zip(num, lab)])#餅圖
    .add(series_name='',center=[790, 320],data_pair=[(j,i) for i,j in zip(num,lab)],radius=['40%','75%'])#環(huán)圖
    .add(series_name='', center=[1262, 320],data_pair=[(j, i) for i, j in zip(num, lab)], rosetype='radius')#南丁格爾圖
).render('../Html/pie_pyecharts2.html')

效果圖

• pie_pyecharts3.py 觀看時(shí)間區(qū)間評(píng)論統(tǒng)計(jì)

# coding=gbk
import csv

from pyecharts import options as opts
from pyecharts.globals import ThemeType
from sympy.combinatorics import Subset
from wordcloud import WordCloud

with open('../Spiders/data.csv') as csvfile:
    reader = csv.reader(csvfile)

    data2 = [int(row[1].strip('')[0:2]) for row in reader]


    #print(data2)
print(type(data2))

#先變成集合得到seq中的所有元素，避免重復(fù)遍歷
set_seq = set(data2)
list = []
for item in set_seq:
    list.append((item,data2.count(item)))  #添加元素及出現(xiàn)個(gè)數(shù)
list.sort()
print(type(list))
#print(list)


with open("time2.csv", "w+", newline='', encoding='utf-8') as f:
    writer = csv.writer(f, delimiter=',')
    for i in list:                # 對(duì)于每一行的，將這一行的每個(gè)元素分別寫在對(duì)應(yīng)的列中
        writer.writerow(i)


n = 4 #分成n組
m = int(len(list)/n)
list2 = []
for i in range(0, len(list), m):
    list2.append(list[i:i+m])

print("凌晨 : ",list2[0])
print("上午 : ",list2[1])
print("下午 : ",list2[2])
print("晚上 : ",list2[3])

with open('time2.csv') as csvfile:
    reader = csv.reader(csvfile)
    y1 = [int(row[1]) for row in reader]

    print(y1)

n =6
groups = [y1[i:i + n] for i in range(0, len(y1), n)]

print(groups)

x=['凌晨','上午','下午','晚上']
y1=[]
for y1 in groups:
    num_sum = 0
    for groups in y1:
        num_sum += groups

print(x)
print(y1)


import csv

from pyecharts import options as opts
from pyecharts.charts import Pie
from random import randint

str_name1 = '點(diǎn)'

num = y1
lab = x
(
    Pie(init_opts=opts.InitOpts(width='1500px',height='500px',theme=ThemeType.LIGHT))#默認(rèn)900，600
        .set_global_opts(
        title_opts=opts.TitleOpts(title="百變大咖秀觀看時(shí)間區(qū)間評(píng)論統(tǒng)計(jì)"
                                  , title_textstyle_opts=opts.TextStyleOpts(font_size=40)),
        legend_opts=opts.LegendOpts(

            pos_top="8%",  # 圖例位置調(diào)整
        ),
    )
    .add(series_name='',center=[260, 300], data_pair=[(j, i) for i, j in zip(num, lab)])#餅圖
   .add(series_name='',center=[1230, 300],data_pair=[(j,i) for i,j in zip(num,lab)],radius=['40%','75%'])#環(huán)圖
    .add(series_name='', center=[750, 300],data_pair=[(j, i) for i, j in zip(num, lab)], rosetype='radius')#南丁格爾圖
).render('../Html/pie_pyecharts3.html')

效果圖

小伙伴們，快快用實(shí)踐一下吧！如果在學(xué)習(xí)過程中，有遇到任何問題，歡迎加我好友，我拉你進(jìn)Python學(xué)習(xí)交流群共同探討學(xué)習(xí)。

------------------- End -------------------

往期精彩文章推薦：

• 手把手教你用Pyecharts庫對(duì)淘寶數(shù)據(jù)進(jìn)行可視化展示

• 手把手教你用Pandas庫對(duì)淘寶原始數(shù)據(jù)進(jìn)行數(shù)據(jù)處理和分詞處理

• 盤點(diǎn)Flask與數(shù)據(jù)庫的交互插件--Flask-Sqlalchemy
  

• 盤點(diǎn)那些年我們一起玩過的網(wǎng)絡(luò)安全工具
  

歡迎大家點(diǎn)贊，留言，轉(zhuǎn)發(fā)，轉(zhuǎn)載，感謝大家的相伴與支持

想加入Python學(xué)習(xí)群請(qǐng)?jiān)诤笈_(tái)回復(fù)【入群】

萬水千山總是情，點(diǎn)個(gè)【在看】行不行

/今日留言主題/

隨便說一兩句吧~~