Python 爬蟲(chóng)之某殼公寓投訴分析
來(lái)源：詭途?
https://blog.csdn.net/qq_35866846/article/details/109601322
由于今年疫情，加速了長(zhǎng)租公寓的暴雷，某殼公寓頻繁傳出各種負(fù)面新聞，于是把黑貓上關(guān)于某殼公寓的投訴內(nèi)容爬取了下來(lái)并進(jìn)行了分析，就有了這篇完整的數(shù)據(jù)分析實(shí)戰(zhàn)項(xiàng)目，從數(shù)據(jù)獲取到數(shù)據(jù)的分析。
一、數(shù)據(jù)抓取
import requests,timeimport pandas as pdimport numpy as nprequests.packages.urllib3.disable_warnings() # 屏蔽https請(qǐng)求證書(shū)驗(yàn)證警告from fake_useragent import UserAgent # 生成隨機(jī)請(qǐng)求頭# uid請(qǐng)求數(shù)據(jù)，數(shù)據(jù)格式較為規(guī)范，方便處理def request_data_uid(req_s,couid,page,total_page):params = {'couid': couid, # 商家ID'type': '1','page_size': page * 10, # 每頁(yè)10條'page': page, # 第幾頁(yè)# 'callback':'jQuery11',}print(f"正在爬取第{page}頁(yè),共計(jì){total_page}頁(yè)，剩余{total_page-page}頁(yè)")url = 'https://tousu.sina.com.cn/api/company/received_complaints'# 偽造隨機(jī)請(qǐng)求頭header={'user-agent':UserAgent().random}res=req_s.get(url,headers=header,params=params, verify=False)# res = requests.get(url, params=params, verify=False)info_list = res.json()['result']['data']['complaints']result =[]for info in info_list:_data = info['main']# 投訴日期timestamp =float(_data['timestamp'])date = time.strftime("%Y-%m-%d",time.localtime(timestamp))# sn：投訴編號(hào) title :投訴問(wèn)題 appeal：投訴訴求 summary :問(wèn)題說(shuō)明data = [date,_data['sn'],_data['title'],_data['appeal'],_data['summary']]result.append(data)pd_result = pd.DataFrame(result,columns=["投訴日期","投訴編號(hào)","投訴問(wèn)題","投訴訴求","詳細(xì)說(shuō)明"])return pd_result# keywords請(qǐng)求數(shù)據(jù)，數(shù)據(jù)格式相對(duì)混亂#?某梧桐這種沒(méi)有收錄商家ID的公司只能用keywords進(jìn)行檢索處理# 某殼公寓有uid的這種也可以使用keywods進(jìn)行數(shù)據(jù)請(qǐng)求def request_data_keywords(req_s,keyword,page,total_page):# page =1params = {'keywords':keyword, # 檢索關(guān)鍵詞'type': '1','page_size': page * 10, # 每頁(yè)10條'page': page, # 第幾頁(yè)# 'callback':'jQuery11',}print(f"正在爬取第{page}頁(yè),共計(jì){total_page}頁(yè)，剩余{total_page-page}頁(yè)")# url = 'https://tousu.sina.com.cn/api/company/received_complaints'url ='https://tousu.sina.com.cn/api/index/s?'# 偽造隨機(jī)請(qǐng)求頭header={'user-agent':UserAgent().random}res=req_s.get(url,headers=header,params=params, verify=False)# res = requests.get(url, params=params, verify=False)info_list = res.json()['result']['data']['lists']result =[]for info in info_list:_data = info['main']# 投訴日期timestamp =float(_data['timestamp'])date = time.strftime("%Y-%m-%d",time.localtime(timestamp))# sn：投訴編號(hào) title :投訴問(wèn)題 appeal：投訴訴求 summary :問(wèn)題說(shuō)明data = [date,_data['sn'],_data['title'],_data['appeal'],_data['summary']]result.append(data)pd_result = pd.DataFrame(result,columns=["投訴日期","投訴編號(hào)","投訴問(wèn)題","投訴訴求","詳細(xì)說(shuō)明"])return pd_result#生成并保持請(qǐng)求會(huì)話req_s = requests.Session()#?某殼公寓result = pd.DataFrame()total_page = 2507for page in range(1,total_page+1):data = request_data_uid(req_s,'5350527288',page,total_page)result = result.append(data)result['投訴對(duì)象']="某殼公寓"result.to_csv("某殼公寓投訴數(shù)據(jù).csv",index=False)#?某梧桐?關(guān)鍵詞檢索#?某殼公寓為品牌名，工商注冊(cè)名稱為某梧桐資產(chǎn)管理有限公司result = pd.DataFrame()total_page = 56for page in range(1,total_page+1):????data?=?request_data_keywords(req_s,'某梧桐',page,total_page)result = result.append(data)result['投訴對(duì)象']="某梧桐"result.to_csv("某梧桐投訴數(shù)據(jù).csv",index=False)
二、清洗繪圖
import os,reimport pandas as pdimport numpy as np# 數(shù)據(jù)清洗，處理keywords爬取導(dǎo)致的投訴標(biāo)題混亂data_path = os.path.join('data','某梧桐投訴數(shù)據(jù).csv')data =pd.read_csv(data_path)pattern=r'[^\u4e00-\u9fa5\d]'data['投訴問(wèn)題']=data['投訴問(wèn)題'].apply(lambda x: re.sub(pattern,'',x))data.to_csv(data_path,index=False,encoding="utf_8_sig")# 數(shù)據(jù)合并result = pd.DataFrame()for wj in os.listdir('data'):data_path = os.path.join('data',wj)data =pd.read_csv(data_path)result = result.append(data)result.to_csv("data/合并后某殼投訴數(shù)據(jù).csv",index=False,encoding="utf_8_sig")
# 讀取數(shù)據(jù)data = pd.read_csv("data/合并后某殼投訴數(shù)據(jù).csv")# 篩選到截止昨天的數(shù)據(jù)，保證按天數(shù)據(jù)的完整性data = data[data.投訴日期<='2020-11-09']print(f"截至2020-11-09之前，黑貓投訴累計(jì)收到某殼公寓相關(guān)投訴共計(jì) {len(data)} 條")
# 時(shí)間分布處理_data=data.groupby('投訴日期').count().reset_index()[['投訴日期','投訴編號(hào)']]_data.rename(columns={"投訴編號(hào)":"投訴數(shù)量"},inplace = True)# 2020-01-30之前投訴數(shù)量求和num1 = _data[_data.投訴日期<='2020-01-30'].投訴數(shù)量.sum()data0 =pd.DataFrame([['2020-01-30之前',num1]],columns=['投訴日期','投訴數(shù)量'])# 2020-02-01 ~ 2020-02-21號(hào)之間投訴情況分布data1=_data[(_data.投訴日期>='2020-02-01')&(_data.投訴日期<='2020-02-21')]# 2020-02-21 ~ 2020-11-05num2 = _data[(_data.投訴日期>='2020-02-21')&(_data.投訴日期<='2020-11-05')].投訴數(shù)量.sum()# 2020-11-06 ~ 2020-11-09 本數(shù)據(jù)只采集到2020-11-09print(f"2020-11-06當(dāng)天投訴量{_data[_data.投訴日期=='2020-11-06'].iloc[0,1]}條")data2=_data[(_data.投訴日期>'2020-11-06')&(_data.投訴日期<='2020-11-09')]data3=pd.DataFrame([['2020-02-21 ~ 2020-11-05',num2]],columns=['投訴日期','投訴數(shù)量'])new_data = pd.concat([data0,data1,data3,data2])
'''配置繪圖參數(shù)'''import matplotlib.pyplot as plt%matplotlib inlineplt.rcParams['font.sans-serif']=['SimHei']plt.rcParams['font.size']=18plt.rcParams['figure.figsize']=(12,8)plt.style.use("ggplot")new_data.set_index('投訴日期').plot(kind='bar') # 剔除了2020-11-06的數(shù)據(jù)，24093條
2020-01-30之前屬于正常投訴量，偶爾一兩單，2月份因?yàn)橐咔樵?，?dǎo)致投訴量大量增長(zhǎng)，可能是因?yàn)橐咔樵驘o(wú)法保潔，疫情租房補(bǔ)貼之類的，還有被長(zhǎng)租公寓暴雷以及某殼破產(chǎn)之類的負(fù)面新聞給帶起來(lái)的租戶緊張等等。
2020-02-21之后一直到2020-11-05號(hào)投訴量很正常，相比較2020-01-30之前略多，仍在正常經(jīng)營(yíng)可接受范圍內(nèi)。
2020-11-06突然驟增了2萬(wàn)4千多條投訴，異常值影響展示，單獨(dú)剔除出去了，特地去查了一下新聞，看看有沒(méi)有什么大事兒發(fā)生，結(jié)果還真有，據(jù)36氪報(bào)道 2020-11-06某殼公寓關(guān)聯(lián)公司稱被執(zhí)行人，執(zhí)行標(biāo)的超519萬(wàn)元。
自此之后的7、8、9某殼在黑貓的投訴每天維持在2-300的日增，看來(lái)某殼破產(chǎn)的官方辟謠都是扯淡了，也許并不是謠言，也許網(wǎng)傳某殼再現(xiàn)ofo排隊(duì)討債并非空穴來(lái)風(fēng)。
以上還是僅僅從黑貓上獲取到的投訴數(shù)據(jù)，投訴無(wú)門(mén)以及自認(rèn)倒霉的的用戶量又會(huì)有多大呢？
接下來(lái)就看一下，投訴用戶主要投訴的是什么？主要訴求是什么？
三、詞云生成
import jieba# 分詞模塊import reimport collectionsimport PIL.Image as img# pip install PILfrom wordcloud import WordCloudimport PIL.Image as img# pip install PILfrom wordcloud import WordCloud# 投訴詳細(xì)說(shuō)明合并后進(jìn)行分詞all_word=''for line in data.values:word = line[4]all_word = all_word+word# jieba分詞result=list(jieba.cut(all_word))# 投訴問(wèn)題詞云圖wordcloud=WordCloud(width=800,height=600,background_color='white',font_path='C:\\Windows\\Fonts\\msyh.ttc',# 如果存在中文字符需要加載解析的詞典max_font_size=500,min_font_size=20).generate(' '.join(result))image=wordcloud.to_image()# image.show()# 生成圖片展示wordcloud.to_file('某殼公寓投訴詳情.png')#?在本地生成文件展示# 投訴標(biāo)題合并后進(jìn)行分詞all_word=''for line in data.values:word = line[2]all_word = all_word+word# jieba分詞result=list(jieba.cut(all_word))# 生成詞云圖# 投訴問(wèn)題詞云圖wordcloud=WordCloud(width=800,height=600,background_color='white',font_path='C:\\Windows\\Fonts\\msyh.ttc',# 如果存在中文字符需要加載解析的詞典max_font_size=500,min_font_size=20).generate(' '.join(result))image=wordcloud.to_image()# image.show()# 生成圖片展示wordcloud.to_file('某殼公寓投訴問(wèn)題.png')#?在本地生成文件展示# 投訴訴求合并后進(jìn)行分詞all_word=''for line in data.values:word = line[3]all_word = all_word+word# jieba分詞result=list(jieba.cut(all_word))# 生成詞云圖# 投訴問(wèn)題詞云圖wordcloud=WordCloud(width=800,height=600,background_color='white',font_path='C:\\Windows\\Fonts\\msyh.ttc',# 如果存在中文字符需要加載解析的詞典max_font_size=500,min_font_size=20).generate(' '.join(result))image=wordcloud.to_image()# image.show()# 生成圖片展示wordcloud.to_file('某殼公寓投訴訴求.png')#?在本地生成文件展示
某殼公寓投訴詳情 詞云圖
投訴詳情可以看出來(lái)，主要投訴問(wèn)題：提現(xiàn)，活動(dòng)返現(xiàn)（每個(gè)月返多少錢(qián)，我的除了剛開(kāi)始兩個(gè)月正常返現(xiàn)，后面也沒(méi)按時(shí)打款，客服打不通后面就沒(méi)怎么關(guān)注了），主要還有客服聯(lián)系不上，保潔問(wèn)題等！也許好好直面問(wèn)題，投訴可能也沒(méi)那么多。
某殼公寓投訴訴求 詞云圖
投訴用戶的主要訴求大家強(qiáng)烈要求對(duì)某殼公寓做出相應(yīng)處罰并要求退款和賠償。
戀習(xí)Python 關(guān)注戀習(xí)Python，Python都好練 好文章，我在看??
評(píng)論
圖片
表情