干貨好文！基于Scrapy框架的微博評(píng)論爬蟲實(shí)戰(zhàn)

pip install scrapy

scrapy startproject weibo

spiders是存放爬蟲程序的文件夾，將寫好的爬蟲程序放到該文件夾中。items用來定義數(shù)據(jù)，類似于字典的功能。settings是設(shè)置文件，包含爬蟲項(xiàng)目的設(shè)置信息。pipelines用來對(duì)items中的數(shù)據(jù)進(jìn)行進(jìn)一步處理，如：清洗、存儲(chǔ)等。

data = scrapy.Field()

import scrapy
class WeiboSpider(scrapy.Spider):
   name = 'weibo'  #用于啟動(dòng)微博程序
   allowed_domains = ['m.weibo.cn']  #定義爬蟲爬取網(wǎng)站的域名
   start_urls = ['https://m.weibo.cn/comments/hotflow?id=4700480024348767&mid=4700480024348767&max_id_type=0']  #定義起始網(wǎng)頁的網(wǎng)址
  for i in res['data']['data']:
     weibo_item = WeiboItem()
     weibo_item['data'] = re.sub(r'<[^>]*>', '', i['text'])
     # start_url = ['https://m.weibo.cn/comments/hotflow?id=4700480024348767&mid=4700480024348767&'+str(max_id)+'&max_id_type=0']
     yield weibo_item  #將數(shù)據(jù)回傳給items

max_id_type = res['data']['max_id_type']
if int(max_id_type) == 1:
   new_url = 'https://m.weibo.cn/comments/hotflow?id=4700480024348767&mid=4700480024348767&max_id=' + str(
       max_id) + '&max_id_type=1'
else:
   new_url = 'https://m.weibo.cn/comments/hotflow?id=4700480024348767&mid=4700480024348767&max_id=' + str(
       max_id) + '&max_id_type=0'

class CsvItemExporterPipeline(object):
   def __init__(self):
       # 創(chuàng)建接收文件,初始化exporter屬性
       self.file = open('text.csv','ab')
       self.exporter = CsvItemExporter(self.file,fields_to_export=['data'])
       self.exporter.start_exporting()

# Obey robots.txt rules
ROBOTSTXT_OBEY = False

# Disable cookies (enabled by default)
COOKIES_ENABLED = False

# Configure item pipelines
# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
  'weibo.pipelines.CsvItemExporterPipeline': 1,
  'weibo.pipelines.WeiboPipeline': 300,
}

from scrapy import cmdline
#導(dǎo)入cmdline模塊,可以實(shí)現(xiàn)控制終端命令行。
cmdline.execute(['scrapy','crawl','weibo'])
#用execute（）方法，輸入運(yùn)行scrapy的命令。