python爬蟲(chóng)學(xué)習(xí)教程：爬取內(nèi)涵段子【附源碼】

#encoding=utf-8import urllib2 import re  class neihanba():  def spider(self):    '''    爬蟲(chóng)的主調(diào)度器    '''    isflow=True#判斷是否進(jìn)行下一頁(yè)    page=1    while isflow:      url="http://www.neihanpa.com/article/list_5_"+str(page)+".html"      html=self.load(url)      self.deal(html,page)      panduan=raw_input("是否繼續(xù)(y/n)!")      if panduan=="y":        isflow=True        page+=1      else:        isflow=False  def load(self,url):    '''    針對(duì)url地址進(jìn)行全部爬去    :param url: url地址    :return: 返回爬去的內(nèi)容    '''    header = {      "User-Agent": " Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.79 Safari/537.36"    }    request = urllib2.Request(url, headers=header)    response = urllib2.urlopen(request)    html = response.read()    return html  def deal(self,html,page):    '''    對(duì)之前爬去的內(nèi)容進(jìn)行正則匹配，匹配出標(biāo)題和正文內(nèi)容    :param html:之前爬去的內(nèi)容    :param page: 正在爬去的頁(yè)碼    '''    parrten=re.compile('<li class="piclist\d+">(.*?)</li>',re.S)    titleList=parrten.findall(html)    for title in titleList:      parrten1=re.compile('<a href="/article/\d+.html" rel="external nofollow" >(.*)</a>')      ti1=parrten1.findall(title)      parrten2=re.compile('<div class="f18 mb20">(.*?)</div>',re.S)      til2=parrten2.findall(title)      for t in ti1:        tr=t.replace("<b>","").replace("</b>","")        self.writeData(tr,page)      for t in til2:        tr=t.replace("<p>","").replace("</p>","").replace("<br>","").replace("<br />","").replace("&ldquo","\"").replace("&rdquo","\"")        self.writeData(tr,page)  def writeData(self,context,page):    '''    將最終爬去的內(nèi)容寫入文件中    :param context: 匹配好的內(nèi)容    :param page: 當(dāng)前爬去的頁(yè)碼數(shù)    '''    fileName = "di" + str(page) + "yehtml.txt"    with open(fileName, "a") as file:      file.writelines(context + "\n")if __name__ == '__main__':  n=neihanba()  n.spider()