三個(gè)python爬蟲(chóng)項(xiàng)目實(shí)例代碼

#encoding=utf-8import urllib2 import re  class neihanba():  def spider(self):    '''    爬蟲(chóng)的主調(diào)度器    '''    isflow=True#判斷是否進(jìn)行下一頁(yè)    page=1    while isflow:      url="http://www.neihanpa.com/article/list_5_"+str(page)+".html"      html=self.load(url)      self.deal(html,page)      panduan=raw_input("是否繼續(xù)(y/n)!")      if panduan=="y":        isflow=True        page+=1      else:        isflow=False  def load(self,url):    '''    針對(duì)url地址進(jìn)行全部爬去    :param url: url地址    :return: 返回爬去的內(nèi)容    '''    header = {      "User-Agent": " Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.79 Safari/537.36"    }    request = urllib2.Request(url, headers=header)    response = urllib2.urlopen(request)    html = response.read()    return html  def deal(self,html,page):    '''    對(duì)之前爬去的內(nèi)容進(jìn)行正則匹配，匹配出標(biāo)題和正文內(nèi)容    :param html:之前爬去的內(nèi)容    :param page: 正在爬去的頁(yè)碼    '''    parrten=re.compile('<li class="piclist\d+">(.*?)</li>',re.S)    titleList=parrten.findall(html)    for title in titleList:      parrten1=re.compile('<a href="/article/\d+.html" rel="external nofollow" >(.*)</a>')      ti1=parrten1.findall(title)      parrten2=re.compile('<div class="f18 mb20">(.*?)</div>',re.S)      til2=parrten2.findall(title)      for t in ti1:        tr=t.replace("<b>","").replace("</b>","")        self.writeData(tr,page)      for t in til2:        tr=t.replace("<p>","").replace("</p>","").replace("<br>","").replace("<br />","").replace("&ldquo","\"").replace("&rdquo","\"")        self.writeData(tr,page)  def writeData(self,context,page):    '''    將最終爬去的內(nèi)容寫入文件中    :param context: 匹配好的內(nèi)容    :param page: 當(dāng)前爬去的頁(yè)碼數(shù)    '''    fileName = "di" + str(page) + "yehtml.txt"    with open(fileName, "a") as file:      file.writelines(context + "\n")if __name__ == '__main__':  n=neihanba()  n.spider()

#encoding=utf-8import urllibimport urllib2 import re  class zhiLian():  def spider(self,position,workPlace):    '''    爬蟲(chóng)的主調(diào)度器    :param position: 職位    :param workPlace: 工作地點(diǎn)    '''    url="http://sou.zhaopin.com/jobs/searchresult.ashx?"    url+=urllib.urlencode({"jl":workPlace})    url+="&"    url+=urllib.urlencode({"kw":position})    isflow=True#是否進(jìn)行下一頁(yè)的爬去    page=1    while isflow:      url+="&"+str(page)      html=self.load(url)      self.deal1(html,page)      panduan = raw_input("是否繼續(xù)爬蟲(chóng)下一頁(yè)(y/n)!")      if panduan == "y":        isflow = True        page += 1      else:        isflow = False  def load(self,url):    '''    針對(duì)url地址進(jìn)行全部爬去    :param url: url地址    :return: 返回爬去的內(nèi)容    '''    header = {      "User-Agent": " Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.79 Safari/537.36"    }    request = urllib2.Request(url, headers=header)    response = urllib2.urlopen(request)    html = response.read()    return html  def deal1(self,html,page):    '''     對(duì)之前爬去的內(nèi)容進(jìn)行正則匹配，匹配職位所對(duì)應(yīng)的鏈接    :param html:之前爬去的內(nèi)容    :param page: 正在爬去的頁(yè)碼    '''    parrten=re.compile('<a\s+style="font-weight:\s+bold"\s+par="ssidkey=y&amp;ss=\d+&amp;ff=\d+&amp;sg=\w+&amp;so=\d+"\s+href="(.*?)" rel="external nofollow" target="_blank">.*?</a>',re.S)    til=parrten.findall(html)#爬去鏈接    for t in til:      self.deal2(t,page)  def deal2(self,t,page):    '''    進(jìn)行二次爬蟲(chóng)，然后在新的頁(yè)面中對(duì)公司、薪資、工作經(jīng)驗(yàn)進(jìn)行匹配    :param t: url地址    :param page: 當(dāng)前匹配的頁(yè)數(shù)    '''    html=self.load(t)#返回二次爬蟲(chóng)的內(nèi)容    parrten1=re.compile('<a\s+onclick=".*?"\s+href=".*?" rel="external nofollow" \s+target="_blank">(.*?)\s+.*?<img\s+class=".*?"\s+src=".*?"\s+border="\d+"\s+vinfo=".*?"></a>',re.S)    parrten2=re.compile('<li><span>職位月薪：</span><strong>(.*?)&nbsp;<a.*?>.*?</a></strong></li>',re.S)    parrent3=re.compile('<li><span>工作經(jīng)驗(yàn)：</span><strong>(.*?)</strong></li>',re.S)    til1=parrten1.findall(html)    til2=parrten2.findall(html)    til3=parrent3.findall(html)    str=""    for t in til1:      t=t.replace('<img title="專屬頁(yè)面" src="http://img03.zhaopin.cn/2012/img/jobs/icon.png" border="0" />',"")      str+=t      str+="\t"    for t in til2:      str+=t      str += "\t"    for t in til3:      str+=t    self.writeData(str,page)  def writeData(self,context,page):    '''    將最終爬去的內(nèi)容寫入文件中    :param context: 匹配好的內(nèi)容     :param page: 當(dāng)前爬去的頁(yè)碼數(shù)    '''    fileName = "di" + str(page) + "yehtml.txt"    with open(fileName, "a") as file:      file.writelines(context + "\n")if __name__ == '__main__':  position=raw_input("請(qǐng)輸入職位：")  workPlace=raw_input("請(qǐng)輸入工作地點(diǎn)：")  z=zhiLian()  z.spider(position,workPlace)

#encoding=utf-8import urllibimport urllib2 import re  class teiba():  def spider(self,name,startPage,endPage):    url="http://tieba.baidu.com/f?ie=utf-8&"    url+=urllib.urlencode({"kw":name})    for page in range(startPage,endPage+1):      pn=50*(page-1)      urlFull=url+"&"+urllib.urlencode({"pn":pn})      html=self.loadPage(url)      self.dealPage(html,page)   def loadPage(self,url):    header={      "User-Agent":" Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.79 Safari/537.36"    }    request=urllib2.Request(url,headers=header)    response=urllib2.urlopen(request)    html=response.read()    return html  def dealPage(self,html,page):    partten=re.compile(r'<a\s+rel="noreferrer"\s+href="/p/\d+" rel="external nofollow" \s+title=".*?"\s+target="_blank" class="j_th_tit\s+">(.*?)</a>',re.S)    titleList=partten.findall(html)    rstr=r'<span\s+class="topic-tag"\s+data-name=".*?">#(.*?)#</span>'    for title in titleList:      title=re.sub(rstr,"",title)      self.writePage(title,page)  def writePage(self,context,page):    fileName="di"+str(page)+"yehtml.txt"    with open(fileName,"a") as file:      file.writelines(context+"\n")if __name__ == '__main__':  name=raw_input("請(qǐng)輸入貼吧名：")  startPage=raw_input("請(qǐng)輸入起始頁(yè)：")  endPage=raw_input("請(qǐng)輸入終止頁(yè)：")  t=teiba()  t.spider(name,int(startPage),int(endPage))