實(shí)戰(zhàn)|Python輕松實(shí)現(xiàn)動(dòng)態(tài)網(wǎng)頁(yè)爬蟲(附詳細(xì)源碼)

1from?urllib.parse?import?urlencode
2import?csv
3import?random
4import?requests
5import?traceback
6from?time?import?sleep
7from?lxml?import?etree????#lxml為第三方網(wǎng)頁(yè)解析庫(kù)，強(qiáng)大且速度快

 1base_url?=?'http://www.hshfy.sh.cn/shfy/gweb2017/ktgg_search_content.jsp?'??#這里要換成對(duì)應(yīng)Ajax請(qǐng)求中的鏈接
 2
 3headers?=?{
 4????'Connection':?'keep-alive',
 5????'Accept':?'*/*',
 6????'X-Requested-With':?'XMLHttpRequest',
 7????'User-Agent':?'你的User-Agent',
 8????'Origin':?'http://www.hshfy.sh.cn',
 9????'Referer':?'http://www.hshfy.sh.cn/shfy/gweb2017/ktgg_search.jsp?zd=splc',
10????'Accept-Language':?'zh-CN,zh;q=0.9',
11????'Content-Type':?'application/x-www-form-urlencoded',
12????'Cookie':?'你的Cookie'
13}

 1def?get_page(page):
 2????n?=?3
 3????while?True:
 4????????try:
 5????????????sleep(random.uniform(1,?2))??#?隨機(jī)出現(xiàn)1-2之間的數(shù)，包含小數(shù)
 6????????????data?=?{
 7????????????????'yzm':?'yxAH',
 8????????????????'ft':'',
 9????????????????'ktrqks':?'2020-05-22',
10????????????????'ktrqjs':?'2020-06-22',
11????????????????'spc':'',
12????????????????'yg':'',
13????????????????'bg':'',
14????????????????'ah':'',
15????????????????'pagesnum':?page
16????????????}
17????????????url?=?base_url?+?urlencode(data)
18????????????print(url)
19????????????try:
20????????????????response?=?requests.request("POST",url,?headers?=?headers)
21????????????????#print(response)
22????????????????if?response.status_code?==?200:
23????????????????????re?=?response.content.decode('gbk')
24????????????????????#?print(re)
25????????????????????return?re??#?解析內(nèi)容
26????????????except?requests.ConnectionError?as?e:
27????????????????print('Error',?e.args)??#?輸出異常信息
28????????except?(TimeoutError,?Exception):
29????????????n?-=?1
30????????????if?n?==?0:
31????????????????print('請(qǐng)求3次均失敗，放棄此url請(qǐng)求,檢查請(qǐng)求條件')
32????????????????return
33????????????else:
34????????????????print('請(qǐng)求失敗，重新請(qǐng)求')
35????????????????continue

 1def?parse_page(html):
 2????try:
 3????????parse?=?etree.HTML(html)??#?解析網(wǎng)頁(yè)
 4????????items?=?parse.xpath('//*[@id="report"]/tbody/tr')
 5????????for?item?in?items[1:]:
 6????????????item?=?{
 7????????????????'a':?''.join(item.xpath('./td[1]/font/text()')).strip(),
 8????????????????'b':?''.join(item.xpath('./td[2]/font/text()')).strip(),
 9????????????????'c':?''.join(item.xpath('./td[3]/text()')).strip(),
10????????????????'d':?''.join(item.xpath('./td[4]/text()')).strip(),
11????????????????'e':?''.join(item.xpath('./td[5]/text()')).strip(),
12????????????????'f':?''.join(item.xpath('./td[6]/div/text()')).strip(),
13????????????????'g':?''.join(item.xpath('./td[7]/div/text()')).strip(),
14????????????????'h':?''.join(item.xpath('./td[8]/text()')).strip(),
15????????????????'i':?''.join(item.xpath('./td[9]/text()')).strip()
16????????????}
17????????????#print(item)
18????????????try:
19????????????????with?open('./law.csv',?'a',?encoding='utf_8_sig',?newline='')?as?fp:
20????????????????????#?'a'為追加模式（添加）
21????????????????????#?utf_8_sig格式導(dǎo)出csv不亂碼
22????????????????????fieldnames?=?['a',?'b',?'c',?'d',?'e','f','g','h','i']
23????????????????????writer?=?csv.DictWriter(fp,fieldnames)
24????????????????????writer.writerow(item)
25????????????except?Exception:
26????????????????print(traceback.print_exc())??#代替print?e?來(lái)輸出詳細(xì)的異常信息
27????except?Exception:
28????????print(traceback.print_exc())

1????for?page?in?range(1,5):??#這里設(shè)置想要爬取的頁(yè)數(shù)
2????????html?=?get_page(page)
3????????#print(html)
4????????print("第"?+?str(page)?+?"頁(yè)提取完成")

1from?lxml?import?etree
2import?time
3from?selenium?import?webdriver
4from?selenium.?webdriver.support.wait?import?WebDriverWait
5from?selenium.webdriver.support?import?expected_conditions?as?EC
6from?selenium.webdriver.common.by?import?By

1def?main():
2????#?爬取首頁(yè)url
3????url?=?"http://www.hshfy.sh.cn/shfy/gweb2017/flws_list.jsp?ajlb=aYWpsYj3D8crCz"
4????#?定義谷歌webdriver
5????driver?=?webdriver.Chrome('./chromedriver')
6????driver.maximize_window()??#?將瀏覽器最大化
7????driver.get(url)