手把手教你使用Python網(wǎng)絡(luò)爬蟲(chóng)下載一本小說(shuō)（附源碼）

#?coding: utf-8'''筆趣網(wǎng)小說(shuō)下載僅限用于研究代碼勿用于商業(yè)用途請(qǐng)于24小時(shí)內(nèi)刪除'''import requestsimport osfrom bs4 import BeautifulSoupimport time

def book_page_list(book_id):    '''    通過(guò)傳入的書(shū)號(hào)bookid，獲取此書(shū)的所有章節(jié)目錄    :param book_id:    :return: 章節(jié)目錄及章節(jié)地址    '''    url = 'http://www.biquw.com/book/{}/'.format(book_id)    headers = {        'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.71 Safari/537.36'}    response = requests.get(url, headers)    response.encoding = response.apparent_encoding    response = BeautifulSoup(response.text, 'lxml')    booklist = response.find('div', class_='book_list').find_all('a')    return booklist

def book_page_text(bookid, booklist):    '''    通過(guò)書(shū)號(hào)、章節(jié)目錄，抓取每一章的內(nèi)容并存檔    :param bookid:str    :param booklist:    :return:None    '''    try:        for book_page in booklist:            page_name = book_page.text.replace('*', '')            page_id = book_page['href']            time.sleep(3)            url = 'http://www.biquw.com/book/{}/{}'.format(bookid,page_id)            headers = {                'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.71 Safari/537.36'}            response_book = requests.get(url, headers)            response_book.encoding = response_book.apparent_encoding            response_book = BeautifulSoup(response_book.text, 'lxml')            book_content = response_book.find('div', id="htmlContent")            with open("./{}/{}.txt".format(bookid,page_name), 'a') as f:                f.write(book_content.text.replace('\xa0', ''))                print("當(dāng)前下載章節(jié)：{}".format(page_name))    except Exception as e:        print(e)        print("章節(jié)內(nèi)容獲取失敗，請(qǐng)確保書(shū)號(hào)正確，及書(shū)本有正常內(nèi)容。")

if __name__ == '__main__':    bookid = input("請(qǐng)輸入書(shū)號(hào)(數(shù)字)：")    # 如果書(shū)號(hào)對(duì)應(yīng)的目錄不存在，則新建目錄，用于存放章節(jié)內(nèi)容    if not os.path.isdir('./{}'.format(bookid)):        os.mkdir('./{}'.format(bookid))    try:        booklist = book_page_list(bookid)        print("獲取目錄成功！")        time.sleep(5)        book_page_text(bookid, booklist)    except Exception as e:        print(e)????????print("獲取目錄失敗，請(qǐng)確保書(shū)號(hào)輸入正確！")