【教程分享】[Python] 百度小说全文爬取

输入小说的ID号，该ID号可在

中查看

默认存放路径D:/小说

import time,os
import requests
import asyncio
import aiohttp
import aiofiles
 
async def get_chapters_ids(n_id):
    book_url = f'https://dushu.baidu.com/api/pc/getCatalog?data=%7B"book_id":{n_id}%7D'
    t_start = int(time.time())
    tasks =[]
    with requests.get(book_url) as respon:
        dic = respon.json()
        for i in dic['data']['novel']['items']:
            title = i['title']
            chapter_id = i['cid']
            tasks.append(asyncio.create_task(get_chapters(n_id,title,chapter_id)))
        await asyncio.wait(tasks)
    t_over = int(time.time())
    print('下载完毕!')
    print('共用时:',t_over-t_start,'秒')
 
async def get_chapters(n_id,title,chapter_id):
    chapter_url = f'https://dushu.baidu.com/api/pc/getChapterContent?data=%7B"book_id":"{n_id}","cid":"{n_id}|{chapter_id}","need_bookinfo":1%7D'
    # print(chapter_url)
    async with aiohttp.ClientSession() as req:
        async with req.get(chapter_url) as respon:
            dic = await respon.json()
            async with aiofiles.open(f'D:\小说\{title}.txt',mode='w',encoding='utf-8') as f:
                await f.write(dic['data']['novel']['content'])
    print(title,'下载完成')
 
if __name__ =='__main__':
    if not os.path.exists(r'd:\小说'):
        os.mkdir(r'd:\小说')
    novel_id = input('输入小说编号:')
    loop = asyncio.new_event_loop()
    asyncio.set_event_loop(loop)
    loop.run_until_complete(get_chapters_ids(novel_id))

感谢您的来访，获取更多精彩文章请收藏本站。