原理先行
作為一個(gè)資深的小說(shuō)愛(ài)好者，國(guó)內(nèi)很多小說(shuō)網(wǎng)站如出一轍，什么 🖊*閣啊等等，大都是 get 請(qǐng)求返回 html 內(nèi)容，而且會(huì)有標(biāo)志性的dl>dd>等標(biāo)簽。
所以大概的原理，就是先 get 請(qǐng)求這個(gè)網(wǎng)站，然后對(duì)獲取的內(nèi)容進(jìn)行清洗，寫(xiě)進(jìn)文本里面，變成一個(gè) txt，導(dǎo)入手機(jī)，方便看小說(shuō)。
實(shí)踐篇
之前踩過(guò)一個(gè)坑，一開(kāi)始我看了幾頁(yè)小說(shuō)，大概小說(shuō)的內(nèi)容網(wǎng)站是https://www.xxx.com/小說(shuō)編號(hào)/章節(jié)編號(hào).html，一開(kāi)始看前幾章，我發(fā)現(xiàn)章節(jié)編號(hào)是連續(xù)的， 于是我一開(kāi)始想的就是記住起始章節(jié)編號(hào)，然后在循環(huán)的時(shí)候章節(jié)編號(hào)自增就行，后面發(fā)現(xiàn)草率了，可能看個(gè) 100 章之后，章節(jié)列表會(huì)出現(xiàn)斷層現(xiàn)象，這個(gè)具體為啥 還真不知道，按理說(shuō)小說(shuō)編號(hào)固定，可以算是一個(gè)數(shù)據(jù)表，那里面的章節(jié)編號(hào)不就是一個(gè)自增 id 就完了嘛？有懂王可以科普一下！
所以這里要先獲取小說(shuō)的目錄列表，并把目錄列表洗成一個(gè)數(shù)組方便我們后期查找！getList.py文件：
定義一個(gè)請(qǐng)求書(shū)簽的方法
# 請(qǐng)求書(shū)簽地址
def req():
url = "https://www.24kwx.com/book/4/4020/"
strHtml = requests.get(url)
return strHtml.text
將獲取到的內(nèi)容提取出（id:唯一值/或第 X 章小說(shuō)）(name:小說(shuō)的章節(jié)名稱)（key:小說(shuō)的章節(jié) id）
# 定義一個(gè)章節(jié)對(duì)象
class Xs(object):
def __init__(self,id,key,name):
self._id = id
self._key = key
self._name = name
@property
def id(self):
self._id
@property
def key(self):
self._key
@property
def name(self):
self._name
def getString(self):
return 'id:%s,name:%s,key:%s' %(self._id,self._name,self._key)
# 轉(zhuǎn)換成書(shū)列表
def tranceList():
key = 0
name = ""
xsList = []
idrule = r'/4020/(.+?).html'
keyrule = r'第(.+?)章'
html = req()
html = re.split("/dt>",html)[2]
html = re.split("/dl>",html)[0]
htmlList = re.split("/dd>",html)
for i in htmlList:
i = i.strip()
if(i):
# 獲取id
id = re.findall(idrule,i)[0]
lsKeyList = re.findall(keyrule,i)
# 如果有章節(jié)
if len(lsKeyList) > 0 :
key = int(lsKeyList[0])
lsname = re.findall(r'章(.+?)/a>',i)
else :
key = key + 1
# 獲取名字
# lsname = re.findall(r'.html">(.+?)/a>',i)[0]
# name = re.sub('，',' ', lsname, flags=re.IGNORECASE)
name = re.findall(r'.html">(.+?)/a>',i)[0]
xsobj = Xs(id,key,name)
xsList.append(xsobj.getString())
writeList(xsList)
注意一下我：如果你從別的語(yǔ)言轉(zhuǎn) py，第一次寫(xiě)object對(duì)象可能會(huì)比較懵，沒(méi)錯(cuò)因?yàn)樗膐bject是一個(gè)class，這里我創(chuàng)建的對(duì)象就是{id,key,name}但是你寫(xiě)入 txt 的時(shí)候還是要getString，所以后面想想我直接寫(xiě)個(gè){id:xxx,name:xxx,key:xxx}的字符串不就完了，還弄啥class,后面還是想想給兄弟盟留點(diǎn)看點(diǎn)，就留著了
最后寫(xiě)入 txt 文件
# 寫(xiě)入到文本
def writeList(list):
f = open("xsList.txt",'w',encoding='utf-8')
# 這里不能寫(xiě)list，要先轉(zhuǎn)字符串 TypeError: write() argument must be str, not list
f.write('\n'.join(list))
print('寫(xiě)入成功')
# 大概寫(xiě)完的txt是這樣的
id:3798160,name:第1章 孫子，我是你爺爺,key:1
id:3798161,name:第2章 孫子，等等我！,key:2
id:3798162,name:第3章 天上掉下個(gè)親爺爺,key:3
id:3798163,name:第4章 超級(jí)大客戶,key:4
id:3798164,name:第5章 一張退婚證明,key:5
ok ! Last one
這里已經(jīng)寫(xiě)好了小說(shuō)的目錄，那我們就要讀取小說(shuō)的內(nèi)容，同理
先寫(xiě)個(gè)請(qǐng)求
# 請(qǐng)求內(nèi)容地址
def req(id):
url = "https://www.24kwx.com/book/4/4020/"+id+".html"
strHtml = requests.get(url)
return strHtml.text
讀取我們剛剛保存的目錄
def getList():
f = open("xsList.txt",'r', encoding='utf-8')
# 這里按行讀取,讀取完后line是個(gè)數(shù)組
line = f.readlines()
f.close()
return line
定義好一個(gè)清洗數(shù)據(jù)的規(guī)則
contextRule = r'div class="content">(.+?)script>downByJs();/script>'
titleRule = r'h1>(.+?)/h1>'
def getcontext(objstr):
xsobj = re.split(",",objstr)
id = re.split("id:",xsobj[0])[1]
name = re.split("name:",xsobj[1])[1]
html = req(id)
lstitle = re.findall(titleRule,html)
title = lstitle[0] if len(lstitle) > 0 else name
context = re.split('div id="content" class="showtxt">',html)[1]
context = re.split('/div>',context)[0]
context = re.sub('nbsp;|\r|\n','',context)
textList = re.split('br />',context)
textList.insert(0,title)
for item in textList :
writeTxt(item)
print('%s--寫(xiě)入成功'%(title))
再寫(xiě)入文件
def writeTxt(txt):
if txt :
f = open("nr.txt",'a',encoding="utf-8")
f.write(txt+'\n')
最后當(dāng)然是串聯(lián)起來(lái)啦
def getTxt():
# 默認(rèn)參數(shù)配置
startNum = 1261 # 起始章節(jié)
endNum = 1300 # 結(jié)束章節(jié)
# 開(kāi)始主程序
f = open("nr.txt",'w',encoding='utf-8')
f.write("")
if endNum startNum:
print('結(jié)束條數(shù)必須大于開(kāi)始條數(shù)')
return
allList = getList()
needList = allList[startNum-1:endNum]
for item in needList:
getcontext(item)
time.sleep(0.2)
print("全部爬取完成")
完整代碼
getList.py
import requests
import re
# 請(qǐng)求書(shū)簽地址
def req():
url = "https://www.24kwx.com/book/4/4020/"
strHtml = requests.get(url)
return strHtml.text
# 定義一個(gè)章節(jié)對(duì)象
class Xs(object):
def __init__(self,id,key,name):
self._id = id
self._key = key
self._name = name
@property
def id(self):
self._id
@property
def key(self):
self._key
@property
def name(self):
self._name
def getString(self):
return 'id:%s,name:%s,key:%s' %(self._id,self._name,self._key)
# 轉(zhuǎn)換成書(shū)列表
def tranceList():
key = 0
name = ""
xsList = []
idrule = r'/4020/(.+?).html'
keyrule = r'第(.+?)章'
html = req()
html = re.split("/dt>",html)[2]
html = re.split("/dl>",html)[0]
htmlList = re.split("/dd>",html)
for i in htmlList:
i = i.strip()
if(i):
# 獲取id
id = re.findall(idrule,i)[0]
lsKeyList = re.findall(keyrule,i)
# 如果有章節(jié)
if len(lsKeyList) > 0 :
key = int(lsKeyList[0])
lsname = re.findall(r'章(.+?)/a>',i)
else :
key = key + 1
# 獲取名字
# lsname = re.findall(r'.html">(.+?)/a>',i)[0]
# name = re.sub('，',' ', lsname, flags=re.IGNORECASE)
name = re.findall(r'.html">(.+?)/a>',i)[0]
xsobj = Xs(id,key,name)
xsList.append(xsobj.getString())
writeList(xsList)
# 寫(xiě)入到文本
def writeList(list):
f = open("xsList.txt",'w',encoding='utf-8')
# 這里不能寫(xiě)list，要先轉(zhuǎn)字符串 TypeError: write() argument must be str, not list
f.write('\n'.join(list))
print('寫(xiě)入成功')
def main():
tranceList()
if __name__ == '__main__':
main()
writeTxt.py
import requests
import re
import time
# 請(qǐng)求內(nèi)容地址
def req(id):
url = "https://www.24kwx.com/book/4/4020/"+id+".html"
strHtml = requests.get(url)
return strHtml.text
def getList():
f = open("xsList.txt",'r', encoding='utf-8')
# 這里按行讀取
line = f.readlines()
f.close()
return line
contextRule = r'div class="content">(.+?)script>downByJs();/script>'
titleRule = r'h1>(.+?)/h1>'
def getcontext(objstr):
xsobj = re.split(",",objstr)
id = re.split("id:",xsobj[0])[1]
name = re.split("name:",xsobj[1])[1]
html = req(id)
lstitle = re.findall(titleRule,html)
title = lstitle[0] if len(lstitle) > 0 else name
context = re.split('div id="content" class="showtxt">',html)[1]
context = re.split('/div>',context)[0]
context = re.sub('nbsp;|\r|\n','',context)
textList = re.split('br />',context)
textList.insert(0,title)
for item in textList :
writeTxt(item)
print('%s--寫(xiě)入成功'%(title))
def writeTxt(txt):
if txt :
f = open("nr.txt",'a',encoding="utf-8")
f.write(txt+'\n')
def getTxt():
# 默認(rèn)參數(shù)配置
startNum = 1261 # 起始章節(jié)
endNum = 1300 # 結(jié)束章節(jié)
# 開(kāi)始主程序
f = open("nr.txt",'w',encoding='utf-8')
f.write("")
if endNum startNum:
print('結(jié)束條數(shù)必須大于開(kāi)始條數(shù)')
return
allList = getList()
needList = allList[startNum-1:endNum]
for item in needList:
getcontext(item)
time.sleep(0.2)
print("全部爬取完成")
def main():
getTxt()
if __name__ == "__main__":
main()
以上就是python 爬取國(guó)內(nèi)小說(shuō)網(wǎng)站的詳細(xì)內(nèi)容，更多關(guān)于python 爬取小說(shuō)網(wǎng)站的資料請(qǐng)關(guān)注腳本之家其它相關(guān)文章！
您可能感興趣的文章:- Python爬蟲(chóng)入門(mén)教程02之筆趣閣小說(shuō)爬取
- python 爬取小說(shuō)并下載的示例
- python爬取”頂點(diǎn)小說(shuō)網(wǎng)“《純陽(yáng)劍尊》的示例代碼
- Python爬取365好書(shū)中小說(shuō)代碼實(shí)例
- Python實(shí)現(xiàn)的爬取小說(shuō)爬蟲(chóng)功能示例
- Python scrapy爬取起點(diǎn)中文網(wǎng)小說(shuō)榜單
- python爬蟲(chóng)之爬取筆趣閣小說(shuō)升級(jí)版
