from pyquery import PyQuery as pq
import os
from queue import Queue
from threading import Thread
class txtparser(Thread):
def __init__(self,queue):
Thread.__init__(self)
self.queue = queue
#文件夹目录
def run(self):
#path = "E:\辰东\ZheTian\\395020.html"
while True:
content = self.queue.get()
html=""
try:
with open (content,"r",encoding='utf-8') as reader:
html = reader.read()
except Exception:
with open (content,"r") as reader:
html = reader.read()
#print(html)
try:
doc = pq(html)
title = doc("#main .content_read .box_con .bookname h1")
print("标题=====",title.text())
clipname = content.split("\\")[-2]
#junkp = doc(".content").find('p').remove()
passage = doc("#content").text()
except Exception:
continue
print("正文======",str.replace(passage,"<br/>",""))
try:
clipname = str.replace(clipname,"《","")
clipname = str.replace(clipname,"》","")
except Exception:
clipname = clipname
if os.path.exists(clipname):
pass
else:
os.mkdir(clipname)
try:
with open(clipname+"\\"+title.text()+".txt","w",encoding="gbk") as writer:
writer.write(passage)
print("完成{}的写入".format(clipname+"\\"+title.text()+".txt"))
except Exception:
with open("errorecorder.log","a") as writer:
writer.write(clipname+"\\"+title.text()+".txt"+"\r")
print("文件夹名称======",clipname)
def launchtxtparser(parentdir):
rootdir = parentdir
queue = Queue()
print(rootdir)
for i in os.listdir(rootdir):
print(i)
if os.path.isdir(rootdir+"\\"+i):
print(rootdir+"\\"+i)
g = (k for k in os.listdir(rootdir+"\\"+i))
print(next(g))
while True:
try:
filename = next(g)
fullfilename = rootdir+"\\"+i+"\\"+filename
queue.put(fullfilename)
print(fullfilename)
except StopIteration:
print("ooooophs~处理完毕")
break
for i in range(10):
cpc = txtparser(queue)
cpc.daemon=True
cpc.start()
queue.join()
#print(os.listdir(rootdir))
launchtxtparser("E:\月关")
