通过python自动获取小说并下载

最近在学习Python，为了学以致用，就用python来写一个小程序，用来获取静态网页中的小说，代码见下：

import urllib.request
import os

headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.163 "
                  "Safari/535.1",
}


# 主程序，获取每个章节的名称及链接
def main():
    print('开始下载')
    url_list = []
    url = 'http://www.eywedu.com/honglou/01/index.htm'
    res = urllib.request.Request(url, data=None, headers=headers)
    html = urllib.request.urlopen(res).read().decode('gb18030')
    lists = html.split('<A HREF="')
    for i in lists:
        if '第' in i:
            s = i.split(r'</A><BR>')  # 以/a br作字符分割
            s1 = ' '.join(s[0].split())  # 取消多余空格
            url_list.append(s1)
    handle_url(url_list)


# 处理获取的链接，得到每章名称和准确链接
def handle_url(urls):
    if not os.path.exists('红楼梦'):
        os.makedirs('红楼梦')
    net = 'http://www.eywedu.com/honglou/01/'
    page = 1
    for i in urls:
        cut = i.split('" >')
        get_content(net+cut[0], cut[1])
        print('第', page, '章已下完')
        page = page+1


# 根据每章链接获得内容
def get_content(url, name):
    paragraphs = []
    res = urllib.request.Request(url, data=None, headers=headers)
    html = urllib.request.urlopen(res).read().decode('gb18030')
    lists = html.split('<BR>')
    t = 0
    while t < len(lists):
        if t == 0:
            p = lists[t].split(r'2014newad.js"></script></DIV>')[1]
            p = p.replace('\r\n', '')
            paragraphs.append(p)
        elif t == len(lists) - 1:
            p = lists[t].split(r'<!--/HTMLBUILERPART0-->')[0]
            p = p.replace('\r\n', '')
            paragraphs.append(p)
        else:
            p = lists[t].replace('\r\n', '')
            paragraphs.append(p)
        t = t + 1
    save_txt(paragraphs, name)


# 将得到的信息保存
def save_txt(content, name):
    f = open('红楼梦\\'+name+'.txt', "a", encoding='utf-8')
    for i in content:
        f.write(i)
    f.close()


if __name__ == '__main__':
    main()

 

能够成功运行下载：