Python简单爬虫爬取自己博客园所有文章

初学Python，用python写的一个简单爬虫，爬取自己博客园上面的所有文章。

#coding=utf-8
import re
import urllib

def getHtml(url):
    page = urllib.urlopen(url)
    html = page.read()
    return html

def getPage(html):
    reg = r'随笔-([0-9]+)'
    pageCount = re.findall(reg,html)
    return pageCount[0]

def getArticleUrl(html):
    reg = r'(http://www.cnblogs.com/sunniest/p/[0-9]+.html)'
    articleUrl = re.findall(reg,html)
    return articleUrl

def downloadPage(urlList):
    x = 0
    for article in urlList:
        urllib.urlretrieve(article,'%s.html' % x)
        x+=1

article = []
htmlStr = getHtml("http://www.cnblogs.com/sunniest/default.html")
pageCount = getPage(htmlStr)
page = int(pageCount)/40+1
for i in range(1,page+1):
    html = getHtml("http://www.cnblogs.com/sunniest/default.html?page="+str(i))
    articleUrl = getArticleUrl(html)
    article = article.__add__(articleUrl)

article = list(set(article))
downloadPage(article)

爬取后的网页会保存在项目的根目录下，暂时未支持js、css等文件的爬取，所以页面显示效果会比较差。