python抓取网页例子

python抓取网页例子

最近在学习python，刚刚完成了一个网页抓取的例子，通过python抓取全世界所有的学校以及学院的数据，并存为xml文件。数据源是人人网。

因为刚学习python，写的代码还不够Pythonic。

核心代码如下：

#!/usr/bin/python


import urllib.request
from html.parser import HTMLParser  
import json
import time
import xml.dom.minidom
import os


class Dept():
    id = 0
    name = ''
    
class University(Dept):
    depts = []

class City(Dept):
    universities  = []    
    
class Country(Dept):
    cities = []

class MyHtmlParser(HTMLParser):
    def __init__(self):   
        HTMLParser.__init__(self)   
        self.links = []  
        self.depts = [] 
    def handle_starttag(self, tag, attrs):   
        if tag == 'option':
            for att in attrs:
                for a in att:
                    if a != 'value' and a != '':
                        self.depts.append(a)
def readDept(code):
    depts = []
    html = ''
    for word in urllib.request.urlopen('http://www.renren.com/GetDep.do?id=' + str(code)).readlines():
        real = word.strip().decode('gbk') 
        html =  html  + real
    hp = MyHtmlParser()   
    hp.feed(html)  
    for inst in hp.depts:
        dept = Dept()
        dept.name = inst
        depts.append(dept) 
    return depts

def writeXml(city):
    impl = xml.dom.minidom.getDOMImplementation()
    dom = impl.createDocument(None, 'city', None)
    root = dom.documentElement  
    filename = city.name + '.xml'
    if os.path.isfile(filename): 
        os.remove(filename)
    nameE = dom.createElement('name')
    nameT = dom.createTextNode(city.name)
    idE = dom.createElement('id')
    idT = dom.createTextNode(str(city.id))
    nameE.appendChild(nameT)
    idE.appendChild(idT)
    root.appendChild(nameE)
    root.appendChild(idE)
    univs = dom.createElement('universities')
    root.appendChild(univs)
    for uni in city.universities:
#        print('write xml' + city.name + '\t' + uni.name)
        universityE = dom.createElement('university')
        univs.appendChild(universityE)
        uniE = dom.createElement('name')
        uniT = dom.createTextNode(uni.name)
        uidE = dom.createElement('id')
        uidT = dom.createTextNode(str(uni.id))
        uniE.appendChild(uniT)
        uidE.appendChild(uidT)
        universityE.appendChild(uniE)
        universityE.appendChild(uidE)
        deptsE = dom.createElement('departments')
        universityE.appendChild(deptsE)
        for dep in uni.depts:
            deptE = dom.createElement('department')
            deptsE.appendChild(deptE)
            deptNameE = dom.createElement('name')
            deptIdE = dom.createElement('id')
            deptT = dom.createTextNode(dep.name)
            deptIdT = dom.createTextNode(str(dep.id))
            deptNameE.appendChild(deptT)
            deptIdE.appendChild(deptIdT)
            deptE.appendChild(deptNameE)
            
    f= open(filename, 'w', encoding='utf-8')
    dom.writexml(f, addindent='  ', newl='\n',encoding='utf-8')
    print('write xml :' + city.name + '.xml')
    f.close()  
    

def mkdir(path):

    path=path.strip()
    path=path.rstrip("/")
    isExists=os.path.exists(path)
    if not isExists:
        os.makedirs(path)

def readData(content):
    counties = []
    jdata = json.loads(content)
    for i in range(0,100):
        try:
            country = Country()
            country.name = jdata[i]['name'] 
            country.id = jdata[i]['id'] 
            provs = jdata[i]['provs'] 
            for prov in provs:
                city = City()
                city.name = prov['name']
                city.id = prov['id']
                country.cities.append(city)
                city.universities = []
                for dic in prov['univs']:
                    university = University() 
                    university.id = dic['id']
                    university.name = dic['name']
#                    print('get data: \t' + university.name)
                    university.depts = readDept(university.id)
                    city.universities.append(university)
                    print('city = ' + city.name + '\tuniversity = ' + university.name)
                writeXml(city)
            counties.append(country)
        except IndexError:
            break;
    return counties


print('开始时间：' + time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time())))
f = open('data','r' )
content = f.read()
f.close()
counties = readData(content)
print('结束时间：' + time.strftime('%Y-%m-%d %H:%M:%S',time.localtime(time.time())))

其中data是从如下网站拿到的

http://s.xnimg.cn/allunivlist.js