1 """中国天气网爬虫"""
2
3 import requests
4 from bs4 import BeautifulSoup
5
6 HEADERS = {
7 'User-Agent': 'Mozilla/5.0',
8 }
9
10 def parse_detail_page(url, is_html5lib):
11 """爬取具体页面具体数据"""
12
13 respose = requests.get(url, headers=HEADERS)
14 text = respose.content.decode('utf-8')
15 # with open('weather.html', 'w', encoding='utf-8') as fp:
16 # fp.write(text)
17 if is_html5lib == False:
18 soup = BeautifulSoup(text, 'lxml')
19 else:
20 soup = BeautifulSoup(text, 'html5lib')
21 # 以下为具体爬取数据方法
22 conMidtab = soup.find_all('div', attrs={'class':'conMidtab'})
23 tables = conMidtab[0].find_all('table')
24 for table in tables:
25 trs = table.find_all('tr')[2:]
26 for index,tr in enumerate(trs):
27 tds = tr.find_all('td')
28 city_td = tds[0]
29 if index == 0:
30 city_td = tds[1]
31 city = list(city_td.stripped_strings)[0]
32 temp_td = tds[-2]
33 min_temp = list(temp_td.stripped_strings)[0]
34 # 输出城市及其最低温度
35 print({'city': city, 'min_temp': min_temp})
36
37 print("="*40)
38
39 def get_detail_urls(url, base_url):
40 """得到华北、东北、华东、华中、华南、西北、西南、港澳台的具体页面链接"""
41
42 urllists = [] # 具体的页面信息列表
43 respose = requests.get(url, headers=HEADERS)
44 text = respose.content.decode('utf-8')
45 soup = BeautifulSoup(text, 'lxml')
46 # 数据爬取
47 uls = soup.find_all('ul', class_='lq_contentboxTab2')
48 alists = uls[0].find_all('a')
49 for list in alists:
50 newurl = base_url + list['href']
51 urllists.append(newurl)
52
53 return urllists
54
55 def spider():
56 """"""
57
58 # 初始爬取页面
59 src_url = "http://www.weather.com.cn/textFC/hb.shtml"
60 base_url = "http://www.weather.com.cn"
61 urllists = []
62 urllists = get_detail_urls(src_url, base_url)
63 #print(urllists)
64 is_html5lib = False # 爬取页面是否用html5lib库
65 for index,urllist in enumerate(urllists):
66 if index != len(urllists)-1:
67 parse_detail_page(urllist, is_html5lib)
68 else:
69 is_html5lib = True
70 # url = "http://www.weather.com.cn/textFC/gat.shtml"这个页面需要用html5lib库解析，不然数据有错
71 parse_detail_page(urllist, is_html5lib)
72
73 if __name__ == '__main__':
74 spider()
