1 # -*- coding:utf-8 -*-
2 '''
3 清水河畔二手帖子+爬取二手交易帖子
4 '''
5 import requests
6 import json
7 from bs4 import BeautifulSoup
8 import sys
9 import urllib
10 import re
11 from urllib import request,parse
12 import pymongo
13 #by 元帅 uestc 2018.2.28
14 class QSHSpider(object):
15 # 模拟登陆清水河畔
16 def __init__(self):
17 self.headers = {
18 'username':'',
19 'password':'',
20 'Cache - Control': '',
21 'Connection': 'keep - alive',
22 'Cookie':'',
23 'Host':'bbs.uestc.edu.cn',
24 'Referer':'http: // bbs.uestc.edu.cn / member.php?mod = logging & action = login',
25 'Upgrade-Insecure - Requests': '1',
26 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'
27 }
28
29 # 模拟登陆
30 def login(self):
31 request_url = "http://bbs.uestc.edu.cn/member.php?mod=logging&action=login"
32 request_new = requests.get(request_url, headers=self.headers)
33 self.login_request = request_new.text
34
35 def get_shopurl(self):
36 bf = BeautifulSoup(self.login_request,'html.parser')
37 #print(bf)
38 shop_a = bf.find_all('div',id = 'hd')[0].find_all('ul',id = 'mn_F201_menu',class_='p_pop h_pop')[0].find_all('li')[0].find_all('a')[0]
39 self.shopurl = shop_a['href']
40 print('登陆成功！')
41 print('\n')
42 print('您已进入二手帖子专题：' + self.shopurl)
43
44 def get_tieziurls(self):
45 #request_tiezi = requests.get(self.shopurl,headers = self.headers)
46 req = request.Request(url=self.shopurl,headers=self.headers, method="POST")
47 response = request.urlopen(req)
48 content = response.read()
49 res = r"<a.*?href=.*?<\/a>"
50 urls = re.findall(res, content.decode('utf-8'))
51 print('备选主题有：书籍资料；生活用品；交通工具；卡券虚拟；数码硬件；'
52 '拼单；物品租借；其他；版务/投诉；已解决；')
53 searcher = input("请输入需要查找的主题 ")
54 #获取a标签内内容
55 # res = r'<a .*?>(.*?)</a>'
56 # texts = re.findall(res, content.decode('utf-8'))
57 # for t in texts:
58 # print(t)
59 #获取a标签内超链接
60 #urls = re.findall(r'<a.*?href=.*?>\r\n(.+?)<span class="xg1 num">(.*?)</span><\/a>',re.S)
61 #bff = BeautifulSoup(request_tiezi.text,'html.parser')
62 #print(bff)
63 #shop_tc1 = bff.find_all('div',id = 'wp',class_ = 'wp')[0]
64 #.find_all('div',class_ = 'boardnav')[0].find_all('div',id = 'ct',class_ = 'wp cl')[0].find_all('div',class_ = 'mn')
65 #shop_tc1 = bff.select('.wp')
66 #print(shop_tc1)
67 #.find_all('ul',id = 'thread_types',class_ = 'ttp bm cl cttp',style = 'height: auto;')[0]
68 #.find_all('li').find_all('a')[0]
69 # shop_tiezi_url = shop_tc1['href']
70 # print(shop_tiezi_url)
71 for u in urls:
72 if searcher in str(u):
73 ui = u.replace('amp;','')
74 #print(ui)
75 urls1 = re.findall(r'<a .*?>(.*?)</a>' , ui, re.I | re.S | re.M)
76 urls2 = re.findall(r"(?<=href=\").+?(?=\")|(?<=href=\').+?(?=\')" ,ui,re.S |re.I |re.M)
77 print(urls1[0][0:8] + ' ' + urls2[0])
78 self.tieziurls = urls2[0]
79 tags = re.findall(r"\d+\.?\d*",urls2[0])
80 self.pagetag = tags[1]
81 #self.pagetag = re.sub("\D", "", urls2[0])
82 #print(self.pagetag)
83 break
84 def get_tiezi(self):
85 #req_tiezi = request.Request(url=self.tieziurls,headers=self.headers, method="POST")
86 req_tiezi = request.Request(url=self.new_url, headers=self.headers, method="POST")
87 response_tiezi = request.urlopen(req_tiezi)
88 content_tiezi = response_tiezi.read()
89 res_tiezi = r"<a.*?href=.*?<\/a>"
90 urls_tiezi = re.findall(res_tiezi, content_tiezi.decode('utf-8'))
91 for tiezi in urls_tiezi:
92 if 'class="s xst"' in tiezi:
93 tiezi = tiezi.replace('amp;', '')
94 tiezi_title = re.findall(r'<a .*?>(.*?)</a>', tiezi, re.I | re.S | re.M)
95 tiezi_urls = re.findall(r"(?<=href=\").+?(?=\")|(?<=href=\').+?(?=\')" ,tiezi,re.S |re.I |re.M)
96 print('\n\n')
97 print('帖子主题:' +tiezi_title[0] + ' ' + '帖子地址:' + tiezi_urls[0])
98 self.tiezi_url = tiezi_urls[0]
99 infor.get_details()
100 #self.tiezi_data[] = '帖子地址'
101 #数据库
102 tiezi_data = {'帖子' : {'title': tiezi_title[0], 'url' : tiezi_urls[0]}}
103 client = pymongo.MongoClient('localhost',27017)
104 mydb = client['mydb']
105 qingshuihepan = mydb['qingshuihepan']
106 qingshuihepan.insert_one(tiezi_data)
107 def get_pages(self):
108 urls_based = 'http://bbs.uestc.edu.cn/forum.php?mod=forumdisplay&fid=61&typeid={}&filter=typeid&typeid={}&page={}'
109 for i in range(1,3):
110 self.new_url = urls_based.format(self.pagetag,self.pagetag,i)
111 print('\n\n')
112 print('第'+ str(i) + '页' + ' ' '本页网址：' + self.new_url)
113 infor.get_tiezi()
114 def get_details(self):
115 #print(self.tiezi_url)
116 print('本帖详细内容：')
117 req_detail = request.Request(url=self.tiezi_url, headers=self.headers, method="POST")
118 response_detail = request.urlopen(req_detail)
119 content_detail = response_detail.read()
120 #print(content_detail.decode('utf-8'))
121 bs = BeautifulSoup(content_detail,'html.parser')
122 print(bs.find_all(class_='t_f')[0].text.strip())
123 #urls_detail = re.findall(r'<td class="t_f".*?>(.*?)</td>' , content_detail.decode('utf-8'))
124 # for ud in urls_detail:
125 # print(ud)
126
127 if "__main__" == __name__:
128 infor = QSHSpider()
129 infor.login()
130 print('登陆清水河畔ing……')
131 infor.get_shopurl()
132 infor.get_tieziurls()
133 #infor.get_tiezi()
134 infor.get_pages()
