Python将段子写入数据库
#Python2.7 可以优化一下 前10页 每页点赞最多的段子 百思不得姐
# -*- coding: utf-8 -*-
import MySQLdb
import urllib,urllib2
import sys,re
reload(sys)
sys.setdefaultencoding('utf-8')
class TestBDJ():
def __init__(self):
pass
def getbdj(self):
lis = []
for h in range(1,10):
cc = []
tt = []
try:
url = 'http://www.budejie.com/text/'+str(h)
res = urllib.urlopen(url).read()
#print res
m = re.compile('<div class="j-r-list-c-desc">(.*?)</div>.*?<i class="icon-up ui-icon-up"></i> <span>(.*?)</span>',re.S) #当前版本可以这样匹配 没有匹配作者
tt = re.findall(m,res)
for i in tt:
cc.append(i[1]) #把每个段子点赞的数量放入list
aa = cc
aa = map(eval,aa) #需要将点赞的值由string转换成int
#print "befor %s",aa
for b in range(len(aa) - 1): #排序
for i in range(len(aa) - 1):
if aa[i] < aa[i + 1]:
aa[i], aa[i + 1] = aa[i + 1], aa[i]
#print "after %s",aa
for t in range(5): #取每页前5条段子
duanzi = []
index = cc.index(str(aa[t])) #将点赞的数量转回string 根据下标 获取对应的段子
#print tt[index][0].replace("<br />","")
#print tt[index][1]
star = tt[index][1] #点赞人数
word = tt[index][0].replace("<br />","").encode('utf-8') #文字内容
duanzi.append(star)
duanzi.append(word)
lis.append(duanzi)
except Exception,e:
print e
return lis
def load(self):
key = self.getbdj()
if key:
try:
conn = MySQLdb.connect(host='localhost', user='hehehe', passwd='****', db='tester', port=330,charset="utf8") # 加上charset="utf8" 指定编码格式 解决写入mysql中文乱码的问题
cur = conn.cursor()
conn.select_db('tester')
cur.execute('create table if not exists budejie(st int,info TEXT)')
for k in key:
cur.execute('insert into budejie values(%s,%s)',k)
cur.close()
conn.commit()
conn.close()
except MySQLdb.Error, e:
print "Mysql Error %d: %s" % (e.args[0], e.args[1])
return 0
if __name__=='__main__':
h = TestBDJ()
h.load()
Python 3 的需要稍微修改一下
#coding:utf-8
import urllib.request
import re
import importlib,sys
for h in range(1,10):
cc = []
tt = []
try:
url = 'http://www.budejie.com/text/'+str(h)
res = urllib.request.urlopen(url).read()
res = res.decode("utf-8")
# print (res)
m = re.compile('<div class="j-r-list-c-desc">(.*?)</div>.*?<i class="icon-up ui-icon-up"></i> <span>(.*?)</span>',re.S) #当前版本可以这样匹配 没有匹配作者
tt = re.findall(m,res)
for i in tt:
cc.append(i[1]) #把每个段子点赞的数量放入list
aa = cc
aa = list(map(eval,aa)) #需要将点赞的值由string转换成int
#print "befor %s",aa
for b in range(len(aa)-1): #排序
for i in range(len(aa) - 1):
if aa[i] < aa[i + 1]:
aa[i], aa[i + 1] = aa[i + 1], aa[i]
#print "after %s",aa
for t in range(5): #取每页前5条段子
index = cc.index(str(aa[t])) #将点赞的数量转回string 根据下标 获取对应的段子
print (tt[index][0].replace("<br />",""))
print (tt[index][1])
except Exception as e:
print (e)
糗事百科，稍稍做了一下处理
# -*- coding: utf-8 -*-
import urllib,urllib2
import sys,re
reload(sys)
sys.setdefaultencoding('utf-8')
for h in range(1,8):
cc = []
tt = []
try:
url = 'http://www.qiushibaike.com/text/page/'+str(h)+'/?s=4891212'
req = urllib2.Request(url)
req.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36')
res = urllib2.urlopen(req).read()
#print res
m = re.compile('<div class="content">(.*?)</div>.*?<span class="stats-vote"><i class="number">(.*?)</i> 好笑</span>',re.S) #当前版本可以这样匹配 没有匹配作者
tt = re.findall(m,res)
for i in tt:
cc.append(i[1]) #把每个段子点赞的数量放入list
aa = cc
aa = map(eval,aa) #需要将点赞的值由string转换成int
#print "befor %s",aa
for b in range(len(aa) - 1): #排序
for i in range(len(aa) - 1):
if aa[i] < aa[i + 1]:
aa[i], aa[i + 1] = aa[i + 1], aa[i]
#print "after %s",aa
for t in range(5): #取每页前5条段子
index = cc.index(str(aa[t])) #将点赞的数量转回string 根据下标 获取对应的段子
print tt[index][0].replace("<br/>","")
print tt[index][1]
except Exception,e: print e
爬取含图片的段子：
# -*- coding: utf-8 -*-
import urllib,urllib2
import sys,re
reload(sys)
sys.setdefaultencoding('utf-8')
for h in range(1,3):
cc = []
tt = []
try:
url = 'http://www.qiushibaike.com/imgrank/page/'+str(h)+'/?s=4891221'
req = urllib2.Request(url)
req.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36')
res = urllib2.urlopen(req).read()
#print res
m = re.compile('<div class="content">(.*?)</div>.*?target="_blank">.*?<img src=\"(.*?)\" alt=.*?<span class="stats-vote"><i class="number">(.*?)</i> 好笑</span>',re.S) #当前版本可以这样匹配 没有匹配作者
tt = re.findall(m,res)
for i in tt:
cc.append(i[2]) #把每个段子点赞的数量放入list
# print i[0]
# print i[1]
# print i[2]
aa = cc
aa = map(eval,aa) #需要将点赞的值由string转换成int
#print "befor %s",aa
for b in range(len(aa) - 1): #排序
for i in range(len(aa) - 1):
if aa[i] < aa[i + 1]:
aa[i], aa[i + 1] = aa[i + 1], aa[i]
#print "after %s",aa
for t in range(5): #取每页前5条段子
index = cc.index(str(aa[t])) #将点赞的数量转回string 根据下标 获取对应的段子
print index
print tt[index][0]
print tt[index][1]
print tt[index][2]
except Exception,e:
print e
