Python 爬虫实例(4)—— 爬取网易新闻
自己闲来无聊,就爬取了网易信息,重点是分析网页,使用抓包工具详细的分析网页的每个链接,数据存储在sqllite中,这里只是简单的解析了新闻页面的文字信息,并未对图片信息进行解析
仅供参考,不足之处请指正
# coding:utf-8 import random, re import sqlite3 import json from bs4 import BeautifulSoup import sys reload(sys) sys.setdefaultencoding('utf-8') import uuid import requests session = requests.session() def md5(str): import hashlib m = hashlib.md5() m.update(str) return m.hexdigest() def wangyi(): for i in range(1,3): if i ==1: k = "" else: k = "_0" + str(i) url = "http://temp.163.com/special/00804KVA/cm_yaowen" + k + ".js?callback=data_callback" print url headers = { "Host":"temp.163.com", "Connection":"keep-alive", "Accept":"*/*", "User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.75 Safari/537.36 LBBROWSER", "Referer":"http://news.163.com/", "Accept-Encoding":"gzip, deflate, sdch", "Accept-Language":"zh-CN,zh;q=0.8", } result = session.get(url=url,headers=headers).text try: result1 = eval(eval((json.dumps(result)).replace('data_callback(','').replace(')','').replace(' ',''))) except: pass try: for i in result1: tlink = i['tlink'] headers2 = { "Host":"news.163.com", "Connection":"keep-alive", "Accept":"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", "Upgrade-Insecure-Requests":"1", "User-Agent":"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.75 Safari/537.36 LBBROWSER", "Accept-Encoding":"gzip, deflate, sdch", "Accept-Language":"zh-CN,zh;q=0.8", } print "tlinktlinktlinktlink",tlink return_data = session.get(url=tlink,headers=headers2).text try: soup = BeautifulSoup(return_data, 'html.parser') returnSoup = soup.find_all("div", attrs={"id": "endText"})[0] print returnSoup print "===============================" try: returnList = re.findall('<p>(.*?)</p>',str(returnSoup)) content1 = '<-->'.join(returnList) except: content1 ="" try: returnList1 = re.findall('<p class="f_center">(.*?)</p>',str(returnSoup)) content2 = '<-->'.join(returnList1) except: content2 ="" content = content1 +content2 except: content = "" cx = sqlite3.connect("C:\\Users\\xuchunlin\\PycharmProjects\\study\\db.sqlite3", check_same_thread=False) cx.text_factory = str try: print "正在插入链接 %s 数据" % (url) tlink = i['tlink'] title = (i['title']).decode('unicode_escape') commenturl = i['commenturl'] tienum = i['tienum'] opentime = i['time'] print title print tlink print commenturl print tienum print opentime print content url2 = md5(str(tlink)) cx.execute("INSERT INTO wangyi (title,tlink,commenturl,tienum,opentime,content,url)VALUES (?,?,?,?,?,?,?)",(str(title), str(tlink), str(commenturl), str(tienum), str(opentime), str(content), str(url2))) except Exception as e: print e print "cha ru shi bai " cx.commit() cx.close() except: pass wangyi()
如果觉得对您有帮助,麻烦您点一下推荐,谢谢!
好记忆不如烂笔头
好记忆不如烂笔头