1、bs4+urllib
from bs4 import BeautifulSoup
import urllib2
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
###url = 'https://www.qidian.com/search?kw=%E7%AC%91%E5%82%B2%E6%B1%9F%E6%B9%96'
#
#
#
class main():
def __init__(self):
pass
def search(self):
'''查询函数'''
####输入关键字
####访问url，获取信息
####存储信息
print '-'*80
#print '--开始输入关键字: '.decode('utf-8').encode('gbk')
#key = raw_input('key: ')
key = urllib2.quote('笑傲江湖')
url = 'https://www.qidian.com/search?kw='+key
print '访问的网址是--'.decode('utf-8').encode('gbk')+url
#####开始访问
self.spider(url)
def spider(self, url):
print '-'*80
print '开始访问网页'.decode('utf-8').encode('gbk')
print '-'*80
response = urllib2.urlopen(url).read()
obj = BeautifulSoup(response,'html.parser')
div_list = obj.find('div',{'class':'book-img-text'}).find_all('li')
for v in div_list:
name = v.find('h4').find('a').text
intro = v.find('div',{'class':'book-mid-info'}).find('p',{'class':'intro'}).text
print intro
def test(self):
print urllib2.quote('笑傲江湖')
if __name__ == '__main__':
book = main()
book.search()
2、lxml+urllib2
# -*- coding: utf-8 -*-
#
#
#
#
from lxml import etree
import urllib2
url = 'https://www.qidian.com/search?kw=%E7%AC%91%E5%82%B2%E6%B1%9F%E6%B9%96'
def main():
req=urllib2.Request(url)
fd=urllib2.urlopen(req)
html=etree.HTML(fd.read())
li_list=html.xpath('//*[@id="result-list"]/div/ul/li')
for v in li_list:
name = v.xpath('div[2]/h4/a')
instro = v.xpath('div[2]/p[2]')
print name[0].xpath('string(.)').strip()
print instro[0].xpath('string(.)').strip()
if __name__ == '__main__':
main()
3、requests测试登录
# -*- coding: utf-8 -*-
#
#
#
###测试登录
import requests
from bs4 import BeautifulSoup
####模拟的urlhttp://www.jq22.com/emdl.aspx
from bs4 import BeautifulSoup
#######加载cookies模拟登录
url = "http://www.jq22.com"
UA = "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.13 Safari/537.36"
header = { "User-Agent" : UA,"Referer": "http://www.jq22.com/emdl.aspx"}
raw_cookies = 'CityCookie=y; ASP.NET_SessionId=pudp5latoug40dl3sovnr2sm;Hm_lvt_b3a3fc356d0af38b811a0ef8d50716b8=1510823622; Hm_lpvt_b3a3fc356d0af38b811a0ef8d50716b8=1510824618;VisitInfo=AccessCount=3;MydlCookie=yhdlId=144256&yhdlPw=78B0431D56E6E539'
cookies={}
for line in raw_cookies.split(';'):
key,value=line.split('=',1) #1代表只分一次，得到两个数据
cookies[key]=value
s=requests.get(url,headers = header, cookies=cookies)
obj = BeautifulSoup(s.content, 'html.parser')
o = obj.find('div',{'class':'myhome'})
print o
