url请求

1 from urllib.request import urlopen
2 url="****"
3 respones = urlopen(url)
4 content =  respones.read()
5 content = content.decode('utf-8')
6 print(content)

request请求

 1 import requests
 2 url="***"
 3 headers = {'Accept': '*/*',
 4                'Accept-Language': 'en-US,en;q=0.8',
 5                'Cache-Control': 'max-age=0',
 6                'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36',
 7                'Connection': 'keep-alive',
 8                'Referer': 'http://www.baidu.com/'
 9                }
10 res = requests.get (url,headers=headers)#加headers头是为了伪装成浏览器浏览网页
11 print(res.status_code)#打印状态码
12 print(res.text)#打印文本
13 print(res.content)#打印图片或者视频文本都可以

解析库BeautifulSoup

 1 import requests
 2 from bs4 import BeautifulSoup
 3 url="http://news.qq.com/"
 4 headers = {'Accept': '*/*',
 5                'Accept-Language': 'en-US,en;q=0.8',
 6                'Cache-Control': 'max-age=0',
 7                'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36',
 8                'Connection': 'keep-alive',
 9                'Referer': 'http://www.baidu.com/'
10                }
11 res = requests.get (url,headers=headers)#加headers头是为了伪装成浏览器浏览网页
12 Soup = BeautifulSoup(res.text.encode("utf-8"),'lxml')
13 en = Soup.find_all('en',attrs={'class':'f14 124'})
14 for i in en:
15     title = i.a.get_text()
16     link = i.a['href']
17     print({
18         '标题':title,
19         '链接':link
20     })

 

解析库lxml，xpath表达法

 1 import requests
 2 from lxml import etree
 3 url="http://news.qq.com/"
 4 headers = {'Accept': '*/*',
 5                'Accept-Language': 'en-US,en;q=0.8',
 6                'Cache-Control': 'max-age=0',
 7                'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36',
 8                'Connection': 'keep-alive',
 9                'Referer': 'http://www.baidu.com/'
10                }
11 html = requests.get (url,headers=headers)#加headers头是为了伪装成浏览器浏览网页
12 con  = etree.HTML(html.text)
13 title = con.xpath('//en[@class="f14 124"]/a/text()')
14 link = con.xpath('//en[@class="f14 124"]/a/@href')
15 for i in zip(title,link):
16     print({
17         '标题':i[0],
18         '链接':i[1]
19     })

selesct方法

 1 import requests
 2 from bs4 import BeautifulSoup
 3 url="http://news.qq.com/"
 4 headers = {'Accept': '*/*',
 5                'Accept-Language': 'en-US,en;q=0.8',
 6                'Cache-Control': 'max-age=0',
 7                'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36',
 8                'Connection': 'keep-alive',
 9                'Referer': 'http://www.baidu.com/'
10                }
11 res = requests.get (url,headers=headers)#加headers头是为了伪装成浏览器浏览网页
12 Soup = BeautifulSoup(res.text.encode("utf-8"),'lxml')
13 en = Soup.select('en[class="f12 124"] a')
14 for i in en:
15     title = i.a.get_text()
16     link = i.a['href']
17     print({
18         '标题':i[0],
19         '链接':i[1]
20     })