网络爬虫-京东商品页面的爬取

1、京东页面商品的爬取

import requests

def getHTMLText(url):
    try:
        r=requests.get(url)
        r.raise_for_status() #如果状态不是200，触发HTTPError异常
        r.encoding=r.apparent_encoding
        return r.text[:1000]
    except:
        return "产生异常"

if __name__=="__main__":
    url="https://item.jd.com/100000177748.html"
    print(getHTMLText(url))

2、亚马逊页面商品的爬取

更改user-agent访问头部属性，让代码模拟浏览器来向亚马逊服务器提供http请求

import requests

def getHTMLText(url):
    try:
        kv={'user-agent':'Mozilla/5.0'}
        r=requests.get(url,headers=kv)
        r.raise_for_status() #如果状态不是200，触发HTTPError异常
        r.encoding=r.apparent_encoding
        return r.text[:1000]
    except:
        return "产生异常"

if __name__=="__main__":
    url="https://www.aliyun.com/?utm_content=se_1000301881"
    print(getHTMLText(url))

3、百度/360搜索关键字提交

两大搜索引擎关键词URL

#!/usr/bin/python3
import requests

kv={'wd':'python'}
url='http://www.baidu.com'
r=requests.get(url,params=kv)
print(r.status_code)
print(r.request.url)

4、网络图片的爬取与存储

#!/usr/bin/python3
import requests

path='F:/章若楠.jpg'
url='http://n.sinaimg.cn/sinacn20112/200/w720h1080/20181211/4894-hprknvu2906379.jpg'
r=requests.get(url)
print(r.status_code)

with open(path,'wb') as f:
    f.write(r.content)

print(path+' 保存成功')

 4.1、网络图片的爬取与存储（优化版）

引入os模块，将图片保存在指定目录下；try except捕获异常

#!/usr/bin/python3
import requests
import os

url1='http://n.sinaimg.cn/sinacn20112/200/w720h1080/20181211/4894-hprknvu2906379.jpg'
url2='https://wx2.sinaimg.cn/mw690/9b6feba7ly1g1xme8o229j21sc2dsx6p.jpg'
url3='https://wx3.sinaimg.cn/mw690/9b6feba7ly1g1xmdm0ib6j21o728xx6q.jpg'
url4='https://wx1.sinaimg.cn/mw690/9b6feba7ly1g1qq8m4wkbj22ds1scb29.jpg'
url5='https://wx3.sinaimg.cn/mw690/9b6feba7ly1g1ihqc347ej22c02c07wt.jpg'

list_url=[url1,url2,url3,url4,url5]
print('url长度为',len(list_url))

root='F:/妹子资源/'
list_path=[]
for num in range(len(list_url)):
    list_path.append(root+'章若楠'+str(num)+'.jpg')
    print('保存路径为：'+list_path[num])

def get_resource(url,path):
    try:
        if not os.path.exists(root):
            os.mkdir(root)
        if not os.path.exists(path):
            r=requests.get(url)
            with open(path,'wb') as f:
                f.write(r.content)
            print('资源爬取成功')
        else:
            print('该资源已存在')
    except:
        print('爬取失败')

for url,path in zip(list_url,list_path):
    get_resource(url,path)
    #print(url,path)