python爬虫小实例
1、python爬取贴吧壁纸 1.1、获取整个页面数据 #coding=utf-8 import urllib def getHtml(url): page = urllib.urlopen(url) html = page.read() return html html = getHtml("http://tieba.baidu.com/p/2738151262") print html 复制代码 1.2、筛选页面中想要的数据 import re import urllib def getHtml(url): page = urllib.urlopen(url) html = page.read() return html def getImg(html): reg = r'src="(.+?\.jpg)" ' imgre = re.compile(reg) imglist = re.findall(imgre,html) return imglist html = getHtml("http://tieba.baidu.com/p/2460150866") print getImg(html) 1.3、将页面筛选的数据保存到本地 #coding=utf-8 import urllib import re def getHtml(url): page = urllib.urlopen(url) html = page.read() return html def getImg(html): reg = r'src="(.+?\.jpg)" ' imgre = re.compile(reg) imglist = re.findall(imgre,html) x = 0 for imgurl in imglist: urllib.urlretrieve(imgurl,'%s.jpg' % x) x+=1 html = getHtml("http://tieba.baidu.com/p/2460150866") print getImg(html) 抓取昵图网图片 --修改版 #coding=utf-8 import urllib import re def getHtml(url): page = urllib.urlopen(url) html = page.read() return html def getImg(html): reg = r'src="(.*?)" ' imgre = re.compile(reg) imglist = re.findall(imgre,html) x = 0 for imgurl in imglist: urllib.urlretrieve(imgurl,'D:360\\%s.jpg' % x) x+=1 html = getHtml("http://www.nipic.com/show/17742538.html") print getImg(html) 解释: %s意思是字符串参数,就是将变量的值传入到字符串里面,字符串后的'%'后就是写要传入的参数。 在你给出的例子中,就是用x的值替代%s。比如说x=5,那么就是爬取url后面是'5.jpg'这个图片 保存的位置默认为程序的存放目录 如何保存到指定目录:urllib.urlretrieve(imgurl,'D:360\\%s.jpg' % x) https://image.baidu.com/search/detail?ct=503316480&z=0&ipn=false&word 2、python抓取价格 前两个不用加 text #-*—coding:utf8-*- from lxml import etree import urllib import urllib.request #headers构造一个字典,里面保存了user-agent #headers= { 'User-Agent' : 'User-Agent:Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36' } url="http://search.jd.com/Search?keyword=%E6%89%8B%E6%9C%BA&enc=utf-8&pvid=194960f41c994e81ada43edbc276f54b" html = urllib.request.urlopen(url).read() data=html.decode('utf-8') selector = etree.HTML(data) #xpath qiubai_text = selector.xpath('//div/ul/li/div/div/strong/i/text()') #print(qiubai_text) for i in qiubai_text: print(i) 或者 #-*—coding:utf8-*- from lxml import etree import urllib import urllib.request #headers构造一个字典,里面保存了user-agent #headers= { 'User-Agent' : 'User-Agent:Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36' } url="http://search.jd.com/Search?keyword=%E6%89%8B%E6%9C%BA&enc=utf-8&pvid=194960f41c994e81ada43edbc276f54b" html = urllib.request.urlopen(url).read() selector = etree.HTML(html) #xpath qiubai_text = selector.xpath('//div/ul/li/div/div/strong/i/text()') #print(qiubai_text) for i in qiubai_text: print(i) 或者 :注意:这个需要加text html.text #-*—coding:utf8-*- from lxml import etree import requests #headers构造一个字典,里面保存了user-agent #headers= { 'User-Agent' : 'User-Agent:Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36' } url="http://search.jd.com/Search?keyword=%E6%89%8B%E6%9C%BA&enc=utf-8&pvid=194960f41c994e81ada43edbc276f54b" html = requests.get(url) selector = etree.HTML(html.text) #xpath qiubai_text = selector.xpath('//div/ul/li/div/div/strong/i/text()') #print(qiubai_text) for i in qiubai_text: print(i) 3、python爬取昵图网图片 #coding=utf-8 import urllib import re def getHtml(url): page = urllib.urlopen(url) html = page.read() return html def getImg(html): reg = r'src="(.*?)" ' imgre = re.compile(reg) imglist = re.findall(imgre,html) x = 0 for imgurl in imglist: urllib.urlretrieve(imgurl,'D:360\\%s.jpg' % x) x+=1 html = getHtml("http://www.nipic.com/show/17742538.html") print getImg(html) 4、爬音乐 # coding:utf-8 import urllib import urllib.request import re url="http://www.yy8844.cn/ting/ccceo/ceeivi.shtml" html = urllib.request.urlopen(url).read() data=html.decode('GBK') #print(data) music_id = int(re.findall(r'MusicId=(\d+)',data)[0]) music_name = re.findall(r'<title>(.*?)</title>',data)[0].split('/')[0].strip() music_word = re.findall(r'<div class="textgeci_show" id="showtext">(.*?)</div>',data,re.S)[0] article='word' with open("%s.txt" % article,'w') as f: f.write(music_word) #print(music_word) quanurl="http://96.ierge.cn/"'%d/%d/%s' % (music_id//30000,music_id//2000,music_id)+".mp3" #print(quanurl) bata=urllib.request.urlopen(quanurl).read() with open("%s.mp3" % music_name,'wb') as f: f.write(bata) 注意问题: music_word = re.findall(r'<div class="textgeci_show" id="showtext">(.*?)</div>',data,re.S)[0] python中AttributeError解决 【Python 脚本报错】AttributeError:'module' has no attribute 'xxx'的解决方法 http://blog.csdn.net/cn_wk/article/details/50839159 int库的.pyc文件 python 去掉 .pyc http://blog.csdn.net/ubuntu64fan/article/details/48241985 python操作对象属性 http://www.jianshu.com/p/c38a81b8cb38 Python学习日记4|python爬虫常见报错小结及解决方法 http://www.jianshu.com/p/17c921639ad0 #coding=utf-8 from Tkinter import * import tkMessageBox import urllib import json import mp3play import time import threading from pinyin import PinYin import os import stat test = PinYin() test.load_word() stop=0 def music(): if not entry.get(): tkMessageBox.showinfo("温馨提示","搜索内容不能为空") return name = test.hanzi2pinyin_split(entry.get()) html=urllib.urlopen("http://s.music.163.com/search/get/?type=1&s=%s&limit=9"%name).read() js=json.loads(html) n = 0 global x x = [] for i in js['result']['songs']: listbox.insert(n,'%s(%s)'%(i['name'],i['artists'][0]['name'])) n+=1 x.append(i['audio']) count = 0 #isplaying = None def play(): global count count += 1 index=listbox.curselection() var1.set(u"正在加载"+listbox.get(index,last=None)) urllib.urlretrieve(x[index[0]],'tmp%s.mp3'%str(count)) var1.set(u"正在播放"+listbox.get(index,last=None)) mp3=mp3play.load("tmp%s.mp3"%str(count)) mp3.play() time.sleep(mp3.seconds()) import inspect import ctypes def _async_raise(tid, exctype): """raises the exception, performs cleanup if needed""" tid = ctypes.c_long(tid) if not inspect.isclass(exctype): exctype = type(exctype) res = ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, ctypes.py_object(exctype)) if res == 0: raise ValueError("invalid thread id") elif res != 1: ctypes.pythonapi.PyThreadState_SetAsyncExc(tid, None) raise SystemError("PyThreadState_SetAsyncExc failed") def stop_thread(thread): _async_raise(thread.ident, SystemExit) threads=list() t=None def excute(event): global t for i in threads: stop_thread(i) t = threading.Thread(target=play) t.setDaemon(True) t.start() threads.append(t) root = Tk()#创建一个窗口 root.title("云音乐") root.geometry("500x300+500+200") entry=Entry(root)#创建输入框(单行),置父 entry.pack() btn=Button(root,text="搜 索",command=music) btn.pack()#布局方式必须用同一种 var=StringVar() listbox=Listbox(root,width=50,listvariable=var) listbox.bind('<Double-Button-1>',excute) listbox.pack() var1=StringVar() label=Label(root,text="云音乐播放器",fg="purple",textvariable=var1) var1.set("云音乐播放器") label.pack() root.mainloop()#显示窗口