import urllib.request
import re
import random
import urllib.error
uapools=[
'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; 360SE)',
'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',
'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)',
'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; TencentTraveler 4.0)',
'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SE 2.X MetaSr 1.0; SE 2.X MetaSr 1.0; .NET CLR 2.0.50727; SE 2.X MetaSr 1.0)',
'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; 360SE)',
'User-Agent:Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36'
]
def ua(uapools):
thisua=random.choice(uapools)
#print(thisua)
headers=('User-Agent',thisua)
thisip = urllib.request.urlopen(
'http://tvp.daxiangdaili.com/ip/?tid=559796713225033&num=1&foreign=only').read().decode().strip()
print(thisip)
#thisip='127.0.0.1:3333'
proxy = urllib.request.ProxyHandler({'http': thisip})
opener = urllib.request.build_opener(proxy, urllib.request.HTTPHandler)
opener.addheaders=[headers]
urllib.request.install_opener(opener)
keyname='python'
key=urllib.request.quote(keyname)
for i in range(5,6):
url='http://weixin.sogou.com/weixin?query='+key+'&type=2&page='+str(i)+'&ie=utf8'
print(url)
ua(uapools)
try:
data=urllib.request.urlopen(url).read().decode('utf-8','ignore')
pat='<div class="txt-box">.*?<a target="_blank" href="(.*?)"'
data_list=re.compile(pat,re.S).findall(data)
print(data_list)
for j in range(len(data_list)):
addr=data_list[j].replace('amp;','')
print(addr)
article_name='d:\\data\\weixin\\'+str(i)+str(j)+'.html'
urllib.request.urlretrieve(addr,filename=article_name)
except urllib.error.URLError as err:
if hasattr(err, "code"):
print(err.code)
if hasattr(err, "reason"):
print(err.reason)