简单的python爬虫 爬的乌云漏洞类型

import urllib.request
import re

starturl="http://wooyun.org/searchbug.php?q=%E9%87%91%E8%9E%8D"

def get_html_response(url):
    html_response = urllib.request.urlopen(url).read().decode('utf-8')
    return html_response

def geturl(starturl):
    a=get_html_response(starturl)
    childurl=(re.findall(r'/bugs/wooyun-\w*-\w*\b',a))
    return childurl

def get_nextpage(starturl):
    d=get_html_response(starturl)
    nextpage=(re.findall(r'searchbug.php\?q=金融&pNO=\w',d))
    return nextpage

result=[]
result2=[]

for i in get_nextpage(starturl):
    result+=geturl('http://wooyun.org/'+re.sub('金融','%E9%87%91%E8%9E%8D',i))
    #扫描各种漏洞的url地址放入result中
result=set(result)#去除result中重复的地址
#print(len(result))
#测试结果为87，正确

#信息存储+文件处理
output=open("D:\\a.txt",'w+')

for i in result:
    k=get_html_response('http://wooyun.org/'+re.sub('金融','%E9%87%91%E8%9E%8D',i))#下载页面到k，可以把
    info=re.findall(r'<h3>\w*：\s*\w*\s*</h3>',k)#空白字符用/s,寻找所有适用于<h3>标签的文字
    for j in info:
        j=re.sub(r'：\s',':',j)
        j=re.sub(r'<h3>','',j)
        j=re.sub(r'</h3>',',',j)
        j=j.split()
        output.writelines(j)
        output.writelines("\n")
output.close()

mentor要求的，简略分析乌云关于金融方面的漏洞并归类

人生第一个能跑的有意义的程序，好开心

 

2014/8/14根据导师要求做了修改

import urllib.request
import re

starturl="http://www.wooyun.org/searchbug.php?q=6YeR6J6N"

def get_html_response(url):
    html_response = urllib.request.urlopen(url).read().decode('utf-8')
    return html_response

def geturl(starturl):
    a=get_html_response(starturl)
    childurl=(re.findall(r'/bugs/wooyun-\w*-\w*\b',a))
    return childurl

def get_nextpage(starturl):
    d=get_html_response(starturl)
    nextpage=(re.findall(r'searchbug.php\?q=6YeR6J6N&pNO=\w',d))
    return nextpage

result=[]for i in get_nextpage(starturl):
    result+=geturl('http://wooyun.org/'+re.sub('金融','6YeR6J6N',i))
    #扫描各种漏洞的url地址放入result中
result=set(result)#去除result中重复的地址
#print(len(result))
#测试结果为87，正确

#信息存储+文件处理
output=open("D:\\wooyun.txt",'w+')

for i in result:
    k=get_html_response('http://wooyun.org/'+re.sub('金融','%E9%87%91%E8%9E%8D',i))#下载页面到k
    info=re.findall(r'<h3>\w*：\s*\w*\s*</h3>',k)#空白字符用/s,寻找所有适用于<h3>标签的文字
    detail=re.findall(r'<p class="detail">.*.</p>',k)
    for j in info:
        j=re.sub(r'：\s',':',j)
        j=re.sub(r'<h3>','',j)
        j=re.sub(r'</h3>','',j)
        j=j.split()
        output.writelines(j)
        output.writelines("\n")
    for j in detail:
        j=re.sub(r'：\s',':',j)
        j=re.sub(r'<p\sclass="detail">','',j)
        j=re.sub(r'</p>','',j)
        j=re.sub(r'"\starget="_blank"><img\ssrc="/upload/.*.width="600"/></a>','',j)
        j=re.sub(r'<a href="',' http://www.wooyun.org',j)
        j=j.split()
        output.writelines(j)
        output.writelines("\n")
    output.writelines("\n")
output.close()