python爬虫：爬取医药数据库drugbank
python爬虫：爬取医药数据库drugbank
这个是帮朋友做的，难点就是他们有一个反爬虫机制，用request一直不行，后面我就用selenium直接把网页copy下来，然后再来解析本地的html文件，就木有问题啦。
现在看来，写得有点傻，多包涵。
# -*- coding:utf-8 -*-
import os
import time
import datetime
import codecs
from lxml import etree
from selenium import webdriver
import csv
#控制编码，全英文网页，用不着
# import sys
# reload(sys)
# sys.setdefaultencoding('utf-8')
# # date格式转为string格式
today = datetime.date.today()
today_string = today.strftime('%Y-%m-%d')
#通过浏览器得到网页页面--反反爬虫
def html_getter(site,file_name):
driver = webdriver.Firefox()
# chromedriver = r'C:\Program Files (x86)\Google\Chrome\Application\chromedriver.exe'
# os.environ['webdriver.chrome.driver'] = chromedriver
# driver = webdriver.Chrome(chromedriver)
driver.get(site)
driver.maximize_window() # 将浏览器最大化显示
time.sleep(5) # 控制间隔时间，等待浏览器反映
# 保存页面
source_code = driver.find_element_by_xpath("//*").get_attribute("outerHTML")
f = codecs.open(file_name, 'w+', 'utf8')
f.write(source_code)
f.close()
#打开保存在本地的html文件
def file_html(file_name):
f = open(file_name,'r')
html = f.read()
f.close()
return html
#写入csv，也可以有其他写入方式，这个地方就csv啦
def csv_writer(ll):
headers = ['drug','inter','snp_rs_id','Allele_name','Defining_change','Adverse_Reaction','ref','href','original_title']
with open('drugbank.csv','a') as f:
f_csv = csv.writer(f)
f_csv.writerow(headers)
f_csv.writerows(ll)
#用xpath解析网页，得到表格数据，我就是这么爱xpath，不喜欢正则表达式
def data_get(html):
selector = etree.HTML(html)
tbody=selector.xpath('/html/body/main/table/tbody/tr')
for each in tbody:
# #1.'drug'
drug_name=each.xpath('td[1]/strong/text()')[0]
drug_sn=each.xpath('td[1]/a/text()')[0]
drug=drug_name+' '+drug_sn
# #print(drug)
# #2.'Interacting Gene/Enzyme'
int=each.xpath('td[2]')[0]
inter=int.xpath('string(.)')
# print(inter)
# #3.'SNP RS ID'
snp=each.xpath('td[3]/a/text()')
if snp:
snp_rs_id=snp[0]
else:
snp_rs_id='Not Available '
#print snp_rs_id
#4.Allele name
Allele=each.xpath('td[4]/text()')
if Allele:
Allele_name=Allele[0]
else:
Allele_name='Not Available '
# #print Allele_name
# #5.'Defining change'
Defining=each.xpath('td[5]/text()')
if Defining:
Defining_change=Defining[0]
else:
Defining_change='Not Available '
# print Defining_change
# 6.'Adverse Reaction'
Adverse=each.xpath('td[6]/text()')
if Adverse:
Adverse_Reaction=Adverse[0]
else:
Adverse_Reaction='Not Available '
# print Adverse_Reaction
#7.'Reference(s)'
ref=each.xpath('td[7]/span/a/text()')[0]
href=each.xpath('td[7]/span/a/@href')[0]
original_title=each.xpath('td[7]/span/a/@data-original-title')[0]
# print ref
# print(href)
# print(original_title)
tt=(drug,inter,snp_rs_id,Allele_name,Defining_change,Adverse_Reaction,ref,href,original_title)
ll.append(tt)
#print ll
if __name__ == '__main__':
ll=[]
for i in range(1,5):
page_num=i
site='http://www.drugbank.ca/genobrowse/snp-adr?page='+str(page_num)
#get the html through webdriver
file_name=unicode(today_string)+u'drugbank_'+unicode(str(page_num))+u'.html'
html_getter(site,file_name)
html=file_html(file_name)
data_get(html)
csv_writer(ll)
转载于:https://www.cnblogs.com/miranda-tang/p/5508359.html
文章知识点与官方知识档案匹配，可进一步学习相关知识
Python入门技能树网络爬虫urllib47226 人正在系统学习中
