爬虫案例之Pubmed数据库下载

代码

# encoding=utf-8
import os, time, re
import urllib.request
import urllib.parse
import ssl

ssl._create_default_https_context = ssl._create_unverified_context
retmax = 500
FAILURE = 0
SUCCESS = 1
startNum = 1

BASE = 'NARA'
FILES= ['Losartan','Valsartan','Irbesartan','Eprosartan','Candesartan','Telmisartan','Olmesartan']

# BASE = 'Triptans'
# FILES = ['Sumatriptan','Zolmitriptan',
#          'Naratriptan','Rizatriptan','Almotriptan',
#          'Frovatriptan','Eletriptan']


if not os.path.exists(BASE):
    os.mkdir(BASE)


def lastline(fd):
    lastNum = 0
    print(fd)
    if not os.path.isfile(fd):
        f_check = open(fd, 'w')
        f_check.close()
    f = open(fd, 'r')
    lines = f.readlines()
    f.close()
    if lines:
        lastNum = int(lines[-1].strip())
    return lastNum


def Download(drug, sleep_time, query_key, webenv, endNum):
    lastNum = lastline('{}/{}/checkpoint.txt'.format(BASE,drug))
    if lastNum == 0:
        start = startNum - 1
    else:
        start = lastNum + retmax
    # print(lastNum)
    f_append = open('{}/{}/checkpoint.txt'.format(BASE, drug), 'a')
    f_error = open('{}/{}/error.txt'.format(BASE, drug), 'a')
    try:
        for retstart in range(start, endNum, retmax):
            time.sleep(sleep_time)
            print('\tdownloading: %d - %d' % (retstart + 1, retstart + retmax))
            urllib.request.urlretrieve('https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?'
                                       'db=pubmed&query_key=%s&WebEnv=%s&retstart=%s&retmax=%s&retmode=xml' % (
                                           query_key, webenv, retstart, retmax),
                                       '%s/%s/%d-%d.xml' % (BASE, drug, retstart + 1, retstart + retmax))

            f_append.write('%d\n' % (retstart))
            f_append.flush()
    except Exception as ex:
        print(ex)
        # print('\t\tbad connection!')
        raise Exception()
        return FAILURE
    finally:
        f_append.close()
        f_error.close()

    print('Downloading  is done........................')
    return SUCCESS


def Download_auto(fun, drug, query_key, webenv, endNum, sleep_time=5):
    while True:
        try:
            value = fun(drug, sleep_time, query_key, webenv, endNum)
            if value == SUCCESS:
                break
        except Exception as e:
            sleep_time += 5
            print('prolong sleep time:', sleep_time)


def main(drug):
    """主函数"""
    if not os.path.exists('{}/{}'.format(BASE,drug)):
        os.mkdir('{}/{}'.format(BASE,drug))
    query = '%s[TIAB]+OR+%s[MH]' % (drug, drug)
    url = 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=pubmed&term={}&usehistory=y'.format(query)
    history = urllib.request.urlopen(url)
    content = history.read().decode()
    pattern = re.compile('<Count>(\d+)</Count>.*<QueryKey>(\d+)</QueryKey>.*<WebEnv>(\S+)</WebEnv>')
    s = pattern.search(content)
    count = s.group(1)
    query_key = s.group(2)
    webenv = s.group(3)
    print('total counts: %s' % count)
    endNum = int(count)
    print(endNum)
    Download_auto(Download,drug, query_key, webenv, endNum)


if __name__ == '__main__':
    start = time.time()
    list(map(main, FILES))
    print(time.time() - start)