python实现自动重启本程序的方法 技术的漩涡

python实现自动重启本程序的方法

http://www.jb51.net/article/69174.htm

 

import requests, time

url_l = []
with open('DISTINCT_url.404.notailaba.txt', 'r', encoding='utf-8') as fo:
    for i in fo:
        url = '%s%s' % ('http://', i.replace('\n', ''))
        url_l.append(url)
le, c = len(url_l), 0

import random

while True:
    for i in url_l:
        # 处理再某个url处停顿
        if int(time.time()) % 10 == 0:
            print(int(time.time()))
            time.sleep(1)
            break
        c += 1
        if 'iask.sina' in i:
            continue
        with open('DISTINCT_url.break.log', 'r', encoding='utf-8') as fc:
            for breaker in fc:
                break
        if c < int(breaker):
            continue
        if c % 50 == 0:
            time.sleep(random.randint(0, 3))
        with open('DISTINCT_url.break.log', 'w', encoding='utf-8') as flog:
            flog.write(str(c))

        print(c, '/', le, '---', i)
        try:
            try:
                req = requests.get(i)
                if req.status_code == 404:
                    with open('DISTINCT_url.404.notailaba.RES.txt', 'a', encoding='utf-8') as fr:
                        s = '%s%s' % (i.replace('http://', ''), '\n')
                        print(s)
                        fr.write(s)
                req.close()
            except:
                continue
        except:
            continue


卡顿的原因是连续的同域名url的请求，目标服务器做了限制。

import requests, time

url_l = []
with open('DISTINCT_url.404.notailaba.txt', 'r', encoding='utf-8') as fo:
    for i in fo:
        url = '%s%s' % ('http://', i.replace('\n', ''))
        url_l.append(url)
le, c = len(url_l), 0

import random

while True:
    for i in url_l:
        # 处理再某个url处停顿
        # if int(time.time()) % 10 == 0:
        #     print(int(time.time()))
        #     time.sleep(1)
        #     break
        c += 1
        if 'iask.sina' in i:
            continue
        with open('DISTINCT_url.break.log', 'r', encoding='utf-8') as fc:
            for breaker in fc:
                break
        if c < int(breaker):
            continue
        if c % 10 == 0:
            time.sleep(random.randint(1, 3))
        with open('DISTINCT_url.break.log', 'w', encoding='utf-8') as flog:
            flog.write(str(c))

        print(c, '/', le, '---', i)
        try:
            try:
                req = requests.get(i)
                if req.status_code == 404:
                    with open('DISTINCT_url.404.notailaba.RES.txt', 'a', encoding='utf-8') as fr:
                        s = '%s%s' % (i.replace('http://', ''), '\n')
                        print(s)
                        fr.write(s)
                req.close()
            except:
                continue
        except:
            continue



改进办法，休息
优化，统计url

结合对实际url的列表清单，目前的对观察数据的实验表明：此处离保证了qps再可承受范围

首先从从业务的角度分析出现的问题，然后借助技术工具去解决它！而不是不考虑业务场景，仅仅是剥离为一个技术问题，进而进入技术的漩涡。