python3 无限爬虫

import requests
from queue import SimpleQueue
import re
 
q = SimpleQueue()
 
crawled = set()
 
def has_crawled(url):
    url = url.replace('http://', '').replace('https://', '').replace('/', '')
    if url in crawled:
        return True
    crawled.add(url)
    return False
 
u = 'http://baidu.com'
q.put(u)
has_crawled(u)
 
while not q.empty():
    url = q.get()
    try:
        r = requests.get(url)
    except BaseException:
        break
    except Exception:
        continue
    print(url)
    ls = re.findall('(https?://[A-Za-z0-9.]*?)[^A-Za-z0-9.]', r.text)
    for e in ls:
        if not has_crawled(e):
            q.put(e)