布隆去重

#bloom_filter
from bloom_filter import BloomFilter
# 生成一个装1亿大小的
bloom = BloomFilter(max_elements=100000000, error_rate=0.1)
# 向bloom添加URL
bloom.add('https://www.tianyancha.com/company/23402373')
#判断URL是否在
if 'https://www.tianyancha.com/company/23402373' in bloom:
    print("该url存在")