数据采集分析

数据采集分析

1.名词定义

• 采集请求CollectRequest:将数据采集封装为采集请求的数据结构
• 监控项选择器MetricSelector：用来基于不同的资源类型查询出来可以支持的监控项列表
• 采集调度器Scheduler: 将采集请求分发给不同的实例节点上，实现类可以kafka，通过不同机器消费来实现调度到不同机器上
• 生产者Producer：生产者，用于封装好采集请求，然后采集调度器去进行调度
• 消费采集请求的消费者Spider：
• 执行器executor：真正干采集活的代码逻辑

2.监控项选择器MetricSelector的设计

selector接口

public interface MetricSelector {

    Set<CollectMetric> select(ResourceType resourceType);

}

实现类

public class MetricSelectorImpl implements MetricSelector {

    public MetricSelectorImpl() {
    }

    @Override
    public Set<CollectMetric> select(ResourceType resourceType) {
        Set<CollectMetric> resourceMetrics = MetadataCache.metricMetadata.getMetrics(resourceType);
        return resourceMetrics;
    }

}

监控项元数据的定义

public class MetricMetadata {

    private Map<ResourceType, Set<CollectMetric>> metric = Maps.newConcurrentMap();

    public MetricMetadata(List<CollectRule> collectRules) {
        init(collectRules);
    }

    private void init(List<CollectRule> collectRules) {
        metric = collectRules.stream()
                .map(CollectMetric::new)
                .collect(Collectors.groupingByConcurrent(CollectMetric::getResourceType, Collectors.toSet()));
    }

    public Set<CollectMetric> getMetrics(ResourceType resourceType) {
        return metric.getOrDefault(resourceType, Sets.newConcurrentHashSet());
    }

}

3.采集调度器设计

调度器接口

public interface Scheduler {

    void into(List<CollectRequest> requests);

    List<CollectRequest> out();
}

kafka实现调度器接口的调度实现

public class KafkaScheduler implements Scheduler {

    private static final Logger logger = LoggerFactory.getLogger(KafkaScheduler.class);

    protected KafkaProducer<String, String> producer;
    protected KafkaConsumer cnsumer;

    protected String schedulerTopic;

    public KafkaScheduler() {
        this.cnsumer = BeanContext.getBean("fetchConsumer");
        this.producer = BeanContext.getBean("stringkafkaProducer");
        Environment environment = BeanContext.getApplicationContext().getEnvironment();
        this.schedulerTopic = environment.getProperty("kafka.topic.scheduler");
    }

    @Override // 把采集请求发送到kafka中
    public void into(List<CollectRequest> requests) {
        for (CollectRequest request : requests) {
            ProducerRecord<String, String> record = new ProducerRecord<>(schedulerTopic, JSON.toJSONString(request));
            try {
                producer.send(record, (metadata, exception) -> Optional.ofNullable(exception).ifPresent(ex -> logger.error("send kafka failed", ex)));
            } catch (Exception e) {
             logger.error("kafka 发送异常",e);
            }
        }
    }

    @Override // 从kafka中取出采集请求
    public List<CollectRequest> out() {
        List<CollectRequest> requests = Lists.newArrayList();
        ConsumerRecords<String, String> records = cnsumer.poll(Duration.ofSeconds(10));
        for (ConsumerRecord<String, String> record : records) {
            String value = record.value();
            requests.add(JSON.parseObject(value, CollectRequest.class));
        }
        return requests;
    }

}

4.Producer的设计

Producer接口的设计

public interface Producer {

    void produce();
}

抽象类AbstractProducer

public abstract class AbstractProducer implements Producer {

    private static final Logger logger = LoggerFactory.getLogger(AbstractProducer.class);

    private static final String UMP_PRODUCER_KEY = "delta.fetch.producer";

    private static final String fetchProduceRequestCountKey = "delta.fetch.bizMonitor.produceRequestCount";
    private static final String fetchProduceResourceCountKey = "delta.fetch.bizMonitor.produceResourceCount";
	// 监控项选择器
    private final MetricSelector selector = new MetricSelectorImpl();
	// 调度器，将采集请求发到哪里
    private final Scheduler scheduler;

    private final RedisUtils4Jedis jedis;

    public AbstractProducer() {
        this.jedis = BeanContext.getBean(RedisUtils4Jedis.class);
        this.scheduler = new KafkaScheduler();
    }

    public abstract ResourceType resourceType();
	// redis锁名字
    public abstract String getRedisKey();

    /***
     * 基于资源类型获取要采集的监控项列表
     * @return {"serviceCode": {"origin.metric": "format.metric"}}
     */
    public Set<CollectMetric> getMetrics(ResourceType resourceType) {
        return selector.select(resourceType);
    }
    // 获取采集请求
    public Map<ProductionInfo, List<CollectRequest>> getRequest(Set<CollectResource> resources, Set<CollectMetric> metricSet) {
        throw new RuntimeException();
    }
	// 限速器，按照一定速率往调度器里放，防止压力太大。后面调用崩
    public abstract RateLimiter getRateLimiter();

    @Override
    public void produce() {
        RLock lock = jedis.getLock(getRedisKey());
        RateLimiter rateLimiter = getRateLimiter();
        CallerInfo info = null;
        try {
            info = Profiler.registerInfo(getUmpKey());
            boolean b = lock.tryLock();
            if (b) {
				// 获取采集的监控时间段范围
                long targetTime = getTargetTime();
                ResourceType resourceType = resourceType();
				// 后去要采集的资源列表
                Set<CollectResource> resourceSet = getResource(resourceType);
                logger.info("[Producer] sourceType : {} , size : {}", resourceType(), resourceSet.size());
				// ProductionInfo 对象没有业务用途，单纯为了业务监控
                Map<ProductionInfo, List<CollectRequest>> requestList = getRequest(resourceSet, getMetrics(resourceType));
                for (Map.Entry<ProductionInfo, List<CollectRequest>> productionInfoListEntry : requestList.entrySet()) {
                    ProductionInfo productionInfo = productionInfoListEntry.getKey();
                    List<CollectRequest> collectRequests = productionInfoListEntry.getValue();
                    AtomicInteger sendCount = new AtomicInteger(0);
                    for (CollectRequest request : collectRequests) {
                        rateLimiter.acquire();
                        request.setTargetTime(targetTime);
                        request.setProduceTime(System.currentTimeMillis());
						.// 将采集请求分发到kafka中
                        scheduler.into(Lists.newArrayList(request));
						// 发送技术自+1
                        sendCount.incrementAndGet();
                    }
                    Map<String, Number> requestCountMap = new HashMap<>();
                    requestCountMap.put(productionInfo.getSubType(), sendCount.get());
                    Profiler.sourceDataByNum(fetchProduceRequestCountKey, requestCountMap);

                    Map<String, Number> resourceTotalMap = new HashMap<>();
                    resourceTotalMap.put(productionInfo.getSubType(), productionInfo.getSubTypeCount());

                    Profiler.sourceDataByNum(fetchProduceResourceCountKey, resourceTotalMap);
                    logger.info("[Producer] end , sourceType : {} , resource size:{}, send count: {} ", productionInfo.getSubType(), productionInfo.getSubTypeCount(), sendCount.get());
                }
            }
        } catch (Exception e) {
            logger.error("[Producer] error, ", e);
            Profiler.functionError(info);
        } finally {
            if (lock.isLocked() && lock.isHeldByCurrentThread()) {
                lock.unlock();
            }
            Profiler.registerInfoEnd(info);
        }
    }

    private String getUmpKey() {
        return UMP_PRODUCER_KEY;
    }

    protected Set<CollectResource> getResource(ResourceType resourceType) {
        return MetadataCache.resourceMetadata.getResource(resourceType);
    }

    protected long getTargetTime() {
        return DateUtils.getBeforeUTCMinutes(0);
    }

}

MysqlHostProducer的生产者（实现类）

/**
 * 自建数据库监控数据
 * from ； dba zabbix

 */
public class MysqlHostProducer extends AbstractProducer {

    private static final ResourceType type = ResourceType.MYSQL_HOST;

    private static final String KEY = "mysqlHostResourceWriter_v2.2";

    private static final String COLLECTOR_KEY = "zabbixCollector";

    @Override
    public ResourceType resourceType() {
        return type;
    }

	// 基于资源和监控项生成采集请求
    @Override
    public Map<ProductionInfo, List<CollectRequest>> getRequest(Set<CollectResource> resources, Set<CollectMetric> metrics) {
        List<List<CollectResource>> partition = Lists.partition(Lists.newArrayList(resources), 10);
        Map<ProductionInfo, List<CollectRequest>> resultMap = Maps.newConcurrentMap();
        List<CollectRequest> collectRequests = partition.stream()
                .map(collectResources -> createCollectRequest(collectResources, metrics, COLLECTOR_KEY))
                .collect(Collectors.toList());
        ProductionInfo productionInfo = new ProductionInfo();
        productionInfo.setResourceType(resourceType());
        productionInfo.setResourceTotal(resources.size());
        productionInfo.setSubTypeCount(collectRequests.size());
        productionInfo.setSubType(resourceType().name());
        resultMap.put(productionInfo, collectRequests);
        return resultMap;
    }

    protected CollectRequest createCollectRequest(List<CollectResource> resources, Set<CollectMetric> metric, String collector) {
        Set<String> ips = resources.stream()
                .map(CollectResource::getResourceId)
                .collect(Collectors.toSet());
        CollectRequest collectRequest = new CollectRequest();
        ZabbixRequest zabbixRequest = new ZabbixRequest(collectRequest);
        zabbixRequest.setIps(ips);
        zabbixRequest.setMetrics(metric);
        zabbixRequest.setCollector(collector);
        zabbixRequest.setEndTime(String.valueOf(DateUtils.getBeforeUTCSecond(0)));
        zabbixRequest.setStartTime(String.valueOf(DateUtils.getBeforeUTCSecond(-1)));
        zabbixRequest.setMetricSourceEnum(MetricSourceEnum.ZABBIX);
        return collectRequest;
    }

    @Override
    public String getRedisKey() {
        return KEY;
    }

    @Override
    public RateLimiter getRateLimiter() {
        return RateLimiter.create(40.0);
    }


    @Override
    protected long getTargetTime() {
        return delta.fetch.utils.DateUtils.getBeforeUTCMinutes(-1);
    }

}

5.Spider的设计

后台起一个线程，一直从调度器里取出CollectRequest，然后进行处理
Spider.java

public class Spider implements Runnable {

    private static final Logger logger = LoggerFactory.getLogger(Spider.class);

    private final ReentrantLock lock = new ReentrantLock();
    private final Condition condition = lock.newCondition();

    private int emptySleepTime = 5000;

    private Scheduler scheduler;

    private CollectorFactory factory;

    private int parallelism = 10;

    protected ExecutorService executorService;

    private List<Pipeline> pipelines = new ArrayList<>();

    private List<Listener> listeners;

    private final List<Checker> checkers = Lists.newArrayList(new ExceptionChecker(), new TimeoutChecker(), new DataEnoughChecker());

    public static Spider create() {
        return new Spider();
    }

    public void runAsync() {
        Thread thread = new Thread(this);
        thread.setDaemon(true);
        thread.start();
    }

    public void start() {
        runAsync();
    }

    public List<Listener> getListeners() {
        return listeners;
    }

    public void setListeners(List<Listener> listeners) {
        this.listeners = listeners;
    }

    public Spider parallel(int parallelism) {
        this.parallelism = parallelism;
        if (parallelism <= 0) {
            throw new IllegalArgumentException("threadNum should be more than one!");
        }
        return this;
    }

    public Spider parallel(ExecutorService executorService, int parallelism) {
        this.parallelism = parallelism;
        if (parallelism <= 0) {
            throw new IllegalArgumentException("threadNum should be more than one!");
        }
        this.executorService = executorService;
        return this;
    }

    public Spider addPipeline(Pipeline pipeline) {
        this.pipelines.add(pipeline);
        return this;
    }

    public Spider setPipelines(List<Pipeline> pipelines) {
        this.pipelines = pipelines;
        return this;
    }

    public Spider setScheduler(Scheduler scheduler) {
        this.scheduler = scheduler;
        return this;
    }

    protected void initComponent() {
        if (pipelines.isEmpty()) {
            pipelines.add(new PrintPipeline());
        }

        this.executorService = ThreadUtils.newDaemonFixedThreadPool("collector-thread", parallelism * 3, parallelism);
        factory = new CollectorFactory();
    }

	// 死循环，从队列里取数据然后新开线程去处理
    @Override
    public void run() {
        initComponent();
        while (!Thread.currentThread().isInterrupted()) {
            List<CollectRequest> requests = scheduler.out();
            if (requests == null || requests.isEmpty()) {
                waitData();
            } else {
                for (CollectRequest request : requests) {
                    executorService.execute(() -> {
                        try {
                            request.setCollectTime(System.currentTimeMillis());
                            processRequest(request);
                            onSuccess(request);
                        } catch (Exception ex) {
                            logger.error("spider execute error ", ex);
                            onError(request);
                        } finally {
                            notifyData();
                        }
                    });
                }
            }
        }
    }
	// 处理采集请求
    private void processRequest(CollectRequest request) {
		// 获取采集器
        Collector collector = factory.matchDownloader(request);
		// 采集器执行采集任务,详细见标题7
        CollectResponse response = collector.collect(request);
        //当前如果出现异常仅记录到日志
        checkers.forEach(checker -> checker.check(request, response));
		// 将采集的结果发送到pipelines中去，详细见标题8
        pipelines.forEach(pipeline -> pipeline.process(response));
    }

    protected void onError(CollectRequest request) {
        //todo 重试这个批次？
        if (CollectionUtils.isNotEmpty(listeners)) {
            for (Listener listener : listeners) {
                listener.onFailure(request);
            }
        }
    }

    protected void onSuccess(CollectRequest request) {
        if (CollectionUtils.isNotEmpty(listeners)) {
            for (Listener listener : listeners) {
                listener.onSuccess(request);
            }
        }
    }

    private void waitData() {
        lock.lock();
        try {
            condition.await(emptySleepTime, TimeUnit.MILLISECONDS);
        } catch (InterruptedException ignore) {
        } finally {
            lock.unlock();
        }
    }

    private void notifyData() {
        try {
            lock.lock();
            condition.signalAll();
        } finally {
            lock.unlock();
        }
    }
}

6.采集器工厂CollectorFactory设计

用于基于不同的采集请求找到对应的采集器

public class CollectorFactory {

    protected Reflections reflections;

    private final Map<String, Collector> downloaders;

    public CollectorFactory() {
        this.reflections = new Reflections(
                ConfigurationBuilder.build("delta.fetch.spider.collector", Thread.currentThread().getContextClassLoader())
                        .setMetadataAdapter(new CollectorJavaReflectionAdapter())
                        .setExpandSuperTypes(false));
        this.downloaders = new ConcurrentHashMap<>();
        loadDownloaderBean(reflections);
    }

    private void loadDownloaderBean(Reflections reflections) {
        Set<Class<?>> downloaderBeans = reflections.getTypesAnnotatedWith(Downloader.class);
        for (Class<?> downloaderBean : downloaderBeans) {
            addDownloaderBean(downloaderBean);
        }
    }

    private void addDownloaderBean(Class<?> spiderBeanClass) {
        Downloader downloader = spiderBeanClass.getAnnotation(Downloader.class);
        String name = downloader.value();
        try {
            downloaders.put(name.toLowerCase(Locale.ROOT), (Collector) spiderBeanClass.newInstance());
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }

    public Collector matchDownloader(CollectRequest request) {
        String serviceCode = request.getHeaders().getOrDefault(RequestConstants.COLLECTOR_SELECTOR, "default").toLowerCase(Locale.ROOT);
        return downloaders.getOrDefault(serviceCode, new DefaultCollector());
    }
}

7.采集器的设计

Collector

public interface Collector {

    CollectResponse collect(CollectRequest request);

    void release();

}

默认采集器

@Downloader("default")
public class DefaultCollector implements Collector {

    @Override
    public CollectResponse collect(CollectRequest request) {
        return CollectResponse.create(request);
    }

    @Override
    public void release() {

    }
}

8.Pipeline的设计

public interface Pipeline {

    CollectResponse process(CollectResponse response);
}

发送kafka的pipeline

public class OriginalKafkaSenderPipeline implements Pipeline {

    private static final Logger logger = LoggerFactory.getLogger(OriginalKafkaSenderPipeline.class);

    protected KafkaProducer<String, String> producer;

    protected Map<ResourceType, String> mqTopic;

    public OriginalKafkaSenderPipeline() {
        this.producer = BeanContext.getBean("stringkafkaProducer");
        this.mqTopic = BeanContext.getBean("mqTopic");
    }

    @Override
    public CollectResponse process(CollectResponse response) {
        try {
            if (response instanceof OriginalCollectResponse) {
                OriginalCollectResponse mdcCollectResponse = (OriginalCollectResponse) response;
                Object monitorData = mdcCollectResponse.getMonitorData();
                if (Objects.nonNull(monitorData) && shouldSend(mdcCollectResponse.getRequest())) {
                    List<Object> objects = sendPreprocess(mdcCollectResponse);
                    for (Object object : objects) {
                        String sendmsg = JsonUtils.toJSONString(object);
                        ProducerRecord<String, String> record = new ProducerRecord<>(getTopic(mdcCollectResponse.getRequest()), sendmsg);
                        producer.send(record, (metadata, exception) -> Optional.ofNullable(exception).ifPresent(ex -> logger.error("send kafka failed", ex)));

                    }
                }
            }
        } catch (Exception ex) {
            logger.error("send to kafka failed", ex);
        }
        return response;
    }

    public List<Object> sendPreprocess(OriginalCollectResponse response) {
        return Lists.newArrayList(response.getMonitorData());
    }

    public Boolean shouldSend(CollectRequest request) {
        if (request instanceof CollectRequest) {
            return true;
        }
        return false;
    }

    public String getTopic(CollectRequest request) {
        throw new RuntimeException("The topic type is not supported");
    }
}