springboot kafka 发送jsons数据到队列 和接受数据

mvaen 包配置  在pom.xml 中放入

 <dependency>
            <groupId>org.springframework.kafka</groupId>
            <artifactId>spring-kafka</artifactId>
        </dependency>

在代码中配置 kafka

import com.fasterxml.jackson.annotation.JsonFormat;

@JsonFormat(shape = JsonFormat.Shape.STRING)
public enum AdKafkaTopic {
   xxxxxx_name(1, 1);


    final int numPartitions;
    final short replicationFactor;

    AdKafkaTopic(int numPartitions, int replicationFactor) {
        this.numPartitions = numPartitions;
        this.replicationFactor = (short) replicationFactor;
    }

    public int getNumPartitions() {
        return numPartitions;
    }

    public short getReplicationFactor() {
        return replicationFactor;
    }
}

import org.apache.kafka.clients.admin.NewTopic;
import org.springframework.beans.BeansException;
import org.springframework.beans.factory.support.DefaultListableBeanFactory;
import org.springframework.context.ApplicationContext;
import org.springframework.context.ApplicationContextAware;
import org.springframework.context.ConfigurableApplicationContext;
import org.springframework.context.annotation.Configuration;

@Configuration
public class KafkaConfig implements ApplicationContextAware {

    NewTopic createFrom(AdKafkaTopic t) {
        return new NewTopic(t.name(), t.getNumPartitions(), t.getReplicationFactor());
    }

    @Override
    public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
        //将applicationContext转换为ConfigurableApplicationContext
        ConfigurableApplicationContext configurableApplicationContext = (ConfigurableApplicationContext) applicationContext;

        DefaultListableBeanFactory defaultListableBeanFactory = (DefaultListableBeanFactory) configurableApplicationContext.getBeanFactory();
        for (AdKafkaTopic t : AdKafkaTopic.values()) {
            defaultListableBeanFactory.registerSingleton("newTopic_" + t.name(), createFrom(t));
        }
    }

import org.springframework.kafka.core.KafkaTemplate;
import org.springframework.stereotype.Service;

@Service
public class KafkaDataSender {
    private final KafkaTemplate<String, String> template;

    public KafkaDataSender(KafkaTemplate<String, String> template) {
        this.template = template;
    }

    <T> void sendKafkaMsg(AdKafkaTopic topic, String key, T value) {
        if (value instanceof String) {
            this.template.send(topic.name(), key, (String) value);
        } else {
            this.template.send(topic.name(), key, JsonUtil.toJson(value));
        }
    }

//封装的发送方法 xxx是实体类

    public void sendSaleCampaign(xxx dto) {
        sendKafkaMsg(AdKafkaTopic.campaign_name_rule, dto.getCampaignId(), dto);
    }

}

 

上边配置好后，在需要发送的地方发送出去

 @Autowired
    private KafkaDataSender kafkaDataSender;



 public void pullSaleCampaign(xxx dto) {
        try {
            kafkaDataSender.sendSaleCampaign(dto);
            log.info("的推送到队列 xle 成功：{}", JsonUtil.toJson(dto));
        }catch (Exception e){
            log.error("的推送到队列 cxxe 失败：{}", JsonUtil.toJson(dto), e);
        }

    }

 

 

以上就是kafka发送数据到队列的方式

linux相关命令

//远程查看队列是否能链接通

kafkacat -b 8.219.216.229:9092 -L | grep 'xxx'

//远程往队列里面放数据
echo "Hello, Kafka!" | kafkacat -b 172.21.184.8:9092 -t xxx

 

消费队列数据

package com.giikin.ads.collector.process2.processor;

import com.giikin.ads.collector.oss.OssAppendWriteService;
import com.giikin.ads.collector.oss.OssTable;
import com.giikin.ads.collector.process2.adStruct.service.TiktokAdAdService;
import lombok.extern.slf4j.Slf4j;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.kafka.annotation.KafkaListener;
import org.springframework.stereotype.Component;

import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors;

@Component
@Slf4j
//@DependsOn(value = "kafkaTopicConfig")
public class SourceDataProcess {
    private final OssAppendWriteService ossAppendWriteService;
   

    @KafkaListener(
            id = "tiktokToOss",
            topics = {
                    "xxx","xxx2"

            },
            groupId = "tToOss",
            containerFactory = "batchFactory"
    )
    public void writeTiktokSource(List<ConsumerRecord<String, String>> records) {
        Map<String, List<ConsumerRecord<String, String>>> collect = records.stream().collect(Collectors.groupingBy(ConsumerRecord::topic));
        collect.forEach((topic, datas) -> {
            List<String> values = datas.stream().map(ConsumerRecord::value).collect(Collectors.toList());
            switch (topic) {
                case "xxx":
                    ossAppendWriteService.writeSync(values, OssTable.xxx);
                    break;
                case "xxxx":
                    break;
                default:
                   break;
            }
        });
    }
}