【Kafka】实时看板案例
项目需求
快速计算双十一当天的订单量和销售金额
项目模型
实现步骤
一、创建topic
bin/kafka-topics.sh --create --topic itcast_order --zookeeper node01:2181,node02:2181,node03:2181 --partitions 5 --replication-factor 2
二、创建maven项目并导入要依赖的jar包
<dependencies>
<dependency>
<groupId>org.apache.kafka</groupId>
<artifactId>kafka-clients</artifactId>
<version>0.10.0.0</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.41</version>
</dependency>
<dependency>
<groupId>redis.clients</groupId>
<artifactId>jedis</artifactId>
<version>2.9.0</version>
</dependency>
<dependency>
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
<version>1.2.17</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<source>1.8</source>
<target>1.8</target>
</configuration>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
<version>2.4.1</version>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>shade</goal>
</goals>
<configuration>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
<mainClass>cn.itcast.realboard.LogOperate</mainClass>
</transformer>
</transformers>
</configuration>
</execution>
</executions>
</plugin>
<plugin>
<artifactId> maven-assembly-plugin </artifactId>
<configuration>
<descriptorRefs>
<descriptorRef>jar-with-dependencies</descriptorRef>
</descriptorRefs>
<archive>
<manifest>
<mainClass>cn.itcast.realboard.LogOperate</mainClass>
</manifest>
</archive>
</configuration>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
三、消费生产代码实现
- 1.创建订单实体类
package cn.itcast.realboard; import com.alibaba.fastjson.JSONObject; import java.text.ParseException; import java.text.SimpleDateFormat; import java.util.Date; import java.util.Random; import java.util.UUID; public class PaymentInfo { private static final long serialVersionUID = -7958315778386204397L; private String orderId;//订单编号 private Date createOrderTime;//订单创建时间 private String paymentId;//支付编号 private Date paymentTime;//支付时间 private String productId;//商品编号 private String productName;//商品名称 private long productPrice;//商品价格 private long promotionPrice;//促销价格 private String shopId;//商铺编号 private String shopName;//商铺名称 private String shopMobile;//商品电话 private long payPrice;//订单支付价格 private int num;//订单数量 /** * <Province>19</Province> * <City>1657</City> * <County>4076</County> */ private String province; //省 private String city; //市 private String county;//县 //102,144,114 private String catagorys; public String getProvince() { return province; } public void setProvince(String province) { this.province = province; } public String getCity() { return city; } public void setCity(String city) { this.city = city; } public String getCounty() { return county; } public void setCounty(String county) { this.county = county; } public String getCatagorys() { return catagorys; } public void setCatagorys(String catagorys) { this.catagorys = catagorys; } public PaymentInfo() { } public PaymentInfo(String orderId, Date createOrderTime, String paymentId, Date paymentTime, String productId, String productName, long productPrice, long promotionPrice, String shopId, String shopName, String shopMobile, long payPrice, int num) { this.orderId = orderId; this.createOrderTime = createOrderTime; this.paymentId = paymentId; this.paymentTime = paymentTime; this.productId = productId; this.productName = productName; this.productPrice = productPrice; this.promotionPrice = promotionPrice; this.shopId = shopId; this.shopName = shopName; this.shopMobile = shopMobile; this.payPrice = payPrice; this.num = num; } public String getOrderId() { return orderId; } public void setOrderId(String orderId) { this.orderId = orderId; } public Date getCreateOrderTime() { return createOrderTime; } public void setCreateOrderTime(Date createOrderTime) { this.createOrderTime = createOrderTime; } public String getPaymentId() { return paymentId; } public void setPaymentId(String paymentId) { this.paymentId = paymentId; } public Date getPaymentTime() { return paymentTime; } public void setPaymentTime(Date paymentTime) { this.paymentTime = paymentTime; } public String getProductId() { return productId; } public void setProductId(String productId) { this.productId = productId; } public String getProductName() { return productName; } public void setProductName(String productName) { this.productName = productName; } public long getProductPrice() { return productPrice; } public void setProductPrice(long productPrice) { this.productPrice = productPrice; } public long getPromotionPrice() { return promotionPrice; } public void setPromotionPrice(long promotionPrice) { this.promotionPrice = promotionPrice; } public String getShopId() { return shopId; } public void setShopId(String shopId) { this.shopId = shopId; } public String getShopName() { return shopName; } public void setShopName(String shopName) { this.shopName = shopName; } public String getShopMobile() { return shopMobile; } public void setShopMobile(String shopMobile) { this.shopMobile = shopMobile; } public long getPayPrice() { return payPrice; } public void setPayPrice(long payPrice) { this.payPrice = payPrice; } public int getNum() { return num; } public void setNum(int num) { this.num = num; } @Override public String toString() { return "PaymentInfo{" + "orderId='" + orderId + '\'' + ", createOrderTime=" + createOrderTime + ", paymentId='" + paymentId + '\'' + ", paymentTime=" + paymentTime + ", productId='" + productId + '\'' + ", productName='" + productName + '\'' + ", productPrice=" + productPrice + ", promotionPrice=" + promotionPrice + ", shopId='" + shopId + '\'' + ", shopName='" + shopName + '\'' + ", shopMobile='" + shopMobile + '\'' + ", payPrice=" + payPrice + ", num=" + num + '}'; } public String random() throws ParseException { this.orderId = UUID.randomUUID().toString().replaceAll("-", ""); this.paymentId = UUID.randomUUID().toString().replaceAll("-", ""); this.productPrice = new Random().nextInt(1000); this.promotionPrice = new Random().nextInt(500); this.payPrice = new Random().nextInt(480); this.shopId = new Random().nextInt(200000)+""; this.catagorys = new Random().nextInt(10000)+","+new Random().nextInt(10000)+","+new Random().nextInt(10000); this.province = new Random().nextInt(23)+""; this.city = new Random().nextInt(265)+""; this.county = new Random().nextInt(1489)+""; String date = "2015-11-11 12:22:12"; SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); try { this.createOrderTime = simpleDateFormat.parse(date); } catch (ParseException e) { e.printStackTrace(); } JSONObject obj = new JSONObject(); String jsonString = obj.toJSONString(this); return jsonString; // return new Gson().toJson(this); } }
- 2.定义log4j.properties配置文件
在/src/main/resources/
目录下创建log4h.properties
文件### 设置### log4j.rootLogger = debug,stdout,D,E ### 输出信息到控制抬 ### log4j.appender.stdout = org.apache.log4j.ConsoleAppender log4j.appender.stdout.Target = System.out log4j.appender.stdout.layout = org.apache.log4j.PatternLayout log4j.appender.stdout.layout.ConversionPattern = [%-5p] %d{yyyy-MM-dd HH:mm:ss,SSS} method:%l%n%m%n ### 输出DEBUG 级别以上的日志到=E://logs/error.log ### log4j.appender.D = org.apache.log4j.DailyRollingFileAppender #log4j.appender.D.File = /Users/zhaozhuang/Desktop/logs/log.log log4j.appender.D.File = /export/servers/orderLogs/orderinfo.log log4j.appender.D.Append = true log4j.appender.D.Threshold = DEBUG log4j.appender.D.layout = org.apache.log4j.PatternLayout #log4j.appender.D.layout.ConversionPattern = %-d{yyyy-MM-dd HH:mm:ss} [ %t:%r ] - [ %p ] %m%n log4j.appender.D.layout.ConversionPattern = %m%n ### 输出ERROR 级别以上的日志到=E://logs/error.log ### log4j.appender.E = org.apache.log4j.DailyRollingFileAppender #log4j.appender.E.File = /Users/zhaozhuang/Desktop/logs/error.log log4j.appender.E.File = /export/servers/orderLogs/ordererror.log log4j.appender.E.Append = true log4j.appender.E.Threshold = ERROR log4j.appender.E.layout = org.apache.log4j.PatternLayout #log4j.appender.E.layout.ConversionPattern = %-d{yyyy-MM-dd HH:mm:ss} [ %t:%r ] - [ %p ] %m%n log4j.appender.E.layout.ConversionPattern = %m%n
-
3.开发日志生产代码
package cn.itcast.realboard; import org.apache.log4j.Logger; import java.text.ParseException; public class LogOperate { private static Logger printLogger = Logger.getLogger("printLogger"); public static void main(String[] args) throws ParseException, InterruptedException { PaymentInfo paymentInfo = new PaymentInfo(); while(true){ String random = paymentInfo.random(); System.out.println(random); printLogger.info(random); Thread.sleep(1000); } } }
-
4.将程序打包并上传服务器运行
上传后执行以下命令
java -jar day10_Project-1.0-SNAPSHOT-jar-with-dependencies.jar
-
5.开发flume配置文件,实现收集数据到kafka
开发flume配置文件
cd /export/servers/apache-flume-1.6.0-cdh5.14.0-bin/
vim file_kafka.conf
#为我们的source channel sink起名 a1.sources = r1 a1.channels = c1 a1.sinks = k1 #指定我们的source收集到的数据发送到哪个管道 a1.sources.r1.channels = c1 #指定我们的source数据收集策略 a1.sources.r1.type = TAILDIR a1.sources.r1.positionFile = /var/log/flume/taildir_position.json a1.sources.r1.filegroups = f1 a1.sources.r1.filegroups.f1 = /export/servers/orderLogs/orderinfo.log #指定我们的channel为memory,即表示所有的数据都装进memory当中 a1.channels.c1.type = memory #指定我们的sink为kafka sink,并指定我们的sink从哪个channel当中读取数据 a1.sinks.k1.channel = c1 a1.sinks.k1.type = org.apache.flume.sink.kafka.KafkaSink a1.sinks.k1.kafka.topic = itcast_order a1.sinks.k1.kafka.bootstrap.servers = node01:9092,node02:9092,node03:9092 a1.sinks.k1.kafka.flumeBatchSize = 20 a1.sinks.k1.kafka.producer.acks = 1
启动flume
bin/flume-ng agent -c conf/ -f conf/file_kafka.conf -n a1
-
6.启动kafka,开始消费数据
cd /export/servers/kafka_2.11-1.0.0/
bin/kafka-console-consumer.sh --from-beginning --bootstrap-server node01:9092,node02:9092,node03:9092 --topic itcast_order
四、消费消费代码实现
- 1.定义redis工具类
package cn.itcast.realboard; import org.junit.Test; import redis.clients.jedis.Jedis; import redis.clients.jedis.JedisPool; import redis.clients.jedis.JedisPoolConfig; public class JedisUtils { private static JedisPool pool = null; /** * 获取jedis连接池 **/ public static JedisPool getPool(){ if(pool == null){ //创建jedis连接池配置 JedisPoolConfig config = new JedisPoolConfig(); //最大连接数 config.setMaxTotal(20); //最大空闲连接 config.setMaxIdle(5); //创建redis连接池 pool = new JedisPool(config,"node01",6379,3000); } return pool; } /** * 获取jedis连接 **/ public static Jedis getConn(){ return getPool().getResource(); } @Test public void getJedisTest() throws Exception { Jedis jedis = getPool().getResource(); jedis.incrBy("mine", 5); jedis.close(); } }
- 2.开发Kafka消费代码
package cn.itcast.realboard; import com.alibaba.fastjson.JSONObject; import org.apache.kafka.clients.consumer.ConsumerRecord; import org.apache.kafka.clients.consumer.ConsumerRecords; import org.apache.kafka.clients.consumer.KafkaConsumer; import org.apache.kafka.clients.consumer.OffsetAndMetadata; import org.apache.kafka.common.TopicPartition; import redis.clients.jedis.Jedis; import redis.clients.jedis.JedisPool; import java.util.*; public class MyKafkaConsumer { /** * 消费itcast_order里面的数据 * * @param args */ public static void main(String[] args) { Properties props = new Properties(); props.put("bootstrap.servers", "node01:9092"); props.put("group.id", "test"); //以下两行代码 ---消费者自动提交offset值 props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); props.put("value.deserializer", "org.apache.kafka.common.serialization.StringDeserializer"); props.put("enable.auto.commit", "false"); props.put("auto.commit.interval.ms", "1000"); KafkaConsumer<String, String> kafkaConsumer = new KafkaConsumer<String, String>(props); kafkaConsumer.subscribe(Arrays.asList("itcast_order")); while (true) { ConsumerRecords<String, String> consumerRecords = kafkaConsumer.poll(3000); //获取所有分区 Set<TopicPartition> partitions = consumerRecords.partitions(); for (TopicPartition topicPartition : partitions) { List<ConsumerRecord<String, String>> records = consumerRecords.records(topicPartition); for (ConsumerRecord<String, String> record : records) { //获取Jedis客户端 JedisPool jedisPool = JedisUtils.getPool(); Jedis jedis = jedisPool.getResource(); //获取json字符串 String value = record.value(); //把字符串转换成对象 PaymentInfo paymentInfo = JSONObject.parseObject(value, PaymentInfo.class); //获取付款额度 long payPrice = paymentInfo.getPayPrice(); //redis中的Key一般是约定俗成的,实际工作中一般都有业务逻辑 //求取平台销售总额度 jedis.incrBy("itcast:order:total:price:date",payPrice); //求取平台下单人数 jedis.incr("itcast:order:total:user:date"); //平台销售数量(简单认为一个订单一个商品) jedis.incr("itcast:order:total:num:date"); //每个商品的销售额 jedis.incrBy("itcast:order:"+paymentInfo.getProductId()+"price:date",payPrice); //每个商品的购买人数 jedis.incr("itcast:order:"+paymentInfo.getProductId()+":user:date"); //每个商品销售数量 jedis.incr("itcast:order:"+paymentInfo.getProductId()+":num:date"); //每个店铺的总销售额 jedis.incrBy("itcast:order:"+paymentInfo.getShopId()+":price:date",payPrice); //每个店铺的购买人数 jedis.incr("itcast:order:"+paymentInfo.getShopId()+":user:date"); //每个店铺的销售数量 jedis.incr("itcast:order:"+paymentInfo.getShopId()+":num:date"); jedis.close(); //处理业务逻辑 /** 平台运维角度统计指标 平台总销售额度 redisRowKey设计 itcast:order:total:price:date 平台今天下单人数 redisRowKey设计 itcast:order:total:user:date 平台商品销售数量 redisRowKey设计 itcast:order:total:num:date 商品销售角度统计指标 每个商品的总销售额 Redis的rowKey设计itcast:order:productId:price:date 每个商品的购买人数 Redis的rowKey设计itcast:order:productId:user:date 每个商品的销售数量 Redis的rowKey设计itcast:order:productId:num:date 店铺销售角度统计指标 每个店铺的总销售额 Redis的rowKey设计itcast:order:shopId:price:date 每个店铺的购买人数 Redis的rowKey设计itcast:order:shopId:user:date 每个店铺的销售数量 Redis的rowKey设计itcast:order:shopId:num:date */ } long offset = records.get(records.size() - 1).offset(); Map<TopicPartition, OffsetAndMetadata> topicPartitionOffsetAndMetadataMap = Collections.singletonMap(topicPartition, new OffsetAndMetadata(offset)); kafkaConsumer.commitSync(topicPartitionOffsetAndMetadataMap); } } } }