Hudi-Flink消费kafka将增量数据实时写入Hudi(java)
零、步骤
一、Flink SQL集成Kafka
1.创建topic(一分区一备份)
flink-topic
2.准备flink-sql-connector-kafka_2.12-1.13.1.jar，放入flink/lib下
3.启动client,指定jar
./sql-client.sh embedded -j ../lib/flink-sql-connector-kafka_2.12-1.13.1.jar shell
设置分析结果展示模式为：set execution.result-mode=tableau;
4.创建表，映射到kafka topic
kafka topic中数据是CSV文件格式，有三个字段，user_id、item_id、behavior，从kafka消费数据时，设置从最新偏移量开始
CREATE TABLE test_kafka( `user_id` BIGINT, `item_id` BIGINT, `behavior` STRING ) WITH( 'connector' = 'kafka', 'topic'='flink-topic', 'properties.bootstrap.servers' = 'localhost:9092', 'properties.group.id' = 'test-group-10001', 'scan.startup.mode' = 'latest-offset', 'format' = 'csv' ); Flink SQL> select * from test_kafka; +----+----------------------+----------------------+--------------------------------+ | op | user_id | item_id | behavior | +----+----------------------+----------------------+————————————————+
5.kafka写入数据
kafka-console-producer.sh --broker-list localhost:9092 —-topic flink-topic 1001,90001,click 1001,90001,browser 1001,90001,click 1002,90002,click 1002,90003,click 1003,90001,order 1004,90001,order
MacBook-Pro:bin FengZhen$ kafka-console-producer.sh --broker-list localhost:9092 --topic flink-topic >1001,90001,click 1001,90001,browser 1001,90001,click 1002,90002,click 1002,90003,click 1003,90001,order 1004,90001,order >>>>>>> 数据可实时查询处理 Flink SQL> select * from test_kafka; +----+----------------------+----------------------+--------------------------------+ | op | user_id | item_id | behavior | +----+----------------------+----------------------+--------------------------------+ | +I | 1001 | 90001 | click | | +I | 1001 | 90001 | browser | | +I | 1001 | 90001 | click | | +I | 1002 | 90002 | click | | +I | 1002 | 90003 | click | | +I | 1003 | 90001 | order | | +I | 1004 | 90001 | order |
二、代码实现
package com.zhen.hudi; import org.apache.flink.streaming.api.environment.StreamExecutionEnvironment; import org.apache.flink.table.api.EnvironmentSettings; import org.apache.flink.table.api.Table; import org.apache.flink.table.api.TableEnvironment; import org.apache.flink.table.api.bridge.java.StreamTableEnvironment; import static org.apache.flink.table.api.Expressions.$; /** * @Author FengZhen * @Date 3/9/22 10:17 PM * @Description 基于Flink SQL Connector实现：实时消费topic中数据，转换处理后，实时存储到hudi表中 */ public class FlinkSQLHudiDemo { public static void main(String[] args) { //1.获取表的执行环境 StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment(); //并行度设置为1 env.setParallelism(1); //TODO: 由于增量将数据写入到Hudi表，所以需要启动Flink Checkpoint 检查点 env.enableCheckpointing(5 * 1000); EnvironmentSettings settings = EnvironmentSettings .newInstance() .inStreamingMode()//设置流式模式 .build(); StreamTableEnvironment tableEnv = StreamTableEnvironment.create(env, settings); //2.创建输入表，TODO:从kafka消费数据 tableEnv.executeSql( "CREATE TABLE order_kafka_source(\n" + " `orderId` STRING,\n" + " `userId` STRING,\n" + " `orderTime` STRING,\n" + " `ip` STRING,\n" + " `orderMoney` DOUBLE,\n" + " `orderStatus` INT\n" + ")\n" + "WITH(\n" + " 'connector' = 'kafka',\n" + " 'topic'='order-topic',\n" + " 'properties.bootstrap.servers' = 'localhost:9092',\n" + " 'properties.group.id' = 'gid-1001',\n" + " 'scan.startup.mode' = 'latest-offset',\n" + " 'format' = 'json',\n" + " 'json.fail-on-missing-field' = 'false',\n" + " 'json.ignore-parse-errors' = 'true'\n" + ")\n" ); //3.转换数据，可以使用SQL，也可以是TableAPI Table etlTable = tableEnv .from("order_kafka_source") //添加字段：hudi数据合并的字段，时间戳 .addColumns( $("orderId").substring(0,17).as("ts") ) //添加字段：Hudi表分区字段，"orderTime": 2022-03-09 22:21:13.124 .addColumns( $("orderTime").substring(0, 10).as("partition_day") ); tableEnv.createTemporaryView("view_order", etlTable); //4.创建输出表，TODO:关联到hudi表，指定hudi表名称，存储路径，字段名称等信息 tableEnv.executeSql( "CREATE TABLE order_hudi_sink(\n" + " `orderId` STRING PRIMARY KEY NOT ENFORCED,\n" + " `userId` STRING,\n" + " `orderTime` STRING,\n" + " `ip` STRING,\n" + " `orderMoney` DOUBLE,\n" + " `orderStatus` INT,\n" + " `ts` STRING,\n" + " `partition_day` STRING\n" + ")\n" + "PARTITIONED BY (partition_day)\n" + "WITH(\n" + " 'connector' = 'hudi',\n" + " 'path'='hdfs://localhost:9000/hudi-warehouse/flink_hudi_order',\n" + " 'table.type' = 'MERGE_ON_READ',\n" + " 'write.operation' = 'upsert',\n" + " 'hoodie.datasource.write.recordkey.field' = 'orderId',\n" + " 'write.precombine.field' = 'ts',\n" + " 'write.tasks' = '1'\n" + ")\n" ); //5.通过子查询的方式，将数据写入输出表 tableEnv.executeSql( "INSERT INTO order_hudi_sink " + "SELECT orderId, userId, orderTime, ip, orderMoney, orderStatus, ts, partition_day FROM view_order" ); } }
kafka数据生成工具类
package com.zhen.hudi.streaming import java.util.Properties import org.apache.commons.lang3.time.FastDateFormat import org.apache.kafka.clients.producer.{KafkaProducer, ProducerRecord} import org.apache.kafka.common.serialization.StringSerializer import org.json4s.jackson.Json import scala.util.Random /** * 订单实体类（Case Class） * * @param orderId 订单ID * @param userId 用户ID * @param orderTime 订单日期时间 * @param ip 下单IP地址 * @param orderMoney 订单金额 * @param orderStatus 订单状态 */ case class OrderRecord( orderId: String, userId: String, orderTime: String, ip: String, orderMoney: Double, orderStatus: Int ) /** * @Author FengZhen * @Date 3/3/22 9:54 PM * @Description TODO * 模拟生产订单数据，发送到Kafka Topic中 * Topic中每条数据Message类型为String，以JSON格式数据发送 * 数据转换： * 将Order类实例对象转换为JSON格式字符串数据（可以使用json4s类库） */ object MockOrderProducer { def main(args: Array[String]): Unit = { var producer: KafkaProducer[String, String] = null try { // 1. Kafka Client Producer 配置信息 val props = new Properties() props.put("bootstrap.servers", "localhost:9092") props.put("acks", "1") props.put("retries", "3") // props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer") // props.put("value.serializer", "org.apache.kafka.common.serialization.StringSerializer") props.put("key.serializer", classOf[StringSerializer].getName) props.put("value.serializer", classOf[StringSerializer].getName) // 2. 创建KafkaProducer对象，传入配置信息 producer = new KafkaProducer[String, String](props) // 随机数实例对象 val random: Random = new Random() // 订单状态：订单打开 0，订单取消 1，订单关闭 2，订单完成 3 val allStatus = Array(0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) while (true) { // 每次循环 模拟产生的订单数目 val batchNumber: Int = random.nextInt(1) + 1 (1 to batchNumber).foreach { number => val currentTime: Long = System.currentTimeMillis() val orderId: String = s"${getDate(currentTime)}%06d".format(number) val userId: String = s"${1 + random.nextInt(5)}%08d".format(random.nextInt(1000)) val orderTime: String = getDate(currentTime, format = "yyyy-MM-dd HH:mm:ss.SSS") val orderMoney: String = s"${5 + random.nextInt(500)}.%02d".format(random.nextInt(100)) val orderStatus: Int = allStatus(random.nextInt(allStatus.length)) // 3. 订单记录数据 val orderRecord: OrderRecord = OrderRecord( orderId, userId, orderTime, getRandomIp, orderMoney.toDouble, orderStatus ) // 转换为JSON格式数据 val orderJson = new Json(org.json4s.DefaultFormats).write(orderRecord) println(orderJson) // 4. 构建ProducerRecord对象 val record = new ProducerRecord[String, String]("order-topic", orderId, orderJson) // 5. 发送数据：def send(messages: KeyedMessage[K,V]*), 将数据发送到Topic producer.send(record) } Thread.sleep(random.nextInt(500) + 5000) } } catch { case e: Exception => e.printStackTrace() } finally { if (null != producer) producer.close() } } /** =================获取当前时间================= */ def getDate(time: Long, format: String = "yyyyMMddHHmmssSSS"): String = { val fastFormat: FastDateFormat = FastDateFormat.getInstance(format) val formatDate: String = fastFormat.format(time) // 格式化日期 formatDate } /** ================= 获取随机IP地址 ================= */ def getRandomIp: String = { // ip范围 val range: Array[(Int, Int)] = Array( (607649792, 608174079), //36.56.0.0-36.63.255.255 (1038614528, 1039007743), //61.232.0.0-61.237.255.255 (1783627776, 1784676351), //106.80.0.0-106.95.255.255 (2035023872, 2035154943), //121.76.0.0-121.77.255.255 (2078801920, 2079064063), //123.232.0.0-123.235.255.255 (-1950089216, -1948778497), //139.196.0.0-139.215.255.255 (-1425539072, -1425014785), //171.8.0.0-171.15.255.255 (-1236271104, -1235419137), //182.80.0.0-182.92.255.255 (-770113536, -768606209), //210.25.0.0-210.47.255.255 (-569376768, -564133889) //222.16.0.0-222.95.255.255 ) // 随机数：IP地址范围下标 val random = new Random() val index = random.nextInt(10) val ipNumber: Int = range(index)._1 + random.nextInt(range(index)._2 - range(index)._1) // 转换Int类型IP地址为IPv4格式 number2IpString(ipNumber) } /** =================将Int类型IPv4地址转换为字符串类型================= */ def number2IpString(ip: Int): String = { val buffer: Array[Int] = new Array[Int](4) buffer(0) = (ip >> 24) & 0xff buffer(1) = (ip >> 16) & 0xff buffer(2) = (ip >> 8) & 0xff buffer(3) = ip & 0xff // 返回IPv4地址 buffer.mkString(".") } }
