Flink 实现订单支付实时监控
需求
对订单信息流进行监控,15分钟之内没有支付的发出警告
Flink CEP 实现
import org.apache.flink.cep.scala.{CEP, PatternStream} import org.apache.flink.cep.scala.pattern.Pattern import org.apache.flink.streaming.api.TimeCharacteristic import org.apache.flink.streaming.api.scala._ import org.apache.flink.streaming.api.windowing.time.Time import org.apache.flink.util.Collector import scala.collection.Map object OrderTimeout { case class OrderEvent(orderId: String, eventType: String, eventTime: Long) def main(args: Array[String]): Unit = { val env = StreamExecutionEnvironment.getExecutionEnvironment env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime) env.setParallelism(1) val stream: KeyedStream[OrderEvent, String] = env .fromElements( OrderEvent("order_1", "create", 2000L), OrderEvent("order_2", "create", 3000L), OrderEvent("order_2", "pay", 4000L) ) .assignAscendingTimestamps(_.eventTime) .keyBy(_.orderId) //定义匹配模板 val pattern: Pattern[OrderEvent, OrderEvent] = Pattern .begin[OrderEvent]("create").where(_.eventType.equals("create")) .next("pay").where(_.eventType.equals("pay"))//严格近邻 .within(Time.minutes(15))//15分钟之内 //将流和匹配模板输入,得到匹配后的流 val patternedStream: PatternStream[OrderEvent] = CEP.pattern(stream, pattern) // 用来输出超时订单的侧输出标签 val orderTimeoutOutput: OutputTag[String] = new OutputTag[String]("timeout") // 用来处理超时订单的函数 val timeoutFunc: (Map[String, Iterable[OrderEvent]], Long, Collector[String]) => Unit = (map: Map[String, Iterable[OrderEvent]], ts: Long, out: Collector[String]) => { println("ts" + ts) // 2s + 5s 5s的最大延迟时间 //这个名字是之前在定义模式时每个个体模式取得名字 val orderStart: OrderEvent = map("create").head // 等价于map.getOrElse("create", null).iterator.next() // 将报警信息发送到侧输出流去 out.collect(orderStart.orderId + "没有支付!") } //处理没有超时订单的函数 //map是Scala的map,注意导包的准确!!! val selectFunc: (Map[String, Iterable[OrderEvent]], Collector[String]) => Unit = (map: Map[String, Iterable[OrderEvent]], out: Collector[String]) => { val order: OrderEvent = map("pay").head out.collect(order.orderId + "已经支付!") } val outputStream = patternedStream //柯里化,传入三个参数 // 第一个参数:用来输出超时事件的侧输出标签 // 第二个参数:用来输出超时事件的函数 // 第三个参数:用来输出没有超时的事件的函数 .flatSelect(outputTag = orderTimeoutOutput)(patternFlatTimeoutFunction = timeoutFunc)(patternFlatSelectFunction = selectFunc) outputStream.print() outputStream.getSideOutput(new OutputTag[String]("timeout")).print() env.execute() } }
Flink 底层API实现
import org.apache.flink.api.common.state.{ValueState, ValueStateDescriptor} import org.apache.flink.streaming.api.TimeCharacteristic import org.apache.flink.streaming.api.functions.KeyedProcessFunction import org.apache.flink.streaming.api.scala._ import org.apache.flink.util.Collector object OrderTimeoutWithoutCep { case class OrderEvent(orderId: String, eventType: String, eventTime: Long) def main(args: Array[String]): Unit = { val env = StreamExecutionEnvironment.getExecutionEnvironment env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime) env.setParallelism(1) println("111") val stream: DataStream[String] = env .fromElements( OrderEvent("order_1", "create", 2000L), OrderEvent("order_2", "create", 3000L), OrderEvent("order_2", "pay", 4000L), OrderEvent("order_1", "pay", 10000L) ) .setParallelism(1) .assignAscendingTimestamps(_.eventTime) .keyBy(_.orderId) .process(new OrderTimeoutFunc) val timeoutOutput = new OutputTag[String]("timeout") stream.getSideOutput(timeoutOutput).print() stream.print() env.execute() } class OrderTimeoutFunc extends KeyedProcessFunction[String, OrderEvent, String] { lazy val orderState: ValueState[OrderEvent] = getRuntimeContext.getState( new ValueStateDescriptor[OrderEvent]("saved order", classOf[OrderEvent]) ) override def processElement(value: OrderEvent, ctx: KeyedProcessFunction[String, OrderEvent, String]#Context, out: Collector[String]): Unit = { if (value.eventType.equals("create")) { // 到来的事件是下订单事件 if (orderState.value() == null) { // 要判空,因为pay事件可能先到 orderState.update(value) // 将create事件存到状态变量 ctx.timerService().registerEventTimeTimer(value.eventTime + 5000L) } } else { orderState.update(value) // 将pay事件保存到状态变量 out.collect("已经支付的订单ID是:" + value.orderId) } } override def onTimer(timestamp: Long, ctx: KeyedProcessFunction[String, OrderEvent, String]#OnTimerContext, out: Collector[String]): Unit = { val order: OrderEvent = orderState.value() //如果状态仍然为create则报警 if (order != null && order.eventType.equals("create")) { ctx.output(new OutputTag[String]("timeout"), "超时订单的ID为:" + order.orderId) } orderState.clear() } } }