MetaQ对接SparkStreaming示例代码
由于JavaReceiverInputDStream<String> lines = ssc.receiverStream(Receiver<T> receiver) 中 没有直接对接MetaQ的工具，当然可以实用使用spark streaming已经有的工具进行转接，这里不建议，所以可以继承Receiver类重写onStart()方法
import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.Date; import java.util.concurrent.Executor; import org.apache.spark.storage.StorageLevel; import org.apache.spark.streaming.receiver.Receiver; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.dinpay.bdp.rcp.domain.Order; import com.taobao.metamorphosis.Message; import com.taobao.metamorphosis.client.MessageSessionFactory; import com.taobao.metamorphosis.client.MetaClientConfig; import com.taobao.metamorphosis.client.MetaMessageSessionFactory; import com.taobao.metamorphosis.client.consumer.ConsumerConfig; import com.taobao.metamorphosis.client.consumer.MessageConsumer; import com.taobao.metamorphosis.client.consumer.MessageListener; import com.taobao.metamorphosis.utils.ZkUtils.ZKConfig; public abstract class MetaQReceiver<T> extends Receiver<T>{ private static final long serialVersionUID = -3240967436204273248L; Logger logger=LoggerFactory.getLogger(MetaQReceiver.class); private static final DateFormat df = new SimpleDateFormat("yyyyMMdd"); private String zkConnect; private String zkRoot; private String topic; private String group; public MetaQReceiver(String zkConnect,String zkRoot,String topic,String group) { super(StorageLevel.MEMORY_ONLY()); this.zkConnect=zkConnect; this.zkRoot=zkRoot; this.topic=topic; this.group=group; } @Override public void onStart() { try{ final MetaClientConfig metaClientConfig = new MetaClientConfig(); final ZKConfig zkConfig = new ZKConfig(); zkConfig.zkConnect = this.zkConnect;// "127.0.0.1:2181"; zkConfig.zkRoot = this.zkRoot;// "/meta"; metaClientConfig.setZkConfig(zkConfig); final MessageSessionFactory sessionFactory = new MetaMessageSessionFactory( metaClientConfig); ConsumerConfig consumerConfig = new ConsumerConfig(group); // 默认最大获取延迟为5秒，这里设置成100毫秒，请根据实际应用要求做设置。 consumerConfig.setMaxDelayFetchTimeInMills(100); final MessageConsumer consumer = sessionFactory .createConsumer(consumerConfig); // subscribe topic consumer.subscribe(topic, 1024 * 1024, new MessageListener() { @Override public void recieveMessages(final Message message) { try{ //T t=message2Object(new String(message.getData(),"utf-8")); logger.info("Receive message " + new String(message.getData())); String orderJson = new String(message.getData()); Order order = ParameterDataUtil.getObject(orderJson, Order.class); String cardNo = order.getCard_no(); String yyyyMMdd = df.format(new Date()); String payclassId = order.getPayclass_id(); String cntKey = "DK_CNT_" + cardNo + "_" + payclassId + "_" + yyyyMMdd; logger.info(cntKey); System.out.println(cntKey); T result = (T) cntKey; if(result!=null){ store(result); } }catch(Exception e){ logger.error("message2Object error",e); } } @Override public Executor getExecutor() { return null; } }); consumer.completeSubscribe(); }catch(Exception e){ throw new RuntimeException("metaq error",e); } } @Override public void onStop() { } //public abstract T message2Object(String message) throws Exception; }
下面该段代码可以减掉，若有需要转Object可以在此进行处理
public class MetaQReceiverStreaming extends MetaQReceiver<String>{ private static final long serialVersionUID = -2290689243756756929L; public MetaQReceiverStreaming(String zkConnect, String zkRoot, String topic, String group) { super(zkConnect, zkRoot, topic, group); } /*@Override public String message2Object(String message) throws Exception { return message; }*/ }
接下来通过spark streaming进行metaq的消息处理
ort java.util.Arrays; import java.util.List; import org.apache.log4j.Level; import org.apache.log4j.Logger; import org.apache.spark.SparkConf; import org.apache.spark.api.java.function.FlatMapFunction; import org.apache.spark.api.java.function.Function2; import org.apache.spark.api.java.function.PairFunction; import org.apache.spark.streaming.Durations; import org.apache.spark.streaming.api.java.JavaDStream; import org.apache.spark.streaming.api.java.JavaPairDStream; import org.apache.spark.streaming.api.java.JavaReceiverInputDStream; import org.apache.spark.streaming.api.java.JavaStreamingContext; import com.dinpay.bdp.rcp.util.Constant; import com.dinpay.bdp.rcp.util.MetaQReceiverStreaming; import com.google.common.base.Optional; import com.sun.xml.bind.v2.runtime.reflect.opt.Const; import scala.Tuple2; /** * @author ll */ public class MetaqStreamingCount { public static void main(String[] args) { String zkConnect=Constant.METAZK; String zkRoot="/meta"; String topic=Constant.METATOPIC; String group=Constant.METAGROUP; //屏蔽日志 Logger.getLogger("org.apache.spark").setLevel(Level.OFF); SparkConf sparkConf = new SparkConf().setAppName("MetaqStreamingCount").setMaster("local[2]"); JavaStreamingContext ssc = new JavaStreamingContext(sparkConf, Durations.seconds(1)); JavaReceiverInputDStream<String> lines = ssc.receiverStream(new MetaQReceiverStreaming(zkConnect,zkRoot,topic,group)); JavaDStream<String> words = lines.flatMap(new FlatMapFunction<String, String>() { @Override public Iterable<String> call(String line) throws Exception { return Arrays.asList(line.split(" ")); } }); JavaPairDStream<String, Integer> pairs = words.mapToPair(new PairFunction<String, String, Integer>() { @Override public Tuple2<String, Integer> call(String word) { return new Tuple2<>(word, 1); } }); JavaPairDStream<String, Integer> wordsCount = pairs.updateStateByKey(new Function2<List<Integer>, Optional<Integer>, Optional<Integer>>() {//对相同的Key，进行Value的累计（包括Local和Reducer级别同时Reduce） @Override public Optional<Integer> call(List<Integer> values, Optional<Integer> state){ //第一个参数就是key传进来的数据，第二个参数是已经有的数据 Integer updateValue = 0;//如果第一次，state没有，updateValue为0，如果有就获取 if(state.isPresent()){ updateValue = state.get(); } //遍历batch传进来的数据可以一直加，随着时间的流式会不断的累加相同key的value结果 for (Integer value : values) { updateValue += value; } return Optional.of(updateValue);//返回更新的值 } }); wordsCount.print(); //需要将结果保存到Codis中 ssc.checkpoint("checkpoint"); ssc.start(); ssc.awaitTermination(); ssc.close(); } }
