sparkStreaming消费kafka-1.0.1方式：direct方式（存储offset到Hbase）
话不多说，可以看上篇博文，关于offset存储到zookeeper
https://www.cnblogs.com/niutao/p/10547718.html
本篇博文主要告诉你如何将offset写到Hbase做存储：
最后存储到Hbase的展现形式：
testDirect:co:1552667595000 column=info:0, timestamp=1552667594784, value=66 testDirect:co:1552667595000 column=info:1, timestamp=1552667594784, value=269 testDirect:co:1552667595000 column=info:2, timestamp=1552667594784, value=67 testDirect:co:1552667600000 column=info:0, timestamp=1552667599864, value=66 testDirect:co:1552667600000 column=info:1, timestamp=1552667599864, value=269 testDirect:co:1552667600000 column=info:2, timestamp=1552667599864, value=67 testDirect:co:1552667605000 column=info:0, timestamp=1552667604778, value=66 testDirect:co:1552667605000 column=info:1, timestamp=1552667604778, value=269 testDirect:co:1552667605000 column=info:2, timestamp=1552667604778, value=67 testDirect:co:1552667610000 column=info:0, timestamp=1552667609777, value=66 testDirect:co:1552667610000 column=info:1, timestamp=1552667609777, value=269
版本：
scala:2.11.8
spark:2.11
hbase:1.2.0-cdh5.14.0
遇到的问题：
`java.lang.IllegalStateException: Consumer is not subscribed to any topics or assigned any partitions`
分析原因：
从指定的主题或者分区获取数据，在poll之前，你没有订阅任何主题或分区是不行的,每一次poll,消费者都会尝试使用最后一次消费的offset作为接下来获取数据的start offset，最后一次消费的offset也可以通过seek(TopicPartition, long)设置或者自动设置
通过源码可以找到：
public ConsumerRecords<K, V> poll(long timeout) { acquire(); try { if (timeout < 0) throw new IllegalArgumentException("Timeout must not be negative"); // 如果没有任何订阅，抛出异常 if (this.subscriptions.hasNoSubscriptionOrUserAssignment()) throw new IllegalStateException("Consumer is not subscribed to any topics or assigned any partitions"); // 一直poll新数据直到超时 long start = time.milliseconds(); // 距离超时还剩余多少时间 long remaining = timeout; do { // 获取数据，如果自动提交，则进行偏移量自动提交，如果设置offset重置，则进行offset重置 Map<TopicPartition, List<ConsumerRecord<K, V>>> records = pollOnce(remaining); if (!records.isEmpty()) { // 再返回结果之前，我们可以进行下一轮的fetch请求，避免阻塞等待 fetcher.sendFetches(); client.pollNoWakeup(); // 如果有拦截器进行拦截，没有直接返回 if (this.interceptors == null) return new ConsumerRecords<>(records); else return this.interceptors.onConsume(new ConsumerRecords<>(records)); } long elapsed = time.milliseconds() - start; remaining = timeout - elapsed; } while (remaining > 0); return ConsumerRecords.empty(); } finally { release(); } }
解决：
因此，需要订阅当前的topic才能消费，我之前使用的api是：（适用于非新--已经被消费者消费过的）
因此，需要订阅当前的topic才能消费，我之前使用的api是：（适用于非新--已经被消费者消费过的） `val inputDStream1 = KafkaUtils.createDirectStream[String, String]( ssc, PreferConsistent, Assign[String, String]( fromOffsets.keys,kafkaParams,fromOffsets) )` 修改：（全新的topic，没有被消费者消费过） `val inputDStream = KafkaUtils.createDirectStream[String, String]( ssc, PreferConsistent, Subscribe[String, String](topics, kafkaParams) )`
完整代码：
package offsetInHbase import kafka.utils.ZkUtils import org.apache.hadoop.hbase.filter.PrefixFilter import org.apache.hadoop.hbase.util.Bytes import org.apache.hadoop.hbase.{TableName, HBaseConfiguration} import org.apache.hadoop.hbase.client.{Scan, Put, ConnectionFactory} import org.apache.kafka.clients.consumer.ConsumerRecord import org.apache.kafka.common.TopicPartition import org.apache.kafka.common.serialization.StringDeserializer import org.apache.spark.streaming.kafka010.ConsumerStrategies._ import org.apache.spark.streaming.kafka010.{OffsetRange, HasOffsetRanges, KafkaUtils} import org.apache.spark.streaming.kafka010.LocationStrategies._ import org.apache.spark.streaming.{Seconds, StreamingContext} import org.apache.spark.{SparkContext, SparkConf} /** * Created by angel */ object KafkaOffsetsBlogStreamingDriver { def main(args: Array[String]) { if (args.length < 6) { System.err.println("Usage: KafkaDirectStreamTest " + "<batch-duration-in-seconds> " + "<kafka-bootstrap-servers> " + "<kafka-topics> " + "<kafka-consumer-group-id> " + "<hbase-table-name> " + "<kafka-zookeeper-quorum>") System.exit(1) } //5 cdh1:9092,cdh2:2181,cdh3:2181 testDirect co testDirect cdh1:2181,cdh2:2181,cdh3:2181 val batchDuration = args(0) val bootstrapServers = args(1).toString val topicsSet = args(2).toString.split(",").toSet val consumerGroupID = args(3) val hbaseTableName = args(4) val zkQuorum = args(5) val zkKafkaRootDir = "kafka" val zkSessionTimeOut = 10000 val zkConnectionTimeOut = 10000 val sparkConf = new SparkConf().setAppName("Kafka-Offset-Management-Blog") .setMaster("local[4]")//Uncomment this line to test while developing on a workstation val sc = new SparkContext(sparkConf) val ssc = new StreamingContext(sc, Seconds(batchDuration.toLong)) val topics = topicsSet.toArray val topic = topics(0) val kafkaParams = Map[String, Object]( "bootstrap.servers" -> bootstrapServers, "key.deserializer" -> classOf[StringDeserializer], "value.deserializer" -> classOf[StringDeserializer], "group.id" -> consumerGroupID, "auto.offset.reset" -> "latest", "enable.auto.commit" -> (false: java.lang.Boolean) ) /* Create a dummy process that simply returns the message as is. */ def processMessage(message:ConsumerRecord[String,String]):ConsumerRecord[String,String]={ message } /* Save Offsets into HBase */ def saveOffsets( TOPIC_NAME:String, GROUP_ID:String, offsetRanges:Array[OffsetRange], hbaseTableName:String, batchTime: org.apache.spark.streaming.Time ) ={ val hbaseConf = HBaseConfiguration.create() hbaseConf.addResource("src/main/resources/hbase-site.xml") val conn = ConnectionFactory.createConnection(hbaseConf) val table = conn.getTable(TableName.valueOf(hbaseTableName)) val rowKey = TOPIC_NAME + ":" + GROUP_ID + ":" + String.valueOf(batchTime.milliseconds) val put = new Put(rowKey.getBytes) for(offset <- offsetRanges){ put.addColumn(Bytes.toBytes("info"),Bytes.toBytes(offset.partition.toString), Bytes.toBytes(offset.untilOffset.toString)) } table.put(put) conn.close() } /* Returns last committed offsets for all the partitions of a given topic from HBase in following cases. - CASE 1: SparkStreaming job is started for the first time. This function gets the number of topic partitions from Zookeeper and for each partition returns the last committed offset as 0 - CASE 2: SparkStreaming is restarted and there are no changes to the number of partitions in a topic. Last committed offsets for each topic-partition is returned as is from HBase. - CASE 3: SparkStreaming is restarted and the number of partitions in a topic increased. For old partitions, last committed offsets for each topic-partition is returned as is from HBase as is. For newly added partitions, function returns last committed offsets as 0 */ def getLastCommittedOffsets( TOPIC_NAME:String, GROUP_ID:String, hbaseTableName:String, zkQuorum:String, zkRootDir:String, sessionTimeout:Int, connectionTimeOut:Int ):Map[TopicPartition,Long] ={ val hbaseConf = HBaseConfiguration.create() hbaseConf.addResource("src/main/resources/hbase-site.xml") val zkUrl = zkQuorum+"/"+zkRootDir val zkClientAndConnection = ZkUtils.createZkClientAndConnection(zkUrl,sessionTimeout,connectionTimeOut) val zkUtils = new ZkUtils(zkClientAndConnection._1, zkClientAndConnection._2,false) val zKNumberOfPartitionsForTopic = zkUtils.getPartitionsForTopics(Seq(TOPIC_NAME)).get(TOPIC_NAME).toList.head.size //Connect to HBase to retrieve last committed offsets val conn = ConnectionFactory.createConnection(hbaseConf) val table = conn.getTable(TableName.valueOf(hbaseTableName)) val startRow = TOPIC_NAME + ":" + GROUP_ID + ":" + String.valueOf(System.currentTimeMillis()) val stopRow = TOPIC_NAME + ":" + GROUP_ID + ":" + 0 val scan = new Scan() val scanner = table.getScanner(scan.setStartRow(startRow.getBytes).setStopRow(stopRow.getBytes).setReversed(true)) val result = scanner.next() //Set the number of partitions discovered for a topic in HBase to 0 var hbaseNumberOfPartitionsForTopic = 0 if (result != null){ //If the result from hbase scanner is not null, set number of partitions from hbase to the number of cells //listCells 获取列族下的列 hbaseNumberOfPartitionsForTopic = result.listCells().size() } val fromOffsets = collection.mutable.Map[TopicPartition,Long]() //初始化时候的hbase if(hbaseNumberOfPartitionsForTopic == 0){ // initialize fromOffsets to beginning for (partition <- 0 to zKNumberOfPartitionsForTopic-1){ fromOffsets += (new TopicPartition(TOPIC_NAME,partition) -> 0)} //增加了topic的分区数 } else if(zKNumberOfPartitionsForTopic > hbaseNumberOfPartitionsForTopic){ // handle scenario where new partitions have been added to existing kafka topic for (partition <- 0 to hbaseNumberOfPartitionsForTopic-1){ val fromOffset = Bytes.toString(result.getValue(Bytes.toBytes("info"),Bytes.toBytes(partition.toString))) fromOffsets += (new TopicPartition(TOPIC_NAME,partition) -> fromOffset.toLong)} //将新增的分区也添加上 for (partition <- hbaseNumberOfPartitionsForTopic to zKNumberOfPartitionsForTopic-1){ fromOffsets += (new TopicPartition(TOPIC_NAME,partition) -> 0)} } else { //initialize fromOffsets from last run for (partition <- 0 to hbaseNumberOfPartitionsForTopic-1 ){ val fromOffset = Bytes.toString(result.getValue(Bytes.toBytes("info"),Bytes.toBytes(partition.toString))) fromOffsets += (new TopicPartition(TOPIC_NAME,partition) -> fromOffset.toLong)} } scanner.close() conn.close() fromOffsets.toMap } val fromOffsets= getLastCommittedOffsets( topic, consumerGroupID, hbaseTableName, zkQuorum, zkKafkaRootDir, zkSessionTimeOut, zkConnectionTimeOut) //刚开始时候启动，全新的topic会报错 val inputDStream = KafkaUtils.createDirectStream[String, String]( ssc, PreferConsistent, Assign[String, String]( fromOffsets.keys,kafkaParams,fromOffsets) ) //如果报错，则使用下面的api // val inputDStream = KafkaUtils.createDirectStream[String, String]( // ssc, // PreferConsistent, // Subscribe[String, String](topics, kafkaParams) // ) /* For each RDD in a DStream apply a map transformation that processes the message. */ inputDStream.foreachRDD((rdd,batchTime) => { val offsetRanges = rdd.asInstanceOf[HasOffsetRanges].offsetRanges offsetRanges.foreach(offset => println(offset.topic, offset.partition, offset.fromOffset,offset.untilOffset)) val newRDD = rdd.map(message => processMessage(message)) newRDD.count() saveOffsets(topic,consumerGroupID,offsetRanges,hbaseTableName,batchTime) //save the offsets to HBase }) println("Number of messages processed " + inputDStream.count()) ssc.start() ssc.awaitTermination() } }
