DStream-03 Kafka offset 原理和源码

DEMO

object KafkaDirectDstream {
  def main(args: Array[String]): Unit = {
    val sparkConf = new SparkConf().setAppName("KafkaDirectDstream")
    sparkConf.setMaster("local[*]")
    sparkConf.set("spark.streaming.kafka.maxRatePerPartition", "1")
    sparkConf.set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
    val streamingContext = new StreamingContext(sparkConf, Seconds(2))
    streamingContext.sparkContext.setLogLevel("ERROR")
    val kafkaParams = Map[String, Object](
      "bootstrap.servers" -> "s1:9092",
      "key.deserializer" -> classOf[StringDeserializer],
      "value.deserializer" -> classOf[StringDeserializer],
      "group.id" -> "p1",
      "auto.offset.reset" -> "earliest",
      "enable.auto.commit" -> (false: java.lang.Boolean)
    )
    val topics = Array("test_topic")
    // 这边就是入口
    val dstream = KafkaUtils.createDirectStream[String, String](
      streamingContext,
      PreferConsistent,
      Subscribe[String, String](topics, kafkaParams)
    )

    dstream.map(record => (record.key, record.value, record.partition(), record.offset()))
      .foreachRDD(rdd => {
        println(s" Time: ${System.currentTimeMillis() / 1000}")
        
        rdd.mapPartitionsWithIndex((p, it) => {
          println(s" partition:${p} count:${it.count(x=>true)}")
          it
        }).foreachPartition(v=>null)

      })
    streamingContext.start()
    streamingContext.awaitTermination()
  }
}

KafkaUtils

如果创建DirectKafkaInputDStream 时如果没有传 perPartitionConfig 则就会使用 PerPartitionConfig

def createDirectStream[K, V](
    ssc: StreamingContext,
    locationStrategy: LocationStrategy,
    consumerStrategy: ConsumerStrategy[K, V]
  ): InputDStream[ConsumerRecord[K, V]] = {
  val ppc = new DefaultPerPartitionConfig(ssc.sparkContext.getConf)
  createDirectStream[K, V](ssc, locationStrategy, consumerStrategy, ppc)
}

def createDirectStream[K, V](
    ssc: StreamingContext,
    locationStrategy: LocationStrategy,
    consumerStrategy: ConsumerStrategy[K, V],
    perPartitionConfig: PerPartitionConfig
  ): InputDStream[ConsumerRecord[K, V]] = {
  new DirectKafkaInputDStream[K, V](ssc, locationStrategy, consumerStrategy, perPartitionConfig)
}

PerPartitionConfig

最关键的就是包含了 spark.streaming.kafka.maxRatePerPartition 和 spark.streaming.kafka.minRatePerPartition

private class DefaultPerPartitionConfig(conf: SparkConf)
    extends PerPartitionConfig {
  val maxRate = conf.getLong("spark.streaming.kafka.maxRatePerPartition", 0)
  val minRate = conf.getLong(" ", 1)

  def maxRatePerPartition(topicPartition: TopicPartition): Long = maxRate
  override def minRatePerPartition(topicPartition: TopicPartition): Long = minRate
}

DirectKafkaInputDStream

在每次发送GenerateJobs的消息时，就会触发Dstream的getOrCompute 或 compute

override def compute(validTime: Time): Option[KafkaRDD[K, V]] = {
  val untilOffsets = clamp(latestOffsets())
  val offsetRanges = untilOffsets.map { case (tp, uo) =>
    val fo = currentOffsets(tp)
    OffsetRange(tp.topic, tp.partition, fo, uo)
  }
  val useConsumerCache = context.conf.getBoolean("spark.streaming.kafka.consumer.cache.enabled",
    true)
  val rdd = new KafkaRDD[K, V](context.sparkContext, executorKafkaParams, offsetRanges.toArray,
    getPreferredHosts, useConsumerCache)

  // Report the record number and metadata of this batch interval to InputInfoTracker.
  val description = offsetRanges.filter { offsetRange =>
    // Don't display empty ranges.
    offsetRange.fromOffset != offsetRange.untilOffset
  }.map { offsetRange =>
    s"topic: ${offsetRange.topic}\tpartition: ${offsetRange.partition}\t" +
      s"offsets: ${offsetRange.fromOffset} to ${offsetRange.untilOffset}"
  }.mkString("\n")
  // Copy offsetRanges to immutable.List to prevent from being modified by the user
  val metadata = Map(
    "offsets" -> offsetRanges.toList,
    StreamInputInfo.METADATA_KEY_DESCRIPTION -> description)
  val inputInfo = StreamInputInfo(id, rdd.count, metadata)
  ssc.scheduler.inputInfoTracker.reportInfo(validTime, inputInfo)

  currentOffsets = untilOffsets
  commitAll()
  Some(rdd)
}

// limits the maximum number of messages per partition 
// 限制每个分区的数据量
// offsets 是最新的每个partition的offset map

protected def clamp(
  offsets: Map[TopicPartition, Long]): Map[TopicPartition, Long] = {

  maxMessagesPerPartition(offsets).map { mmp =>
    mmp.map { case (tp, messages) =>
    	// 从map 通过分区获取 最新的offset
        val uo = offsets(tp)
        // tp 指向 当前offset + messages 和最新的offset 的最小值。
        // message 就是一个批次的条数 offset 到 offset+message 就是下一个批次的offset 范围。
        // 取最小值就是为了避便这个范围超过了 最新的offset
        tp -> Math.min(currentOffsets(tp) + messages, uo)
    }
  }.getOrElse(offsets)
}

// 计算每个partition的offset 这边设计到反压机制，默认反压 是关闭的
protected[streaming] def maxMessagesPerPartition(
  offsets: Map[TopicPartition, Long]): Option[Map[TopicPartition, Long]] = {
  
  //默认没有开启反压，这边就是None
  val estimatedRateLimit = rateController.map { x => {
    val lr = x.getLatestRate()
    if (lr > 0) lr else initialRate
  }}

  // calculate a per-partition rate limit based on current lag
  val effectiveRateLimitPerPartition = estimatedRateLimit.filter(_ > 0) match {
    case Some(rate) =>
      val lagPerPartition = offsets.map { case (tp, offset) =>
        tp -> Math.max(offset - currentOffsets(tp), 0)
      }
      val totalLag = lagPerPartition.values.sum

      lagPerPartition.map { case (tp, lag) =>
        val maxRateLimitPerPartition = ppc.maxRatePerPartition(tp)
        val backpressureRate = lag / totalLag.toDouble * rate
        tp -> (if (maxRateLimitPerPartition > 0) {
          Math.min(backpressureRate, maxRateLimitPerPartition)} else backpressureRate)
      }
    case None => offsets.map { case (tp, offset) => 
    	// 取出spark.streaming.kafka.maxRatePerPartition 每个分区的条数就是 这个配置的值
    	tp -> ppc.maxRatePerPartition(tp).toDouble }
  }

  if (effectiveRateLimitPerPartition.values.sum > 0) {
    val secsPerBatch = context.graph.batchDuration.milliseconds.toDouble / 1000
    Some(effectiveRateLimitPerPartition.map {
      // 然后取 每个批次的秒数 * maxRatePerPartition 和 minRatePerPartition 最大值
      // 也就是 tp-> limit (上个方法的 message)
      case (tp, limit) => tp -> Math.max((secsPerBatch * limit).toLong,
        ppc.minRatePerPartition(tp))
    })
  } else {
    None
  }
}

// 判断是否反压，没有有时None
override protected[streaming] val rateController: Option[RateController] = {
  if (RateController.isBackPressureEnabled(ssc.conf)) {
    Some(new DirectKafkaRateController(id,
      RateEstimator.create(ssc.conf, context.graph.batchDuration)))
  } else {
    None
  }
}

正常情况就是

maxRatePerPartition * second.interval 就是一个批次中一个分区的数据量

maxRatePerPartition * second.interval*partition 就是一个整个批次的处理数据量