思考kafka在做幂等性的时候，producerId在内存中什么时候清理掉

情景

　　最近在看kafka幂等性的源码的时候，在思考一个问题，既然幂等性是通过producerId + Sequence Number来判断是否重复，那么应该在broker缓存中，有保存producerId 和 Sequence Number，那么如果长时间一直使用，是否会由于 producerId 和 Sequence Number 的增长，造成OOM呢？在网上没找到答案，所以本文通过源码，来找到这个答案

代码追踪

　　本文不再从具体的producer代码分析了，直接从ApiKeys的代码开始，直接追踪，前面有不理解的朋友，请参考：【转载】万字长文干货 | Kafka 事务性之幂等性实现 

　　【步骤1】kafka.server.KafkaApis#handle

def handle(request: RequestChannel.Request) {
  try {
    ...
    request.header.apiKey match {
      //处理发上来的请求
      case ApiKeys.PRODUCE => handleProduceRequest(request)
      ...
    }
  } catch {
    ...
  } finally {
    ...
  }
}

　　

　　【步骤2】kafka.server.KafkaApis#handleProduceRequest

def handleProduceRequest(request: RequestChannel.Request) {
  ...

  if (authorizedRequestInfo.isEmpty)
    ...
  else {
    val internalTopicsAllowed = request.header.clientId == AdminUtils.AdminClientId

    //【重点】添加数据
    replicaManager.appendRecords(
      timeout = produceRequest.timeout.toLong,
      requiredAcks = produceRequest.acks,
      internalTopicsAllowed = internalTopicsAllowed,
      isFromClient = true,
      entriesPerPartition = authorizedRequestInfo,
      responseCallback = sendResponseCallback,
      processingStatsCallback = processingStatsCallback)

    ...
  }
}

　　

　　【步骤3】kafka.server.ReplicaManager#appendRecords

def appendRecords(timeout: Long,
                  requiredAcks: Short,
                  internalTopicsAllowed: Boolean,
                  isFromClient: Boolean,
                  entriesPerPartition: Map[TopicPartition, MemoryRecords],
                  responseCallback: Map[TopicPartition, PartitionResponse] => Unit,
                  delayedProduceLock: Option[Lock] = None,
                  processingStatsCallback: Map[TopicPartition, RecordsProcessingStats] => Unit = _ => ()) {
  if (isValidRequiredAcks(requiredAcks)) {
    ...
    //【重点】添加到本地Log
    val localProduceResults = appendToLocalLog(internalTopicsAllowed = internalTopicsAllowed,
      isFromClient = isFromClient, entriesPerPartition, requiredAcks)
    ...
  } else {
    ...
  }
}

　　

　　【步骤4】kafka.server.ReplicaManager#appendToLocalLog

private def appendToLocalLog(internalTopicsAllowed: Boolean,
                             isFromClient: Boolean,
                             entriesPerPartition: Map[TopicPartition, MemoryRecords],
                             requiredAcks: Short): Map[TopicPartition, LogAppendResult] = {
  trace(s"Append [$entriesPerPartition] to local log")
  entriesPerPartition.map { case (topicPartition, records) =>
    brokerTopicStats.topicStats(topicPartition.topic).totalProduceRequestRate.mark()
    brokerTopicStats.allTopicsStats.totalProduceRequestRate.mark()

    // reject appending to internal topics if it is not allowed
    if (Topic.isInternal(topicPartition.topic) && !internalTopicsAllowed) {
      ...
    } else {
      try {
        //获取分区操作对象
        val partitionOpt = getPartition(topicPartition)
        val info = partitionOpt match {
          case Some(partition) =>
            if (partition eq ReplicaManager.OfflinePartition)
              throw new KafkaStorageException(s"Partition $topicPartition is in an offline log directory on broker $localBrokerId")
            //【重点】在分区中添加数据
            partition.appendRecordsToLeader(records, isFromClient, requiredAcks)

          case None => throw new UnknownTopicOrPartitionException("Partition %s doesn't exist on %d"
            .format(topicPartition, localBrokerId))
        }

        ...
      } catch {
        ...
    }
  }
}

　　

　　【步骤5】kafka.cluster.Partition#appendRecordsToLeader

def appendRecordsToLeader(records: MemoryRecords, isFromClient: Boolean, requiredAcks: Int = 0): LogAppendInfo = {
  val (info, leaderHWIncremented) = inReadLock(leaderIsrUpdateLock) {
    leaderReplicaIfLocal match {
      case Some(leaderReplica) =>
        ...

        //【重点】
        val info = log.appendAsLeader(records, leaderEpoch = this.leaderEpoch, isFromClient)
        ...
      case None =>
        ...
    }
  }
  
  ...
}

　　

　　【步骤6】kafka.log.Log#appendAsLeader

def appendAsLeader(records: MemoryRecords, leaderEpoch: Int, isFromClient: Boolean = true): LogAppendInfo = {
  //【重点】添加数据
  append(records, isFromClient, assignOffsets = true, leaderEpoch)
}

 

　　【步骤7】kafka.log.Log#append

private def append(records: MemoryRecords, isFromClient: Boolean, assignOffsets: Boolean, leaderEpoch: Int): LogAppendInfo = {
  maybeHandleIOException(s"Error while appending records to $topicPartition in dir ${dir.getParent}") {
    //【重点】检查和验证数据
    val appendInfo = analyzeAndValidateRecords(records, isFromClient = isFromClient)
    ...
  }
}

　　

　　【步骤8】kafka.log.Log#analyzeAndValidateRecords

private def analyzeAndValidateProducerState(records: MemoryRecords, isFromClient: Boolean):
(mutable.Map[Long, ProducerAppendInfo], List[CompletedTxn], Option[BatchMetadata]) = {
  ...
  
  //遍历所有批次
  for (batch <- records.batches.asScala if batch.hasProducerId) {
    //【重点】这里就是producerId的缓存，返回的结果是 ProducerStateEntry 类型
    val maybeLastEntry = producerStateManager.lastEntry(batch.producerId)
    ...
    if (isFromClient) {
      //【重点】重点就是findDuplicateBatch方法，是检查每个数据，是否发送重复，这个方法来自ProducerStateEntry类里面
      maybeLastEntry.flatMap(_.findDuplicateBatch(batch)).foreach { duplicate =>
        return (updatedProducers, completedTxns.toList, Some(duplicate))
      }
    }
    
    ...
  }
  ...
}

　　

　　【步骤9】kafka.log.ProducerStateEntry#findDuplicateBatch

def findDuplicateBatch(batch: RecordBatch): Option[BatchMetadata] = {
  //如果这个批次的producerEpoch（生产代）不等于当前生产代，就没有重复
  if (batch.producerEpoch != producerEpoch)
     None
  else
    //【重点】检查重复的sequence num
    batchWithSequenceRange(batch.baseSequence, batch.lastSequence)
}

　　

　　【步骤10】kafka.log.ProducerStateEntry#batchWithSequenceRange

def batchWithSequenceRange(firstSeq: Int, lastSeq: Int): Option[BatchMetadata] = {
  //【重点】过滤出重复的数据，只要offset范围在缓存范围有重合，代表有重复数据
  val duplicate = batchMetadata.filter { metadata =>
    firstSeq == metadata.firstSeq && lastSeq == metadata.lastSeq
  }
  duplicate.headOption
}

　　

　　在这里，我们就知道在【步骤8】，里面的这句，就是producerId缓存的地方，进去查看下

//【重点】这里就是producerId的缓存，返回的结果是 ProducerStateEntry 类型
 8     val maybeLastEntry = producerStateManager.lastEntry(batch.producerId)

　　

　　kafka.log.ProducerStateManager#lastEntry

　　可以看到 producerId 就缓存在producers里

//可以看到 producerId 就缓存在producers里
def lastEntry(producerId: Long): Option[ProducerStateEntry] = producers.get(producerId)

 

　　查看下producers是什么类型

private val producers = mutable.Map.empty[Long, ProducerStateEntry]

　　

　　在上下文搜索下，有没有 删除 prodcuerId的动作，找到下面三个地方

　　kafka.log.ProducerStateManager#truncateHead

def truncateHead(logStartOffset: Long) {
  val evictedProducerEntries = producers.filter { case (_, producerState) =>
    !isProducerRetained(producerState, logStartOffset)
  }

  val evictedProducerIds = evictedProducerEntries.keySet

  //【重点】删除producerId
  producers --= evictedProducerIds
  removeEvictedOngoingTransactions(evictedProducerIds)
  removeUnreplicatedTransactions(logStartOffset)

  if (lastMapOffset < logStartOffset)
    lastMapOffset = logStartOffset

  deleteSnapshotsBefore(logStartOffset)
  lastSnapOffset = latestSnapshotOffset.getOrElse(logStartOffset)
}

　　向上追踪

　　kafka.log.ProducerStateManager#truncateAndReload

def truncateAndReload(logStartOffset: Long, logEndOffset: Long, currentTimeMs: Long) {
  // remove all out of range snapshots
  deleteSnapshotFiles(logDir, { snapOffset =>
    snapOffset > logEndOffset || snapOffset <= logStartOffset
  })

  if (logEndOffset != mapEndOffset) {
    producers.clear()
    ongoingTxns.clear()

    // since we assume that the offset is less than or equal to the high watermark, it is
    // safe to clear the unreplicated transactions
    unreplicatedTxns.clear()
    loadFromSnapshot(logStartOffset, currentTimeMs)
  } else {
    //【重点】
    truncateHead(logStartOffset)
  }
}

 

　　kafka.log.Log#recoverSegment

　　　　这个方法都是在Log初始化的时候，才会调用，所以排除

 

　　kafka.log.Log#loadProducerState

　　　　这个方法，向上追踪，一个来源于Log初始化，所以排除

　　　　一个来源于 kafka.log.Log#truncateTo，是follower副本向leader副本同步数据的时候触发，这个比较可疑

 

　　kafka.log.Log#truncateTo

private[log] def truncateTo(targetOffset: Long): Boolean = {
  maybeHandleIOException(s"Error while truncating log to offset $targetOffset for $topicPartition in dir ${dir.getParent}") {
    if (targetOffset < 0)
      throw new IllegalArgumentException(s"Cannot truncate partition $topicPartition to a negative offset (%d).".format(targetOffset))
    if (targetOffset >= logEndOffset) {
      info(s"Truncating to $targetOffset has no effect as the largest offset in the log is ${logEndOffset - 1}")
      false
    } else {
      info(s"Truncating to offset $targetOffset")
      lock synchronized {
        checkIfMemoryMappedBufferClosed()
        if (segments.firstEntry.getValue.baseOffset > targetOffset) {
          truncateFullyAndStartAt(targetOffset)
        } else {
          val deletable = logSegments.filter(segment => segment.baseOffset > targetOffset)
          deletable.foreach(deleteSegment)
          
          //【重点】清理
          activeSegment.truncateTo(targetOffset)
          updateLogEndOffset(targetOffset)
          this.recoveryPoint = math.min(targetOffset, this.recoveryPoint)
          this.logStartOffset = math.min(targetOffset, this.logStartOffset)
          _leaderEpochCache.clearAndFlushLatest(targetOffset)
          loadProducerState(targetOffset, reloadFromCleanShutdown = false)
        }
        true
      }
    }
  }
}

 

　　kafka.cluster.Partition#truncateTo

def truncateTo(offset: Long, isFuture: Boolean) {
  // The read lock is needed to prevent the follower replica from being truncated while ReplicaAlterDirThread
  // is executing maybeDeleteAndSwapFutureReplica() to replace follower replica with the future replica.
  inReadLock(leaderIsrUpdateLock) {
    //【重点】
    logManager.truncateTo(Map(topicPartition -> offset), isFuture = isFuture)
  }
}

　　

　　kafka.server.ReplicaFetcherThread#maybeTruncate

override def maybeTruncate(fetchedEpochs: Map[TopicPartition, EpochEndOffset]): ResultWithPartitions[Map[TopicPartition, Long]] = {
  val fetchOffsets = scala.collection.mutable.HashMap.empty[TopicPartition, Long]
  val partitionsWithError = mutable.Set[TopicPartition]()

  fetchedEpochs.foreach { case (tp, epochOffset) =>
    try {
      val replica = replicaMgr.getReplicaOrException(tp)
      val partition = replicaMgr.getPartition(tp).get
      //如果 epochOffset 有异常
      if (epochOffset.hasError) {
        info(s"Retrying leaderEpoch request for partition ${replica.topicPartition} as the leader reported an error: ${epochOffset.error}")
        partitionsWithError += tp
      } else {
        val fetchOffset =
          if (epochOffset.endOffset == UNDEFINED_EPOCH_OFFSET) {
            warn(s"Based on follower's leader epoch, leader replied with an unknown offset in ${replica.topicPartition}. " +
              s"The initial fetch offset ${partitionStates.stateValue(tp).fetchOffset} will be used for truncation.")
            partitionStates.stateValue(tp).fetchOffset
          } else if (epochOffset.endOffset >= replica.logEndOffset.messageOffset)
            logEndOffset(replica, epochOffset)
          else
            epochOffset.endOffset
        //【重点】
        partition.truncateTo(fetchOffset, isFuture = false)
        replicaMgr.replicaAlterLogDirsManager.markPartitionsForTruncation(brokerConfig.brokerId, tp, fetchOffset)
        fetchOffsets.put(tp, fetchOffset)
      }
    } catch {
      case e: KafkaStorageException =>
        info(s"Failed to truncate $tp", e)
        partitionsWithError += tp
    }
  }

  ResultWithPartitions(fetchOffsets, partitionsWithError)
}

　　

　　kafka.server.AbstractFetcherThread#doWork

override def doWork() {
  //【重点】可能清理
  maybeTruncate()
  val fetchRequest = inLock(partitionMapLock) {
    val ResultWithPartitions(fetchRequest, partitionsWithError) = buildFetchRequest(states)
    if (fetchRequest.isEmpty) {
      trace(s"There are no active partitions. Back off for $fetchBackOffMs ms before sending a fetch request")
      partitionMapCond.await(fetchBackOffMs, TimeUnit.MILLISECONDS)
    }
    handlePartitionsWithErrors(partitionsWithError)
    fetchRequest
  }
  if (!fetchRequest.isEmpty)
    processFetchRequest(fetchRequest)
}

　　

　　kafka.utils.ShutdownableThread#run

override def run(): Unit = {
  info("Starting")
  try {
    while (isRunning)
      //【重点】一直执行，直到isRunning=false
      doWork()
  } catch {
    case e: FatalExitError =>
      shutdownInitiated.countDown()
      shutdownComplete.countDown()
      info("Stopped")
      Exit.exit(e.statusCode())
    case e: Throwable =>
      if (isRunning)
        error("Error due to", e)
  } finally {
     shutdownComplete.countDown()
  }
  info("Stopped")
}

　　

　　ShutdownableThread 的实现类有 kafka.server.AbstractFetcherThread

　　kafka.server.AbstractFetcherThread 的实现类有 kafka.server.ReplicaFetcherThread

　　kafka.server.ReplicaFetcherThread  这个类被创建，然后运行，就会自动清理了

　　kafka.server.ReplicaFetcherManager#createFetcherThread

override def createFetcherThread(fetcherId: Int, sourceBroker: BrokerEndPoint): AbstractFetcherThread = {
  val prefix = threadNamePrefix.map(tp => s"${tp}:").getOrElse("")
  val threadName = s"${prefix}ReplicaFetcherThread-$fetcherId-${sourceBroker.id}"
  //新建ReplicaFetcherThread
  new ReplicaFetcherThread(threadName, fetcherId, sourceBroker, brokerConfig, replicaManager, metrics, time, quotaManager)
}

　　

　　kafka.server.AbstractFetcherManager#addFetcherForPartitions

def addFetcherForPartitions(partitionAndOffsets: Map[TopicPartition, BrokerAndInitialOffset]) {
  lock synchronized {
    val partitionsPerFetcher = partitionAndOffsets.groupBy { case(topicPartition, brokerAndInitialFetchOffset) =>
      BrokerAndFetcherId(brokerAndInitialFetchOffset.broker, getFetcherId(topicPartition.topic, topicPartition.partition))}

    def addAndStartFetcherThread(brokerAndFetcherId: BrokerAndFetcherId, brokerIdAndFetcherId: BrokerIdAndFetcherId) {
      //【重点2】
      val fetcherThread = createFetcherThread(brokerAndFetcherId.fetcherId, brokerAndFetcherId.broker)
      fetcherThreadMap.put(brokerIdAndFetcherId, fetcherThread)
      //【重点3】
      fetcherThread.start
    }

    for ((brokerAndFetcherId, initialFetchOffsets) <- partitionsPerFetcher) {
      val brokerIdAndFetcherId = BrokerIdAndFetcherId(brokerAndFetcherId.broker.id, brokerAndFetcherId.fetcherId)
      fetcherThreadMap.get(brokerIdAndFetcherId) match {
        case Some(f) if f.sourceBroker.host == brokerAndFetcherId.broker.host && f.sourceBroker.port == brokerAndFetcherId.broker.port =>
          // reuse the fetcher thread
        case Some(f) =>
          f.shutdown()
          addAndStartFetcherThread(brokerAndFetcherId, brokerIdAndFetcherId)
        case None =>
          //【重点1】
          addAndStartFetcherThread(brokerAndFetcherId, brokerIdAndFetcherId)
      }

      fetcherThreadMap(brokerIdAndFetcherId).addPartitions(initialFetchOffsets.map { case (tp, brokerAndInitOffset) =>
        tp -> brokerAndInitOffset.initOffset
      })
    }
  }

  info("Added fetcher for partitions %s".format(partitionAndOffsets.map { case (topicPartition, brokerAndInitialOffset) =>
    "[" + topicPartition + ", initOffset " + brokerAndInitialOffset.initOffset + " to broker " + brokerAndInitialOffset.broker + "] "}))
}

　　

　　kafka.server.AbstractFetcherManager#resizeThreadPool

def resizeThreadPool(newSize: Int): Unit = {
  def migratePartitions(newSize: Int): Unit = {
    fetcherThreadMap.foreach { case (id, thread) =>
      val removedPartitions = thread.partitionsAndOffsets
      removeFetcherForPartitions(removedPartitions.keySet)
      if (id.fetcherId >= newSize)
        thread.shutdown()
      //【重点】
      addFetcherForPartitions(removedPartitions)
    }
  }
  lock synchronized {
    val currentSize = numFetchersPerBroker
    info(s"Resizing fetcher thread pool size from $currentSize to $newSize")
    numFetchersPerBroker = newSize
    if (newSize != currentSize) {
      // We could just migrate some partitions explicitly to new threads. But this is currently
      // reassigning all partitions using the new thread size so that hash-based allocation
      // works with partition add/delete as it did before.
      migratePartitions(newSize)
    }
    shutdownIdleFetcherThreads()
  }
}

　　

　　kafka.server.DynamicThreadPool#reconfigure

override def reconfigure(oldConfig: KafkaConfig, newConfig: KafkaConfig): Unit = {
  if (newConfig.numIoThreads != oldConfig.numIoThreads) {
    //【重点】
    server.requestHandlerPool.resizeThreadPool(newConfig.numIoThreads)
  }
  if (newConfig.numNetworkThreads != oldConfig.numNetworkThreads)
    server.socketServer.resizeThreadPool(oldConfig.numNetworkThreads, newConfig.numNetworkThreads)
  if (newConfig.numReplicaFetchers != oldConfig.numReplicaFetchers)
    //【重点】
    server.replicaManager.replicaFetcherManager.resizeThreadPool(newConfig.numReplicaFetchers)
  if (newConfig.numRecoveryThreadsPerDataDir != oldConfig.numRecoveryThreadsPerDataDir)
    server.getLogManager.resizeRecoveryThreadPool(newConfig.numRecoveryThreadsPerDataDir)
  if (newConfig.backgroundThreads != oldConfig.backgroundThreads)
    //【重点】
    server.kafkaScheduler.resizeThreadPool(newConfig.backgroundThreads)
}

　　

　　kafka.server.DynamicBrokerConfig#updateCurrentConfig

private def updateCurrentConfig(): Unit = {
  val newProps = mutable.Map[String, String]()
  newProps ++= staticBrokerConfigs
  overrideProps(newProps, dynamicDefaultConfigs)
  overrideProps(newProps, dynamicBrokerConfigs)
  val oldConfig = currentConfig
  val (newConfig, brokerReconfigurablesToUpdate) = processReconfiguration(newProps, validateOnly = false)
  if (newConfig ne currentConfig) {
    currentConfig = newConfig
    kafkaConfig.updateCurrentConfig(newConfig)

    // Process BrokerReconfigurable updates after current config is updated
    //【重点】
    brokerReconfigurablesToUpdate.foreach(_.reconfigure(oldConfig, newConfig))
  }
}

　　

　　kafka.server.DynamicBrokerConfig#updateBrokerConfig

private[server] def updateBrokerConfig(brokerId: Int, persistentProps: Properties): Unit = CoreUtils.inWriteLock(lock) {
  try {
    val props = fromPersistentProps(persistentProps, perBrokerConfig = true)
    dynamicBrokerConfigs.clear()
    dynamicBrokerConfigs ++= props.asScala
    //【重点】
    updateCurrentConfig()
  } catch {
    case e: Exception => error(s"Per-broker configs of $brokerId could not be applied: $persistentProps", e)
  }
}

　　

　　kafka.server.DynamicBrokerConfig#initialize

private[server] def initialize(zkClient: KafkaZkClient): Unit = {
  currentConfig = new KafkaConfig(kafkaConfig.props, false, None)
  val adminZkClient = new AdminZkClient(zkClient)
  updateDefaultConfig(adminZkClient.fetchEntityConfig(ConfigType.Broker, ConfigEntityName.Default))
  val props = adminZkClient.fetchEntityConfig(ConfigType.Broker, kafkaConfig.brokerId.toString)
  val brokerConfig = maybeReEncodePasswords(props, adminZkClient)
  //【重点】
  updateBrokerConfig(kafkaConfig.brokerId, brokerConfig)
}

　　

　　kafka.server.KafkaServer#startup

       在KafkaServer的启动方法中，初始化Broker的配置

def startup() {
  try {
    ...
    
    val canStartup = isStartingUp.compareAndSet(false, true)
    if (canStartup) {
      ...
      
      // initialize dynamic broker configs from ZooKeeper. Any updates made after this will be
      // applied after DynamicConfigManager starts.
      //【重点】
      config.dynamicConfig.initialize(zkClient)

      ...
    }
  }
  catch {
    ...
  }
}

　　自此，我们就追踪到了，清理producerId的源码了，在KafkaServer启动后，初始化Borker的配置的时候，就会启动一个while循环一直执行dowork()方法，一直做清理的操作，所以不用担心producerId占用内存导致OOM