zookeeper ZAB协议 Follower和leader源码分析
Follower处理逻辑
void followLeader() throws InterruptedException {
//...
try {
//获取leader server
QuorumServer leaderServer = findLeader();
try {
//主动向leader发起连接，TCP连接
connectToLeader(leaderServer.addr, leaderServer.hostname);
//发送follower的，包括last zxid sid，并从leader读取最新的zxid，再把last zxid发送给leader。返回leader zxid
//1. 首先follower发送自己的last zxid和sid，目的是为了leader确认epoch。FOLLOWERINFO
//2. leader返回确认后的epoch。LEADERINFO
//3. follower再次发送自己的最新zxid。ACKEPOCH
//4. 返回new epoch
long newEpochZxid = registerWithLeader(Leader.FOLLOWERINFO);
//check to see if the leader zxid is lower than ours
//this should never happen but is just a safety check
//(注释有点问题，是判断的epoch而不是zxid)
long newEpoch = ZxidUtils.getEpochFromZxid(newEpochZxid);
if (newEpoch < self.getAcceptedEpoch()) {
LOG.error("Proposed leader epoch " + ZxidUtils.zxidToString(newEpochZxid)
+ " is less than our accepted epoch " + ZxidUtils.zxidToString(self.getAcceptedEpoch()));
throw new IOException("Error: Epoch of leader is lower");
}
//和leader开始同步,首先收到一条消息，判断DIFF,TRUNC,SNAP
//
syncWithLeader(newEpochZxid);
QuorumPacket qp = new QuorumPacket();
while (this.isRunning()) {
readPacket(qp);
processPacket(qp);
}
}
}
Leader处理逻辑
void lead() throws IOException, InterruptedException {
try {
self.tick = 0;
//初始化，清理旧的session和创建状态机树
zk.loadData();
leaderStateSummary = new StateSummary(self.getCurrentEpoch(), zk.getLastProcessedZxid());
// 建立监听，同时处理和follower的发现，同步阶段逻辑
cnxAcceptor = new LearnerCnxAcceptor();
cnxAcceptor.start();
newLeaderProposal.packet = new QuorumPacket(NEWLEADER, zk.getZxid(),
null, null);
waitForEpochAck(self.getId(), leaderStateSummary);
self.setCurrentEpoch(epoch);
try {
//等待NEWLEADER_ACK，说明已经同步完成
waitForNewLeaderAck(self.getId(), zk.getZxid(), LearnerType.PARTICIPANT);
}
//开始服务，先发送UPTODATA，
startZkServer();
}
发现阶段逻辑处理
建立LearnerCnxAcceptor监听后，会启动LearnerHandler线程
public void run() {
try {
//读取FOLLOWERINFO，里面包含了follower的sid和peerLastZxid
QuorumPacket qp = new QuorumPacket();
ia.readRecord(qp, "packet");
byte learnerInfoData[] = qp.getData();
long lastAcceptedEpoch = ZxidUtils.getEpochFromZxid(qp.getZxid());
peerLastZxid = ss.getLastZxid();
/* the default to send to the follower */
//默认为全量同步
int packetToSend = Leader.SNAP;
long zxidToSend = 0;
long leaderLastZxid = 0;
/** the packets that the follower needs to get updates from **/
long updates = peerLastZxid;
ReentrantReadWriteLock lock = leader.zk.getZKDatabase().getLogLock();
ReadLock rl = lock.readLock();
try {
rl.lock();
//读取缓存队列中最小的zxid，所有需要同步的最小值
final long maxCommittedLog = leader.zk.getZKDatabase().getmaxCommittedLog();
//读取缓存队列中最大的zxid，所有需要同步的最大值
final long minCommittedLog = leader.zk.getZKDatabase().getminCommittedLog();
//获取当前leader的所有日志CommittedLog
//根据上面可知，只会同步到和leader最后一个已提交日志
//不需要同步，发送一个空的DIFF
if (peerLastZxid == leader.zk.getZKDatabase().getDataTreeLastProcessedZxid()) {
// Follower is already sync with us, send empty diff
LOG.info("leader and follower are in sync, zxid=0x{}",
Long.toHexString(peerLastZxid));
packetToSend = Leader.DIFF;
zxidToSend = peerLastZxid;
} else if (proposals.size() != 0) {
//minCommittedLog <= peerLastZxid <=maxCommittedLog，进行DIFF同步
if ((maxCommittedLog >= peerLastZxid)
&& (minCommittedLog <= peerLastZxid)) {
//这里有一种特殊情况，需要先TRUNK,再DIFF同步
//leader的日志是50001,50002,60001,60002
//follower的日志是50003
//把需要同步的数据加入发送队列
} else if (peerLastZxid > maxCommittedLog) {
//大于maxCommittedLog，直接TRUCK
packetToSend = Leader.TRUNC;
} else {
LOG.warn("Unhandled proposal scenario");
}
}
leaderLastZxid = leader.startForwarding(this, updates);
}
QuorumPacket newLeaderQP = new QuorumPacket(Leader.NEWLEADER,
ZxidUtils.makeZxid(newEpoch, 0), null, null);
//NEWLEADER报文加入发送队列，这时还没有发送任何报文
queuedPackets.add(newLeaderQP);
bufferedOutput.flush();
//Need to set the zxidToSend to the latest zxid
if (packetToSend == Leader.SNAP) {
zxidToSend = leader.zk.getZKDatabase().getDataTreeLastProcessedZxid();
}
//发送SNAP,DIFF,或者TRUNK
oa.writeRecord(new QuorumPacket(packetToSend, zxidToSend, null, null), "packet");
bufferedOutput.flush();
/*如果不是DIFF和TRUNK，直接发送全量信息 */
if (packetToSend == Leader.SNAP) {
leader.zk.getZKDatabase().serializeSnapshot(oa);
oa.writeString("BenWasHere", "signature");
}
bufferedOutput.flush();
// 开始发包
new Thread() {
public void run() {
Thread.currentThread().setName(
"Sender-" + sock.getRemoteSocketAddress());
try {
//发送同步报文
sendPackets();
} catch (InterruptedException e) {
LOG.warn("Unexpected interruption",e);
}
}
}.start();
//等待NEWLEADER_ACK，等到了NEWLEADER_ACK说明已经同步完成
leader.waitForNewLeaderAck(getSid(), qp.getZxid(), getLearnerType());
//等待大多数同步完成，leader starts up
synchronized(leader.zk){
while(!leader.zk.isRunning() && !this.isInterrupted()){
leader.zk.wait(20);
}
//发送UPTODATE报文，learn开始服务
queuedPackets.add(new QuorumPacket(Leader.UPTODATE, -1, null, null));
//正常处理流程
while (true) {
switch (qp.getType()) {
//处理propose，commit
case Leader.ACK:
if (this.learnerType == LearnerType.OBSERVER) {
if (LOG.isDebugEnabled()) {
LOG.debug("Received ACK from Observer " + this.sid);
}
}
syncLimitCheck.updateAck(qp.getZxid());
leader.processAck(this.sid, qp.getZxid(), sock.getLocalSocketAddress());
break;
//和follower保持session信息
case Leader.PING:
// Process the touches
ByteArrayInputStream bis = new ByteArrayInputStream(qp
.getData());
DataInputStream dis = new DataInputStream(bis);
while (dis.available() > 0) {
long sess = dis.readLong();
int to = dis.readInt();
leader.zk.touch(sess, to);
}
break;
case Leader.REVALIDATE:
//延长session时间
case Leader.REQUEST:
//加入处理队列
default:
}
}
}
}
总结
源码差不多看完了，整体挺复杂的，这里总结一下发现和同步的过程。
- newEpoch：提供服务的epoch
- acceptedEpoch：没有确认的epoch，LEADERINFO阶段
- currentEpoch：确认的epoch，接收到UPTODATE后
- lastLoggedZxid：最后处理的日志(包括提交，未提交)
