MongoDB 中索引的构建阶段
2024-04-09 11:24 abce 阅读(79) 评论(0) 编辑 收藏 举报1.X lock
收到创建索引的请求时，会在集合上获取排他的 X 锁。该锁会停止该集合上的所有读/写操作
{"t":{"$date":"2024-03-13T05:29:35.925+00:00"},"s":"I", "c":"INDEX", "id":20438, "ctx":"conn15536","msg":"Index build: registering","attr":{"buildUUID":{"uuid":{"$uuid":"c3fa6f65-337e-424e-95f0-a8081b60b5cc"}},"namespace":"miku.demo","collectionUUID":{"uuid":{"$uuid":"cc364994-d259-464e-bf7e-a02d4746f910"}},"indexes":1,"firstIndex":{"name":"age_-1_name_1"},"command":{"createIndexes":"demo","v":2,"indexes":[{"name":"age_-1_name_1","key":{"age":-1,"name":1}}],"ignoreUnknownIndexOptions":false}}}2.初始化
初始化阶段，MongoDB 会构建三个数据结构：
·一个初始索引元数据条目
·旁路写表，一个临时表，用于保存在索引构建过程中写入创建的键。
·违反约束表：这是另一个临时表，用于保存可能导致键生成错误的所有文档。当文档中索引字段的键无效时，就会发生键生成错误。如可能是在创建唯一索引时，文档的字段值出现了重复。
{"t":{"$date":"2024-03-13T05:29:35.940+00:00"},"s":"I", "c":"INDEX", "id":20384, "ctx":"IndexBuildsCoordinatorMongod-4","msg":"Index build: starting","attr":{"buildUUID":{"uuid":{"$uuid":"c3fa6f65-337e-424e-95f0-a8081b60b5cc"}},"collectionUUID":{"uuid":{"$uuid":"cc364994-d259-464e-bf7e-a02d4746f910"}},"namespace":"miku.demo","properties":{"v":2,"key":{"age":-1,"name":1},"name":"age_-1_name_1"},"specIndex":0,"numSpecs":1,"method":"Hybrid","ident":"miku/index/18--3131258426784611814","collectionIdent":"miku/collection/0--7198494973130030145","maxTemporaryMemoryUsageMB":200}}
{"t":{"$date":"2024-03-13T05:29:35.940+00:00"},"s":"I", "c":"INDEX", "id":20346, "ctx":"IndexBuildsCoordinatorMongod-4","msg":"Index build: initialized","attr":{"buildUUID":{"uuid":{"$uuid":"c3fa6f65-337e-424e-95f0-a8081b60b5cc"}},"collectionUUID":{"uuid":{"$uuid":"cc364994-d259-464e-bf7e-a02d4746f910"}},"namespace":"miku.demo","initializationTimestamp":{"$timestamp":{"t":1710307775,"i":4}}}}
{"t":{"$date":"2024-03-13T05:29:35.940+00:00"},"s":"I", "c":"STORAGE", "id":4847600, "ctx":"IndexBuildsCoordinatorMongod-4","msg":"Index build: waiting for last optime before interceptors to be majority committed","attr":{"buildUUID":{"uuid":{"$uuid":"c3fa6f65-337e-424e-95f0-a8081b60b5cc"}},"collectionUUID":{"uuid":{"$uuid":"cc364994-d259-464e-bf7e-a02d4746f910"}},"deadline":{"$date":"2024-03-13T05:29:45.940Z"},"timeoutMillis":10000,"lastOpTime":{"ts":{"$timestamp":{"t":1710307775,"i":5}},"t":88}}}
{"t":{"$date":"2024-03-13T05:29:35.940+00:00"},"s":"I", "c":"INDEX", "id":20440, "ctx":"conn15536","msg":"Index build: waiting for index build to complete","attr":{"buildUUID":{"uuid":{"$uuid":"c3fa6f65-337e-424e-95f0-a8081b60b5cc"}},"deadline":{"$date":{"$numberLong":"9223372036854775807"}}}}3.IX lock
MongoDB 将排他 X 锁降级成意向排他 IX 锁。MongoDB 会定期获取 IX 锁，用于读/写操作。
rs1 [direct: primary] test> db.adminCommand( { lockInfo: 1 });
{
lockInfo: [
{
resourceId: '{2305843009213693953: Global, 1}',
granted: [
{
mode: 'IX',
convertMode: 'NONE',
enqueueAtFront: false,
compatibleFirst: false,
debugInfo: 'index build: dab3aa01-2f4b-456a-bcf8-70cf01b66264',
clientInfo: { desc: 'IndexBuildsCoordinatorMongod-1', opid: 2581346 }
}
],
pending: []
},
{
resourceId: '{2305843009213693954: Global, 2}',
granted: [
{
mode: 'IX',
convertMode: 'NONE',
enqueueAtFront: false,
compatibleFirst: false,
debugInfo: 'index build: dab3aa01-2f4b-456a-bcf8-70cf01b66264',
clientInfo: { desc: 'IndexBuildsCoordinatorMongod-1', opid: 2581346 }
}
],
pending: []
},
{
resourceId: '{2305843009213693955: Global, 3}',
granted: [
{
mode: 'IX',
convertMode: 'NONE',
enqueueAtFront: false,
compatibleFirst: false,
debugInfo: 'index build: dab3aa01-2f4b-456a-bcf8-70cf01b66264',
clientInfo: { desc: 'IndexBuildsCoordinatorMongod-1', opid: 2581346 }
}
],
pending: []
},
{
resourceId: '{6303114750411443102: Database, 1691428731984055198, miku}',
granted: [
{
mode: 'IX',
convertMode: 'NONE',
enqueueAtFront: false,
compatibleFirst: false,
debugInfo: 'index build: dab3aa01-2f4b-456a-bcf8-70cf01b66264',
clientInfo: { desc: 'IndexBuildsCoordinatorMongod-1', opid: 2581346 }
}
],
pending: []
},
{
resourceId: '{7704558816947887585: Collection, 787029789306805729, miku.demo}',
granted: [
{
mode: 'IX',
convertMode: 'NONE',
enqueueAtFront: false,
compatibleFirst: false,
debugInfo: 'index build: dab3aa01-2f4b-456a-bcf8-70cf01b66264',
clientInfo: { desc: 'IndexBuildsCoordinatorMongod-1', opid: 2581346 }
}
],
pending: []
}
],
ok: 1,
'$clusterTime': {
clusterTime: Timestamp({ t: 1710300387, i: 4 }),
signature: {
hash: Binary(Buffer.from("0000000000000000000000000000000000000000", "hex"), 0),
keyId: Long("0")
}
},
operationTime: Timestamp({ t: 1710300387, i: 4 })
}4.集合扫描
MongoDB 会扫描集合，并在内存或临时磁盘文件中对索引键进行排序。
在这一阶段，MongoDB 会交错进行读/写操作。MongoDB 会为集合中的每个文档生成一个键，并将该键转储到外部排序器中。不过，如果 MongoDB 在收集扫描过程中生成键时发现键生成错误，它会将该键保留在违反约束表中，以便之后处理。然后，MongoDB 会在完成集合扫描后将已排序的键转存到索引中。
{"t":{"$date":"2024-03-13T05:29:38.000+00:00"},"s":"I", "c":"-", "id":51773, "ctx":"IndexBuildsCoordinatorMongod-4","msg":"progress meter","attr":{"name":"Index Build: scanning collection","done":1167700,"total":2000000,"percent":58}}
{"t":{"$date":"2024-03-13T05:29:39.245+00:00"},"s":"I", "c":"INDEX", "id":20391, "ctx":"IndexBuildsCoordinatorMongod-4","msg":"Index build: collection scan done","attr":{"buildUUID":{"uuid":{"$uuid":"c3fa6f65-337e-424e-95f0-a8081b60b5cc"}},"collectionUUID":{"uuid":{"$uuid":"cc364994-d259-464e-bf7e-a02d4746f910"}},"namespace":"miku.demo","totalRecords":2000000,"readSource":"kMajorityCommitted","durationMillis":3000}}
{"t":{"$date":"2024-03-13T05:29:39.245+00:00"},"s":"D1", "c":"INDEX", "id":20392, "ctx":"IndexBuildsCoordinatorMongod-4","msg":"Index build: inserting from external sorter into index","attr":{"index":"age_-1_name_1","buildUUID":{"uuid":{"$uuid":"c3fa6f65-337e-424e-95f0-a8081b60b5cc"}}}}
{"t":{"$date":"2024-03-13T05:29:42.651+00:00"},"s":"I", "c":"INDEX", "id":20685, "ctx":"IndexBuildsCoordinatorMongod-4","msg":"Index build: inserted keys from external sorter into index","attr":{"namespace":"miku.demo","index":"age_-1_name_1","keysInserted":2000000,"durationMillis":3000}}因违反约束条件而产生的错误日志如下所示：
{"t":{"$date":"2024-03-13T07:34:35.894+00:00"},"s":"D1", "c":"INDEX", "id":20676, "ctx":"IndexBuildsCoordinatorMongod-5","msg":"Index build: recording duplicate key conflict on unique index","attr":{"index":"age_1"}}
{"t":{"$date":"2024-03-13T07:34:35.895+00:00"},"s":"D1", "c":"INDEX", "id":20676, "ctx":"IndexBuildsCoordinatorMongod-5","msg":"Index build: recording duplicate key conflict on unique index","attr":{"index":"age_1"}}5.处理旁路写表
现在，MongoDB 使用先进先出（FIFO）方式遍历旁路写表。在整个索引构建期间中，MongoDB 会为写入集合的每个文档生成一个键，并将其放入旁路写表，以便稍后处理。MongoDB 利用快照系统为键的处理设置限制。
6.投票并等待提交仲裁
如果 MongoDB 实例不是副本集的一部分，则跳过该阶段。从 MongoDB v4.4 开始，Mongod 会将 "投票"记录到主节点的内部复制集合中。
如果该节点是主节点，那么它将一直等到获得法定仲裁票数后才会继续索引创建过程。如果是辅助节点，则会一直等待，直到复制遇到"commitIndexBuild"或abortIndexBuild oplog 条目：
如果复制遇到"commitIndexBuild"条目，那么就完成了旁路写表的清空，并进入索引建立流程的下一阶段；否则，它会遇到"abortIndexBuild"条目，以中止索引构建并放弃索引构建任务。
{"t":{"$date":"2024-03-13T05:29:42.664+00:00"},"s":"I", "c":"STORAGE", "id":3856204, "ctx":"IndexBuildsCoordinatorMongod-4","msg":"Index build: received signal","attr":{"buildUUID":{"uuid":{"$uuid":"c3fa6f65-337e-424e-95f0-a8081b60b5cc"}},"action":"Commit quorum Satisfied"}}7.S lock
现在，在该集合上，MongoDB 会将 意向排他 IX 锁提升为共享 S 锁。它将阻止对该集合的所有写操作。
8.完成对临时的旁路写表的处理
在这一阶段，MongoDB 会延长旁路写表中剩余记录的消耗时间。如果进程在处理侧写表中的键时发现键生成错误，则会将这些键保留在违反约束表中，以便稍后处理。
{"t":{"$date":"2024-03-13T05:29:42.664+00:00"},"s":"D1", "c":"INDEX", "id":20689, "ctx":"IndexBuildsCoordinatorMongod-4","msg":"Index build: drained side writes","attr":{"index":"age_-1_name_1","collectionUUID":{"uuid":{"$uuid":"cc364994-d259-464e-bf7e-a02d4746f910"}},"namespace":"miku.demo","numApplied":0,"totalInserted":0,"totalDeleted":0,"durationMillis":0}}9.X lock
这个阶段会将共享的 S 锁升级成排他的 X 锁。
10.删除旁路写表
在此阶段，MongoDB 会执行旁路写表中任何未完成的操作，然后再将其丢弃。如果 Mongod 在旁路写表的键处理过程中发现键生成错误，它会将这些键保留在违反约束表中，以便稍后处理。如果在处理键时发生任何其他错误，索引构建将以错误告终。在此之后，索引（包括数据）已写入集合。
11.处理违反约束表
如果 MongoDB 节点是主节点，那么它将通过先进先出方法消费违反约束表。如果违反约束表中的任何键产生了键生成错误，或者表中的键已满，那么索引构建就会失败，并显示错误 "E11000 重复键错误收集"。主节点会在 oplog 中添加 "abortIndexBuild "条目，通知复制节点应终止并放弃索引构建任务。
{"t":{"$date":"2024-03-13T07:35:05.542+00:00"},"s":"I", "c":"STORAGE", "id":20649, "ctx":"IndexBuildsCoordinatorMongod-5","msg":"Index build: failed","attr":{"buildUUID":{"uuid":{"$uuid":"8675547c-160c-4225-9906-53f4cf8240b8"}},"collectionUUID":{"uuid":{"$uuid":"cc364994-d259-464e-bf7e-a02d4746f910"}},"namespace":"miku.demo","error":{"code":11000,"codeName":"DuplicateKey","errmsg":"E11000 duplicate key error collection: miku.demo index: age_1 dup key: { age: 18 }","keyPattern":{"age":1},"keyValue":{"age":18}}}}
{"t":{"$date":"2024-03-13T07:35:05.545+00:00"},"s":"I", "c":"STORAGE", "id":22206, "ctx":"IndexBuildsCoordinatorMongod-5","msg":"Deferring table drop for index","attr":{"index":"age_1","namespace":"miku.demo","uuid":{"uuid":{"$uuid":"cc364994-d259-464e-bf7e-a02d4746f910"}},"ident":"miku/index/21--3131258426784611814","commitTimestamp":{"$timestamp":{"t":1710315305,"i":2}}}}
{"t":{"$date":"2024-03-13T07:35:05.545+00:00"},"s":"I", "c":"STORAGE", "id":465611, "ctx":"IndexBuildsCoordinatorMongod-5","msg":"Cleaned up index build after abort. ","attr":{"buildUUID":{"uuid":{"$uuid":"8675547c-160c-4225-9906-53f4cf8240b8"}}}}
{"t":{"$date":"2024-03-13T07:35:05.571+00:00"},"s":"I", "c":"STORAGE", "id":51803, "ctx":"IndexBuildsCoordinatorMongod-5","msg":"Slow query","attr":{"type":"none","ns":"miku.demo","command":{"createIndexes":"demo","indexes":[{"v":2,"unique":true,"key":{"age":1},"name":"age_1"}],"lsid":{"id":{"$uuid":"59564965-889e-4aaf-9ef1-f77b50492df5"}},"$clusterTime":{"clusterTime":{"$timestamp":{"t":1710315182,"i":3}},"signature":{"hash":{"$binary":{"base64":"AAAAAAAAAAAAAAAAAAAAAAAAAAA=","subType":"0"}},"keyId":0}},"$db":"miku"},"numYields":4000,"queryHash":"740C02B0","queryFramework":"classic","locks":{"FeatureCompatibilityVersion":{"acquireCount":{"r":2,"w":4007}},"ReplicationStateTransition":{"acquireCount":{"w":4012}},"Global":{"acquireCount":{"r":2,"w":4007}},"Database":{"acquireCount":{"r":1,"w":4007}},"Collection":{"acquireCount":{"r":1,"w":4004,"R":1,"W":2}},"Mutex":{"acquireCount":{"r":5}}},"flowControl":{"acquireCount":4005,"timeAcquiringMicros":6136},"storage":{"data":{"bytesRead":182109559,"timeReadingMicros":47915}},"durationMillis":33261}}
{"t":{"$date":"2024-03-13T07:35:05.574+00:00"},"s":"I", "c":"INDEX", "id":20449, "ctx":"conn17944","msg":"Index build: failed","attr":{"buildUUID":{"uuid":{"$uuid":"8675547c-160c-4225-9906-53f4cf8240b8"}},"error":{"code":11000,"codeName":"DuplicateKey","errmsg":"E11000 duplicate key error collection: miku.demo index: age_1 dup key: { age: 18 }","keyPattern":{"age":1},"keyValue":{"age":18}}}}
{"t":{"$date":"2024-03-13T07:35:06.142+00:00"},"s":"I", "c":"STORAGE", "id":22260, "ctx":"TimestampMonitor","msg":"Removing drop-pending idents with drop timestamps before timestamp","attr":{"timestamp":{"$timestamp":{"t":1710315005,"i":6}}}}否则，MongoDB 会丢弃违反约束的表，并添加一个 "commitIndexBuild"条目。二级节点在复制 oplog 条目后完成相关的索引构建。
{"t":{"$date":"2024-03-13T05:29:42.664+00:00"},"s":"I", "c":"INDEX", "id":20345, "ctx":"IndexBuildsCoordinatorMongod-4","msg":"Index build: done building","attr":{"buildUUID":{"uuid":{"$uuid":"c3fa6f65-337e-424e-95f0-a8081b60b5cc"}},"collectionUUID":{"uuid":{"$uuid":"cc364994-d259-464e-bf7e-a02d4746f910"}},"namespace":"miku.demo","index":"age_-1_name_1","ident":"miku/index/18--3131258426784611814","collectionIdent":"miku/collection/0--7198494973130030145","commitTimestamp":{"$timestamp":{"t":1710307782,"i":4}}}}12.将索引设置成可用状态
MongoDB 通过更新索引元数据，将索引设置为可用状态。然后，它会从集合中释放独占 X 锁。
