redis-quicklist
有了ziplist, 为什么还需要quicklist? 这不是逻辑搞复杂了么， 但比单纯用ziplist, 性能提高显著.
因为quicklist是由多个ziplist组成的双链表，每个ziplist可看成1个结点.
quicklist数据结构：
/* quicklist is a 32 byte struct (on 64-bit systems) describing a quicklist. * 'count' is the number of total entries. * 'len' is the number of quicklist nodes. * 'compress' is: -1 if compression disabled, otherwise it's the number * of quicklistNodes to leave uncompressed at ends of quicklist. * 'fill' is the user-requested (or default) fill factor. */ typedef struct quicklist { quicklistNode *head; //头结点 quicklistNode *tail; //尾结点 unsigned long count; /* total count of all entries in all ziplists */ //元素总数 (所有ziplist结点的内部元素数总和) unsigned int len; /* number of quicklistNodes */ //结点数 (ziplist结点个数) int fill : 16; /* fill factor for individual nodes */ unsigned int compress : 16; /* depth of end nodes not to compress;0=off */ } quicklist;
结点数据结构:
/* quicklistNode is a 32 byte struct describing a ziplist for a quicklist. * We use bit fields keep the quicklistNode at 32 bytes. * count: 16 bits, max 65536 (max zl bytes is 65k, so max count actually < 32k). * encoding: 2 bits, RAW=1, LZF=2. * container: 2 bits, NONE=1, ZIPLIST=2. * recompress: 1 bit, bool, true if node is temporarry decompressed for usage. * attempted_compress: 1 bit, boolean, used for verifying during testing. * extra: 12 bits, free for future use; pads out the remainder of 32 bits */ typedef struct quicklistNode { struct quicklistNode *prev; struct quicklistNode *next; unsigned char *zl; //ziplist指针 unsigned int sz; /* ziplist size in bytes */ unsigned int count : 16; /* count of items in ziplist */ unsigned int encoding : 2; /* RAW==1 or LZF==2 */ unsigned int container : 2; /* NONE==1 or ZIPLIST==2 */ unsigned int recompress : 1; /* was this node previous compressed? */ unsigned int attempted_compress : 1; /* node can't compress; too small */ unsigned int extra : 10; /* more bits to steal for future usage */ } quicklistNode;
每个结点可压缩，有效减少quicklist存储大小，压缩还搞不懂，略!
查询：
时间复杂度: O(n) + O(m) //n=结点数, m=结点内部的元素数
/* Populate 'entry' with the element at the specified zero-based index * where 0 is the head, 1 is the element next to head * and so on. Negative integers are used in order to count * from the tail, -1 is the last element, -2 the penultimate * and so on. If the index is out of range 0 is returned. * * Returns 1 if element found * Returns 0 if element not found */ int quicklistIndex(const quicklist *quicklist, const long long idx, quicklistEntry *entry) { quicklistNode *n; unsigned long long accum = 0; unsigned long long index; int forward = idx < 0 ? 0 : 1; /* < 0 -> reverse, 0+ -> forward */ initEntry(entry); entry->quicklist = quicklist; if (!forward) { index = (-idx) - 1; n = quicklist->tail; //从尾结点倒序查找 } else { index = idx; n = quicklist->head; //从头结点正序查找 } if (index >= quicklist->count) return 0; while (likely(n)) { //循环每个结点，判断查询索引落在哪个结点里 if ((accum + n->count) > index) { break; } else { D("Skipping over (%p) %u at accum %lld", (void *)n, n->count, accum); accum += n->count; n = forward ? n->next : n->prev; } } if (!n) return 0; D("Found node: %p at accum %llu, idx %llu, sub+ %llu, sub- %llu", (void *)n, accum, index, index - accum, (-index) - 1 + accum); entry->node = n; if (forward) { /* forward = normal head-to-tail offset. */ entry->offset = index - accum; //结点内部偏移量 } else { /* reverse = need negative offset for tail-to-head, so undo * the result of the original if (index < 0) above. */ entry->offset = (-index) - 1 + accum; } quicklistDecompressNodeForUse(entry->node); entry->zi = ziplistIndex(entry->node->zl, entry->offset); //查询索引对应的ziplist内部某个指针 ziplistGet(entry->zi, &entry->value, &entry->sz, &entry->longval); //获取索引对应的数据 /* The caller will use our result, so we don't re-compress here. * The caller can recompress or delete the node as needed. */ return 1; }
使用场景
可用于list类型, 满足任意数量元素的队列, 随着量变大，性能基本平稳。
t_list.c
void lrangeCommand(client *c) { robj *o; long start, end, llen, rangelen; if ((getLongFromObjectOrReply(c, c->argv[2], &start, NULL) != C_OK) || (getLongFromObjectOrReply(c, c->argv[3], &end, NULL) != C_OK)) return; if ((o = lookupKeyReadOrReply(c,c->argv[1],shared.emptymultibulk)) == NULL || checkType(c,o,OBJ_LIST)) return; llen = listTypeLength(o); /* convert negative indexes */ if (start < 0) start = llen+start; if (end < 0) end = llen+end; if (start < 0) start = 0; /* Invariant: start >= 0, so this test will be true when end < 0. * The range is empty when start > end or start >= length. */ if (start > end || start >= llen) { addReply(c,shared.emptymultibulk); return; } if (end >= llen) end = llen-1; rangelen = (end-start)+1; /* Return the result in form of a multi-bulk reply */ addReplyMultiBulkLen(c,rangelen); if (o->encoding == OBJ_ENCODING_QUICKLIST) { listTypeIterator *iter = listTypeInitIterator(o, start, LIST_TAIL);//找到索引对应的遍历对象, 内部调用方法: quicklistIndex() while(rangelen--) { //查询的元素数量 listTypeEntry entry; listTypeNext(iter, &entry); //第一次查找第一个元素, 后面查找下一个元素 quicklistEntry *qe = &entry.entry; if (qe->value) { //元素对应的数据 addReplyBulkCBuffer(c,qe->value,qe->sz); } else { addReplyBulkLongLong(c,qe->longval); } } listTypeReleaseIterator(iter); } else { serverPanic("List encoding is not QUICKLIST!"); } }
/* Stores pointer to current the entry in the provided entry structure
* and advances the position of the iterator. Returns 1 when the current
* entry is in fact an entry, 0 otherwise. */
int listTypeNext(listTypeIterator *li, listTypeEntry *entry) {
/* Protect from converting when iterating */
serverAssert(li->subject->encoding == li->encoding);
entry->li = li;
if (li->encoding == OBJ_ENCODING_QUICKLIST) {
return quicklistNext(li->iter, &entry->entry);
} else {
serverPanic("Unknown list encoding");
}
return 0;
}
quicklist.c
/* Get next element in iterator. * * Note: You must NOT insert into the list while iterating over it. * You *may* delete from the list while iterating using the * quicklistDelEntry() function. * If you insert into the quicklist while iterating, you should * re-create the iterator after your addition. * * iter = quicklistGetIterator(quicklist,<direction>); * quicklistEntry entry; * while (quicklistNext(iter, &entry)) { * if (entry.value) * [[ use entry.value with entry.sz ]] * else * [[ use entry.longval ]] * } * * Populates 'entry' with values for this iteration. * Returns 0 when iteration is complete or if iteration not possible. * If return value is 0, the contents of 'entry' are not valid. */ int quicklistNext(quicklistIter *iter, quicklistEntry *entry) { initEntry(entry); if (!iter) { D("Returning because no iter!"); return 0; } entry->quicklist = iter->quicklist; entry->node = iter->current; if (!iter->current) { D("Returning because current node is NULL") return 0; } unsigned char *(*nextFn)(unsigned char *, unsigned char *) = NULL; int offset_update = 0; if (!iter->zi) { //第一次查询 /* If !zi, use current index. */ quicklistDecompressNodeForUse(iter->current); iter->zi = ziplistIndex(iter->current->zl, iter->offset); //第一个元素对应的指针 } else { /* else, use existing iterator offset and get prev/next as necessary. */ if (iter->direction == AL_START_HEAD) { nextFn = ziplistNext; offset_update = 1; } else if (iter->direction == AL_START_TAIL) { nextFn = ziplistPrev; offset_update = -1; } iter->zi = nextFn(iter->current->zl, iter->zi); //ziplist内部下一个元素 iter->offset += offset_update; } entry->zi = iter->zi; entry->offset = iter->offset; if (iter->zi) { //找到元素, 获取元素数据内容 /* Populate value from existing ziplist position */ ziplistGet(entry->zi, &entry->value, &entry->sz, &entry->longval); return 1; } else { //当前结点没有找到元素， 则跳到一个结点查询 /* We ran out of ziplist entries. * Pick next node, update offset, then re-run retrieval. */ quicklistCompress(iter->quicklist, iter->current); if (iter->direction == AL_START_HEAD) { /* Forward traversal */ D("Jumping to start of next node"); iter->current = iter->current->next; iter->offset = 0; } else if (iter->direction == AL_START_TAIL) { /* Reverse traversal */ D("Jumping to end of previous node"); iter->current = iter->current->prev; iter->offset = -1; } iter->zi = NULL; return quicklistNext(iter, entry); } }
............................
