200 行 C 代码实现插件式 NOSQL 存储服务器(一)
2012-11-24 22:30 Haippy 阅读(4268) 评论(2) 编辑 收藏 举报初一看,你肯定会觉得作者在忽悠人吧,200 行 C 代码居然可以实现插件式 NOSQL 存储服务器?是的,200 行 C 代码确实可以做到,但只能是一个极其简单的代码框架而已,不过感兴趣的同鞋可以在此基础上进行扩展。
引言
好了不多说,究竟什么才是插件式( Pluggable )的存储服务器呢,想必大家都用过 MySQL吧,它就是一款典型的插件式关系型数据服务器,MySQL 下面比较有名的存储引擎有 MyISAM, InnoDB, MEMORY 等。另外大名鼎鼎的 Memcached 的开发版(1.6版,即官方代码仓库中的 engine-pu 分支)也开始支持插件式存储引擎,这意味着以后你可以使用 Memcached 的其他存储引擎了,而不是仅仅将数据缓存在内存中。
首先插件式存储服务器的一种可能的设计架构图如下:
按照典型的分层设计模式,最上层是网络连接层(Connectin Layer),其次是中间处理层,包括 Cache 和 各种 Buffers、协议分析器、以及其他工具,再是公共存储抽象服务层,然后最下层是各种存储引擎,上图中列举了内存式,sqlite,leveldb等存储引擎,当然你也可以实现自己的存储引擎。
1、网络连接层(Connection Layer):负责处理网络连接,该层主要考虑如何高效地处理大量并发网络连接,读者可以阅读 Memcached 1.6 的源码来了解更多相关的技术。
2、中间处理层(Median Handle Layer):该层的功能包括 Cache 和各种 Buffers 的管理,协议解析,可能的查询优化,以及其他的工具。
3、公共存储抽象服务层(Common Storage Abstract Service Layer):该层主要和下层的存储引擎进行交互。
4、存储引擎层(Storage Layer):各种存储引擎的实现。
不过,为了保持本文简介和直观,我们在此只关注第3、4层,即公共存储抽象服务层(Common Storage Abstract Service Layer)和 存储引擎层(Storage Layer),
本文以下部分将用简短的 C 代码解释如何实现公共存储抽象服务层(Common Storage Abstract Service Layer)和存储引擎层(Storage Layer),同时为了简单起见,我们没有实现自己的存储引擎,而是借助 sqlite 和 leveldb 作为持久化的存储引擎,另外 in-memory 使用 uthash 作为存储后端。
实现细节
文件描述
├── csas.c ├── csas.h ├── example-engine.c ├── example-engine.h ├── inmemory-engine.c ├── inmemory-engine.h ├── leveldb-engine.c ├── leveldb-engine.h ├── main.c ├── Makefile ├── sqlite-engine.c └── sqlite-engine.h
本文涉及的代码一共 12 个文件(c文件、h文件、Makefile),其中各个文件功能如下:
- csas.h, csas.c:Common Storage Abstract Service(CSAS)实现文件。
- example-engine.c, example-engine.h:如何实现自己的存储引擎,即自定义存储引擎需要实现的接口。
- inmemory-engine.c, inmemory-engine.h:内存式存储引擎,基于 uthash。
- leveldb-engine.c, leveldb-engine.h:持久化存储引擎,基于 google 的 Leveldb。
- sqlite-engine.c, sqlite-engine.h:持久化存储引擎,基于 sqlite。
- main.c:测试文件。
- Makefile:Makefile 文件。
/* * ============================================================================= * * Filename: csas.h * * Description: common storage abstract service. * * Created: 11/24/2012 03:16:19 PM * * Author: Fu Haiping (forhappy), haipingf@gmail.com * Company: ICT ( Institute Of Computing Technology, CAS ) * * ============================================================================= */ #ifndef CSAS_H #define CSAS_H typedef struct engine_s_ engine_t; typedef struct csas_context_s_ csas_context_t; typedef struct engine_operation_s_ engine_operation_t; struct csas_context_s_ { csas_context_t *next; engine_t *engine; /* other stuff here... */ }; struct engine_s_ { char *engine_name; unsigned int version; engine_operation_t *engine_ops; /* more stuff about storage engine here... */ }; struct engine_operation_s_ { int (*put)(engine_t *engine, const char *key, unsigned int key_len, const char *value, unsigned int value_len); char * (*get)(engine_t *engine, const char *key, unsigned int key_len, unsigned int *value_len); int (*delete)(engine_t *engine, const char *key, unsigned int key_len); int (*unsupported_operation)(engine_t *engine); }; csas_context_t * csas_init(engine_t *engine); void csas_destory(csas_context_t *context); int csas_put(csas_context_t *context, const char *key, unsigned int key_len, const char *value, unsigned int value_len); char * csas_get(csas_context_t *context, const char *key, unsigned int key_len, unsigned int *value_len); int csas_delete(csas_context_t *context, const char *key, unsigned int key_len); #endif /* CSAS_H */
/* * ============================================================================= * * Filename: csas.c * * Description: common storage abstract service. * * Created: 11/24/2012 03:16:38 PM * * Author: Fu Haiping (forhappy), haipingf@gmail.com * Company: ICT ( Institute Of Computing Technology, CAS ) * * ============================================================================= */ #include <assert.h> #include <stdlib.h> #include "csas.h" csas_context_t * csas_init(engine_t *engine) { csas_context_t *context = (csas_context_t *)malloc(sizeof(csas_context_t)); context->engine = engine; context->next = NULL; return context; } void csas_destory(csas_context_t *context) { assert(context != NULL); if (context->engine != NULL) { free(context->engine); } free(context); } int csas_put(csas_context_t *context, const char *key, unsigned int key_len, const char *value, unsigned int value_len) { int ret = -1; ret = context->engine->engine_ops->put(context->engine, key, key_len, value, value_len); return ret; } char * csas_get(csas_context_t *context, const char *key, unsigned int key_len, unsigned int *value_len) { char *retstr = NULL; retstr = context->engine->engine_ops->get(context->engine, key, key_len, value_len); return retstr; } int csas_delete(csas_context_t *context, const char *key, unsigned int key_len) { int ret = -1; ret = context->engine->engine_ops->delete(context->engine, key, key_len); }
那么如何实现自己的存储引擎呢,你只要按照以下示例代码实现存储引擎相关的接口即可,目前只需要实现如下接口:
#ifndef EXAMPLE_ENGINE_H #define EXAMPLE_ENGINE_H #include "csas.h" extern engine_t * engine_example_init(); #endif /* EXAMPLE_ENGINE_H */
/* * ============================================================================= * * Filename: example-engine.c * * Description: example storage engine. * * Created: 11/24/2012 03:49:34 PM * * Author: Fu Haiping (forhappy), haipingf@gmail.com * Company: ICT ( Institute Of Computing Technology, CAS ) * * ============================================================================= */ #include <stdio.h> #include <stdlib.h> #include <string.h> #include "example-engine.h" static int put(engine_t *engine, const char *key, unsigned int key_len, const char *value, unsigned int value_len) { printf("hook call to example put.\n"); return 0; } static char * get(engine_t *engine, const char *key, unsigned int key_len, unsigned int *value_len) { printf("hook call to example engine get.\n"); return NULL; } static int delete(engine_t *engine, const char *key, unsigned int key_len) { printf("hook call to example engine delete.\n"); } engine_t * engine_example_init(void) { engine_t *engine = (engine_t *) malloc(sizeof(engine_t)); engine_operation_t *engine_ops = (engine_operation_t *) malloc(sizeof(engine_operation_t)); const char *engine_name = "example engine v0.1"; unsigned int engine_name_len = strlen(engine_name); unsigned int version = 0x1; engine->engine_name = malloc(sizeof(char) * (engine_name_len + 1)); memset(engine->engine_name, 0, (engine_name_len + 1)); strncpy(engine->engine_name, engine_name, engine_name_len); engine->version = version; engine_ops->put = put; engine_ops->get = get; engine_ops->delete = delete; engine->engine_ops = engine_ops; }
测试代码如下:
/* * ============================================================================= * * Filename: main.c * * Description: main routine. * * Created: 11/24/2012 05:58:22 PM * * Author: Fu Haiping (forhappy), haipingf@gmail.com * Company: ICT ( Institute Of Computing Technology, CAS ) * * ============================================================================= */ #include <stdio.h> #include "leveldb-engine.h" #include "sqlite-engine.h" int main(int args, const char *argv[]) { engine_t *engine_leveldb = engine_leveldb_init(); engine_t *engine_sqlite = engine_sqlite_init(); csas_context_t *context_leveldb = csas_init(engine_leveldb); csas_context_t *context_sqlite = csas_init(engine_sqlite); csas_put(context_leveldb, "hello", 5, "world", 5); csas_put(context_sqlite, "hello", 5, "world", 5); }
(待续) 以上是本文前一部分内容,接下来会慢慢完善的,这个小项目的代码整理后会放到 github 上,感兴趣的同学可以一起讨论讨论。:-)