librados部分代码走读
本文内容
本文主要介绍librados中关于C++部分的接口API调用。但并未深入到如何实现，有助于浅尝辄止的大概了解。
- RadosClient.h和RadosClient.cc是用于初始化RadosClient对象，一般用于客户端进行访问，主要操作：
- 连接、断链存储集群
int ping_monitor(std::string mon_id, std::string *result);
//根据poolctx初始化monclient、msgr、objecter等
int connect();
void shutdown();
- 存储池系列操作。RadosClient内部有poolctx对象
//*io = new librados::IoCtxImpl(this, objecter, poolid, CEPH_NOSNAP);
int create_ioctx(const char *name, IoCtxImpl **io);
int create_ioctx(int64_t, IoCtxImpl **io);
int get_fsid(std::string *s);
int64_t lookup_pool(const char *name);
bool pool_requires_alignment(int64_t pool_id);
int pool_requires_alignment2(int64_t pool_id, bool *requires);
uint64_t pool_required_alignment(int64_t pool_id);
int pool_required_alignment2(int64_t pool_id, uint64_t *alignment);
int pool_get_name(uint64_t pool_id, std::string *name, bool wait_latest_map = false);
//列举存储池。
int pool_list(std::list<std::pair<int64_t, std::string> >& ls);
int get_pool_stats(std::list<std::string>& ls, std::map<std::string,::pool_stat_t> *result,
bool *per_pool);
int get_fs_stats(ceph_statfs& result);
bool get_pool_is_selfmanaged_snaps_mode(const std::string& pool);
//创建、删除存储池
int pool_create(std::string& name, int16_t crush_rule=-1);
int pool_create_async(std::string& name, PoolAsyncCompletionImpl *c, int16_t crush_rule=-1);
int pool_get_base_tier(int64_t pool_id, int64_t* base_tier);
int pool_delete(const char *name);
int pool_delete_async(const char *name, PoolAsyncCompletionImpl *c);
- commond操作。mgr、mon等操作接口，处理mon、mgr、log等命令
int mon_command(const std::vector<std::string>& cmd, const bufferlist &inbl,
bufferlist *outbl, std::string *outs);
void mon_command_async(const std::vector<std::string>& cmd, const bufferlist &inbl,
bufferlist *outbl, std::string *outs, Context *on_finish);
int mon_command(int rank,
const std::vector<std::string>& cmd, const bufferlist &inbl,
bufferlist *outbl, std::string *outs);
int mon_command(std::string name,
const std::vector<std::string>& cmd, const bufferlist &inbl,
bufferlist *outbl, std::string *outs);
int mgr_command(const std::vector<std::string>& cmd, const bufferlist &inbl,
bufferlist *outbl, std::string *outs);
int mgr_command(
const std::string& name,
const std::vector<std::string>& cmd, const bufferlist &inbl,
bufferlist *outbl, std::string *outs);
int osd_command(int osd, std::vector<std::string>& cmd, const bufferlist& inbl,
bufferlist *poutbl, std::string *prs);
int pg_command(pg_t pgid, std::vector<std::string>& cmd, const bufferlist& inbl,
bufferlist *poutbl, std::string *prs);
void handle_log(MLog *m);
int monitor_log(const std::string& level, rados_log_callback_t cb,
rados_log_callback2_t cb2, void *arg);
- 我们再看一个python的demo，就能明白上述接口
import rados, sys
//初始化Client对象，连接集群
cluster = rados.Rados(conffile='ceph.conf')
cluster.connect()
cluster_stats = cluster.get_cluster_stats()
//存储池的增删改查
pools = cluster.list_pools()
for pool in pools:
print(pool)
print("\nCreate 'test' Pool")
print("------------------")
cluster.create_pool('test')
print("\nPool named 'test' exists: {}".format(str(cluster.pool_exists('test'))))
print("\nVerify 'test' Pool Exists")
print("-------------------------")
pools = cluster.list_pools()
for pool in pools:
print(pool)
print("\nDelete 'test' Pool")
print("------------------")
cluster.delete_pool('test')
print("\nPool named 'test' exists: {}".format(str(cluster.pool_exists('test'))))
//打开存储池的ioctx，并进行对象及其xattr的增删改查
ioctx = cluster.open_ioctx2(pool_id)
print("\nWriting object 'hw' with contents 'Hello World!' to pool 'data'.")
ioctx.write_full("hw", "Hello World!")
print("\n\nContents of object 'hw'\n------------------------\n")
print(ioctx.read("hw"))
print("\nRemoving object 'hw'")
ioctx.remove_object("hw")
print("\n\nWriting XATTR 'lang' with value 'en_US' to object 'hw'")
ioctx.set_xattr("hw", "lang", "en_US")
print("\n\nGetting XATTR 'lang' from object 'hw'\n")
print(ioctx.get_xattr("hw", "lang"))
object_iterator = ioctx.list_objects()
while True :
try :
rados_object = object_iterator.__next__()
print("Object contents = {}".format(rados_object.read()))
except StopIteration :
break
//关闭ioctx连接
print("\nClosing the connection.")
ioctx.close()
- 另外在官方的test中，也能看到类似的接口访问，示例为测试mon的commond命令。
//src/test/librados/cmd.cc
TEST(LibRadosCmd, MonDescribe) {
rados_t cluster;
ASSERT_EQ("", connect_cluster(&cluster));
char *buf, *st;
size_t buflen, stlen;
char *cmd[2];
cmd[1] = NULL;
cmd[0] = (char *)"{\"prefix\":\"get_command_descriptions\"}";
ASSERT_EQ(0, rados_mon_command(cluster, (const char **)cmd, 1, "", 0, &buf, &buflen, &st, &stlen));
ASSERT_LT(0u, buflen);
rados_buffer_free(buf);
rados_buffer_free(st);
cmd[0] = (char *)"get_command_descriptions";
ASSERT_EQ(-EINVAL, rados_mon_command(cluster, (const char **)cmd, 1, "", 0, &buf, &buflen, &st, &stlen));
rados_buffer_free(buf);
rados_buffer_free(st);
//ASSERT_LT(0u, stlen);
rados_buffer_free(buf);
rados_buffer_free(st);
rados_shutdown(cluster);
}
- IoctxImpl.cc。原名为PoolCtx。针对存储池的上下文操作，从上文例子中我们可以看到，RadosClient中有create_ioctx接口。在操作对象的时候，会大量使用。
- 先看示例，init中连接集群，然后调用ioctx的aio_write函数进行写操作
//test/librados/aio_cxx.cc
TEST(LibRadosAio, SimpleWritePP) {
char buf[128];
memset(buf, 0xcc, sizeof(buf));
bufferlist bl1;
bl1.append(buf, sizeof(buf));
AioTestDataPP test_data;
//初始化集群
ASSERT_EQ("", test_data.init());
auto my_completion = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
ASSERT_TRUE(my_completion);
//写对象
ASSERT_EQ(0, test_data.m_ioctx.aio_write("foo", my_completion.get(), bl1, sizeof(buf), 0));
}
- 常规接口
void queue_aio_write(struct AioCompletionImpl *c);
void complete_aio_write(struct AioCompletionImpl *c);
void flush_aio_writes_async(AioCompletionImpl *c);
void flush_aio_writes();
int64_t get_id() {
return poolid;
}
std::string get_cached_pool_name();
int get_object_hash_position(const std::string& oid, uint32_t *hash_position);
int get_object_pg_hash_position(const std::string& oid, uint32_t *pg_hash_position);
::ObjectOperation *prepare_assert_ops(::ObjectOperation *op);
// snaps
int snap_list(std::vector<uint64_t> *snaps);
int snap_lookup(const char *name, uint64_t *snapid);
int snap_get_name(uint64_t snapid, std::string *s);
int snap_get_stamp(uint64_t snapid, time_t *t);
int snap_create(const char* snapname);
int selfmanaged_snap_create(uint64_t *snapid);
void aio_selfmanaged_snap_create(uint64_t *snapid, AioCompletionImpl *c);
int snap_remove(const char* snapname);
int rollback(const object_t& oid, const char *snapName);
int selfmanaged_snap_remove(uint64_t snapid);
void aio_selfmanaged_snap_remove(uint64_t snapid, AioCompletionImpl *c);
int selfmanaged_snap_rollback_object(const object_t& oid,
::SnapContext& snapc, uint64_t snapid);
// 对象、对象xattr属性接口
int nlist(Objecter::NListContext *context, int max_entries);
uint32_t nlist_seek(Objecter::NListContext *context, uint32_t pos);
uint32_t nlist_seek(Objecter::NListContext *context, const rados_object_list_cursor& cursor);
rados_object_list_cursor nlist_get_cursor(Objecter::NListContext *context);
void object_list_slice(
const hobject_t start,
const hobject_t finish,
const size_t n,
const size_t m,
hobject_t *split_start,
hobject_t *split_finish);
int create(const object_t& oid, bool exclusive);
int write(const object_t& oid, bufferlist& bl, size_t len, uint64_t off);
int append(const object_t& oid, bufferlist& bl, size_t len);
int write_full(const object_t& oid, bufferlist& bl);
int writesame(const object_t& oid, bufferlist& bl,
size_t write_len, uint64_t offset);
int read(const object_t& oid, bufferlist& bl, size_t len, uint64_t off);
int mapext(const object_t& oid, uint64_t off, size_t len,
std::map<uint64_t,uint64_t>& m);
int sparse_read(const object_t& oid, std::map<uint64_t,uint64_t>& m,
bufferlist& bl, size_t len, uint64_t off);
int checksum(const object_t& oid, uint8_t type, const bufferlist &init_value,
size_t len, uint64_t off, size_t chunk_size, bufferlist *pbl);
int remove(const object_t& oid);
int remove(const object_t& oid, int flags);
int stat(const object_t& oid, uint64_t *psize, time_t *pmtime);
int stat2(const object_t& oid, uint64_t *psize, struct timespec *pts);
int trunc(const object_t& oid, uint64_t size);
int cmpext(const object_t& oid, uint64_t off, bufferlist& cmp_bl);
int tmap_update(const object_t& oid, bufferlist& cmdbl);
int exec(const object_t& oid, const char *cls, const char *method, bufferlist& inbl, bufferlist& outbl);
int getxattr(const object_t& oid, const char *name, bufferlist& bl);
int setxattr(const object_t& oid, const char *name, bufferlist& bl);
int getxattrs(const object_t& oid, std::map<std::string, bufferlist>& attrset);
int rmxattr(const object_t& oid, const char *name);
int operate(const object_t& oid, ::ObjectOperation *o, ceph::real_time *pmtime, int flags=0);
int operate_read(const object_t& oid, ::ObjectOperation *o, bufferlist *pbl, int flags=0);
int aio_operate(const object_t& oid, ::ObjectOperation *o,
AioCompletionImpl *c, const SnapContext& snap_context,
int flags, const blkin_trace_info *trace_info = nullptr);
int aio_operate_read(const object_t& oid, ::ObjectOperation *o,
AioCompletionImpl *c, int flags, bufferlist *pbl, const blkin_trace_info *trace_info = nullptr);
- librados.hpp和librados_cxx.cc是针对ceph api的c++接口的实现。主要类为：ListObject、ObjectOperation、ObjectWriteOperation、ObjectReadOperation、IoCtx、Rados等等
- 先看示例。读取配置，连接集群，初始化某个存储池的ioctx，即可调用io中的相关函数进行对象级别的操作。
C代码demo
err = rados_conf_read_file(cluster, "/path/to/myceph.conf");
err = rados_connect(cluster);
rados_ioctx_t io;
char *poolname = "mypool";
err = rados_ioctx_create(cluster, poolname, &io);
err = rados_write_full(io, "greeting", "hello", 5);
rados_ioctx_destroy(io);
rados_shutdown(cluster);
C++代码demo
//test/librados/aio_cxx.cc
TEST(LibRadosAio, RoundTripWriteFullPP2)
{
Rados cluster;
std::string pool_name = get_temp_pool_name();
ASSERT_EQ("", create_one_pool_pp(pool_name, cluster));
IoCtx ioctx;
cluster.ioctx_create(pool_name.c_str(), ioctx);
auto my_completion1 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
ObjectWriteOperation op;
char buf[128];
memset(buf, 0xcc, sizeof(buf));
bufferlist bl;
bl.append(buf);
op.write_full(bl);
op.set_op_flags2(LIBRADOS_OP_FLAG_FADVISE_DONTNEED);
ioctx.aio_operate("test_obj", my_completion1.get(), &op);
{
TestAlarm alarm;
ASSERT_EQ(0, my_completion1->wait_for_complete());
}
EXPECT_EQ(0, my_completion1->get_return_value());
auto my_completion2 = std::unique_ptr<AioCompletion>{Rados::aio_create_completion()};
bl.clear();
ObjectReadOperation op1;
op1.read(0, sizeof(buf), &bl, NULL);
op1.set_op_flags2(LIBRADOS_OP_FLAG_FADVISE_DONTNEED|LIBRADOS_OP_FLAG_FADVISE_RANDOM);
ioctx.aio_operate("test_obj", my_completion2.get(), &op1, 0);
{
TestAlarm alarm;
ASSERT_EQ(0, my_completion2->wait_for_complete());
}
EXPECT_EQ(0, my_completion2->get_return_value());
ASSERT_EQ(0, memcmp(buf, bl.c_str(), sizeof(buf)));
ioctx.remove("test_obj");
destroy_one_pool_pp(pool_name, cluster);
}
- ListObject主要实现了object的list功能
class CEPH_RADOS_API ListObject
{
public:
const std::string& get_nspace() const;
const std::string& get_oid() const;
const std::string& get_locator() const;
ListObject();
~ListObject();
ListObject( const ListObject&);
ListObject& operator=(const ListObject& rhs);
private:
ListObject(ListObjectImpl *impl);
friend class librados::NObjectIteratorImpl;
friend std::ostream& operator<<(std::ostream& out, const ListObject& lop);
ListObjectImpl *impl;
};
- ObjectOperation,父类，由WriteOperation和ReadOperation继承并负责某些函数的具体实现
- ObjectWriteOperation，负责写对象
- ObjectReadOperation，负责读对象
- IoCtx，负责对外接口，完成上述operation的op调用。
