Messenger::send_message(Message *m, dest)
AsyncMessenger::send_message(Message *m, dest)
--|AsyncMessenger::_send_message(Message *m, dest)
----|conn=_lookup_conn(dest.addr) ===============================AsyncMessenger::conns <--accpet_conn() <--bind<--accpeting_conns[]<--add_accept() or create_connect
----|AsyncMessenger::submit_message(Message *m,conn,dest,...)
------|conn->send_message(m)
--------|out_q[priority].emplace_back(std::move(bl),m) #放入队列
--------|EventCenter::dispatch_event_external(write_handler) #回调操作放入event中心，wakeup线程执行
----------|external_event.push_back(write_handler)
----------|wakeup()
|
|
w->center.process_events
|
C_handle_write
|
|
--|write_handler = new C_handle_write(this)
C_handle_write::do_request(int fd)
--|conn->handle_write()
----|bufferlist data;m=_get_next_outgoing(&data); #out_q
----|AsyncConnection::write_message(m,data,more)
------|AsyncConnection::outcoming_bl <--bl
------|AsyncConnection::_try_send(bool more)
--------|AsyncConnection::connectedSocket cs->send(outcoming_bl,more)
----------|connectedSocket::_csi->send(outcoming_bl,more) #std::unique_ptr<ConnectedSocketImpl> _csi;
ConnectedSocketImpl:: virtual ssizet_t send(bl,more) <=== RDMAConnectedSocketImpl::send
----------|RDMAConnectedSocketImpl::send(outcoming_bl,more)
------------|RDMAConnectedSocketImpl::pending_bl <-bl
------------|RDMAConnectedSocketImpl::_sunmit_by_write(more)
--------------|RDMAConnectedSocketImpl::_sunmit_send_and_write(more,is_worker)
or
|RDMAConnectedSocketImpl::_sunmit_send_data(is_worker)
----------------|msg = get_send_msg_worker()/get_send_msg_polling()
----------------|RDMAConnectedSocketImpl::post_send_msg(msg)
------------------|pending_bl-->msg.data-->wr.id=&msg or pending_bl-->write_res->bl-->send_bl[i]
------------------|ibv_post_send(qp,&wr,&bad_wr)
|
|
qp
|
|
AsyncConnection::AsyncConnection()
AsyncMessenger::accpet_conn(Worker *w,ConnectedSocket cli_socket, addr,flag)
messager-->create-->new AsyncMessenger()
--lookup_or_create_singleton_object<StackSingleton>(single,...)
----new StackSingleton
--single.ready()
-----stack=NetWorkStack::create(cct,type,name,dev_info)
-------return std::make_shared<RDMAStack>(c,t,n,dev_info) #RDMAStack.cc:783
---------:NetWorkStack(cct,t,n)
------------w=create_worker(cct,type,name)
---------:ib(cct,dev_info)
------------get_rdma_device_info("public")
---------:dispatcher(cct,this) #RDMAStack 构造函数时，创建dispatcher并赋值给rdma_dispatcher
messager-->bind
--for i:processor
processor->bind()
----f = worker->listen(listen_addr,opts,listen_socket)
worker->center.submit_to(f,...)
RDMAWorker::listen
-----get_stack()->get_infiniband().init()
-----dispatcher->polling_start()
-----p= new RDMAConnectedSocketImpl(cct,&get_stack()->get_infiniband(),&get_infiniband()->get_dispatcher(),this) #connect和accpet都会new一个
--------qp=infiniband->create_queue_pair(cct,s->get_tx_cq(),s->get_rx_cq(),...)
RDMADispatcher::polling_start()
|--|tx_cc=get_stack()->get_infiniband().create_comp_channel(cct);
|--|rx_cc=...
|--|tx_cq=...
|--|--|cq=new Infiniband::CompletionQueue(cct,*this,CQ_DEPTH,cc);
|--|--|cq->init();
|--|--|--|cq=ibv_create_cq(...)
|--|rx_cq=...
|--|ceph_pthread_setname(&RDMADispatcher::polling,this);
AsyncConnection::process()
--AsyncConnection::process_connection()
AsyncConnection(CephContext *cct, AsyncMessenger *m, DispatchQueue *q,
Worker *w, bool m2, bool local)
AsyncMessenger::create_connect(entity_addr_t &addr,int type,bool is_separate_wk)
--w=AsyncMessenger::stack->ge_worker();
--conn = new AsyncConnection(cct,this,dispatch_queue,w,is_separate_wk,logArray)
----| dispatch_queue(q)，async_msgr(m)、
----|read_handler = new C_handle_read(this);
----|write_handler = new C_handle_write(this);
----|write_callback_handler = new C_handle_write_callback(this);
----|wakeup_handler = new C_time_wakeup(this);
----|tick_handler = new C_tick_wakeup(this);
--conn->connect(addr,type);
----set_peer_type(type);
----set_peer_addrs(addrs);
----_connect();
------center->dispatch_event_external(read_handler);---------------->EventCenter-->worker 调用read_handler->switch()状态机->_process_connection
--conn[addr] = conn;
--return conn
class C_handle_read : public EventCallback {
AsyncConnectionRef conn;
public:
explicit C_handle_read(AsyncConnectionRef c): conn(c) {}
void do_request(uint64_t fd_or_id) override {
conn->process();
}
};
_process_connection
--switch(state)状态机
----worker->connect(get_peer_addr(),opts,&cs)----------------------->RDMAWorker->connect(entity_addr_t &addr,SocketOptions &opts,ConnectedSocket *socket);
// 连接成功后，将socket fd加入epoll进行管理
----center->create_file_event(cs.fd, EVENT_READABLE, read_handler);
----state = STATE_CONNECTING_RE;
RDMAWorker::connect(&addr,SocketOptions &opts,ConnectedSocket *socket)
--get_stack()->get_infiniband().init();
--dispatcher->polling_start()
--p = RDMAConnectedSocketImpl(cct,&get_stack()->get_infiniband(),&get_stack()->get_stack()->get_dispatcher(),this)
--r=p->try_connect(entity_addr_t &addr,SocketOptions &opts);
--std::unique_ptr<RDMAConnectedSocketImpl> csi(p);
--*socket = ConnectedSocket(std::move(csi));
--get_stack()->get_infiniband().init();
--device = device_list->get_device(device_name.c_str());
--device->binding_port(cct, ib_port,gid_index);
--pd = new ProtectionDomain(cct, device);
----ProtectionDomain::pd=ibv_alloc_pd(device->ctxt)
---->infiniband::pd =ProtectionDomain::pd
--infiniband::sr_info.create_tx_queue(tx_queue_len);
--if(_support_srq){srq=create_shared_receive_queue(rx_queue_len,MAX_SHARED_RX_SGE_COUNT);}
--dispatcher->polling_start()
RDMADispatcher::polling_start()
|--|tx_cc=get_stack()->get_infiniband().create_comp_channel(cct);
|--|rx_cc=...
|--|tx_cq=...
|--|--|cq=new Infiniband::CompletionQueue(cct,*this,CQ_DEPTH,cc);
|--|--|cq->init();
|--|--|--|cq=ibv_create_cq(...)
|--|rx_cq=...
|--|ceph_pthread_setname(&RDMADispatcher::polling,this);
|--|t = std::thread(&RDMADispatcher::polling,this);
|--|--polling()
|--|----while(ture)
|--|------|r = poll(channel_poll,2,cct->_conf->ms_async_rdma_poll_timeout_ms)
|--|------|while(r >0 && tx_cq->get_cq_event())
|--|------|--{handle_tx_event()}
|--|------|while(r >0 && rx_cq->get_cq_event())
|--|------|--{handle_rx_event()}
|--|------|if(!r)
|--|------|--{handle_async_event()}
--r=p->try_connect(entity_addr_t &addr,SocketOptions &opts);
----tcp_fd = net.connect(peer_addr, opts.connect_bind_addr);
----worker->center.create_file_event(tcp_fd, EVENT_READABLE, con_handler);<---RDMAConnectedSocketImpl():con_handler(new C_handle_connection(this))
class C_handle_connection : public EventCallback {
RDMAConnectedSocketImpl *csi;
bool active;
public:
C_handle_connection(RDMAConnectedSocketImpl *w): csi(w), active(true) {}
void do_request(int fd) {
if (active)
csi->handle_connection();
}
void close() {
active = false;
}
};
};
RDMAConnectedSocketImpl::RDMAConnectedSocketImpl(CephContext *cct, Infiniband* ib, RDMADispatcher* s,
RDMAWorker *w):
cct(cct),
connected(0),
error(0),
infiniband(ib),
dispatcher(s),
worker(w),
lock("RDMAConnectedSocketImpl::lock"),
is_server(false),
con_handler(new C_handle_connection(this)),
active(false), detached(false)
--qp=infiniband->create_queue_pair(cct,s->get_tx_cq(),s->get_rx_cq(),...)
----Infiniband::QueuePair *qp = new QueuePair(
cct, *this, type, ib_physical_port, srq, tx, rx, max_send_wr, max_recv_wr);
----qp->init()
------qp = ibv_create_cq(pd,ibv_qp_init_attr qpia);
------ret = ibv_modify_qp(qp,&qpa,mask);
------if(!srq){infiniband.post_receives_to_rq(max_recv_wr,this)}
--infiniband->SetNewItem(my_msg.qpn,bitmap_index,stamp_item_index);
infiniband->send_msg_size : 用send发送的最大消息尺寸（数据+含头部），即小于这个尺寸的消息，可以用send发送。
