OSD数据读流程1

 

消息从(pipe.cc) pipe->reader() 处理后，若ms_can_fast_dispatch()就fast_dispatch()（DispatchQueue.cc )；

否则 in_q->enqueue()进入队列。

 

fast_dispatch()

---ms_fast_dispatch()【OSD.cc】

    将message转化为OpRequestRef op，后续直接对这个op进行处理

  1 //zym 处理client发来的各种消息
  2 void OSD::ms_fast_dispatch(Message *m)
  3 {
  4   FUNCTRACE(cct);
  5   //判断osd服务是否正在关闭，若是则减少一个message的引用，引用为0时空间会被释放。
  6   if (service.is_stopping()) {
  7     m->put();
  8     return;
  9   }
 10 
 11   // peering event?
 12   switch (m->get_type()) {
 13   case CEPH_MSG_PING:
 14     dout(10) << "ping from " << m->get_source() << dendl;
 15     m->put();
 16     return;
 17   case MSG_MON_COMMAND:
 18     handle_command(static_cast<MMonCommand*>(m));
 19     return;
 20   case MSG_OSD_FORCE_RECOVERY:
 21     handle_fast_force_recovery(static_cast<MOSDForceRecovery*>(m));
 22     return;
 23   case MSG_OSD_SCRUB2:
 24     handle_fast_scrub(static_cast<MOSDScrub2*>(m));
 25     return;
 26 
 27   case MSG_OSD_PG_CREATE2:
 28     return handle_fast_pg_create(static_cast<MOSDPGCreate2*>(m));
 29   case MSG_OSD_PG_QUERY:
 30     return handle_fast_pg_query(static_cast<MOSDPGQuery*>(m));
 31   case MSG_OSD_PG_NOTIFY:
 32     return handle_fast_pg_notify(static_cast<MOSDPGNotify*>(m));
 33   case MSG_OSD_PG_INFO:
 34     return handle_fast_pg_info(static_cast<MOSDPGInfo*>(m));
 35   case MSG_OSD_PG_REMOVE:
 36     return handle_fast_pg_remove(static_cast<MOSDPGRemove*>(m));
 37 
 38     // these are single-pg messages that handle themselves
 39   case MSG_OSD_PG_LOG:
 40   case MSG_OSD_PG_TRIM:
 41   case MSG_OSD_BACKFILL_RESERVE:
 42   case MSG_OSD_RECOVERY_RESERVE:
 43     {
 44       MOSDPeeringOp *pm = static_cast<MOSDPeeringOp*>(m);
 45       if (require_osd_peer(pm)) {
 46     enqueue_peering_evt(
 47       pm->get_spg(),
 48       PGPeeringEventRef(pm->get_event()));
 49       }
 50       pm->put();
 51       return;
 52     }
 53   }
 54   
 55   //将message结构转变成OpRequest结构，有智能指针op指向。
 56   //op的类型是 typedef boost::intrusive_ptr<OpRequest> Ref 
 57   OpRequestRef op = op_tracker.create_request<OpRequest, Message*>(m);//OpTracker op_tracker;
 58   {
 59 #ifdef WITH_LTTNG
 60     osd_reqid_t reqid = op->get_reqid();
 61 #endif
 62     tracepoint(osd, ms_fast_dispatch, reqid.name._type,
 63         reqid.name._num, reqid.tid, reqid.inc);
 64   }//tracepoint的作用？
 65 
 66   if (m->trace)
 67     op->osd_trace.init("osd op", &trace_endpoint, &m->trace);
 68 
 69   // note sender epoch, min req's epoch
 70   //获取epoch，什么是epoch？m的继承类是那些？
 71   op->sent_epoch = static_cast<MOSDFastDispatchOp*>(m)->get_map_epoch();
 72   op->min_epoch = static_cast<MOSDFastDispatchOp*>(m)->get_min_epoch();
 73   ceph_assert(op->min_epoch <= op->sent_epoch); // sanity check!
 74 
 75 //延时执行
 76   service.maybe_inject_dispatch_delay();
 77 
 78 //如果不是CEPH_MSG_OSD_OP消息 或者 has_features(CEPH_FEATUREMASK_RESEND_ON_SPLIT) 直接加入队列
 79   if (m->get_connection()->has_features(CEPH_FEATUREMASK_RESEND_ON_SPLIT) ||
 80       m->get_type() != CEPH_MSG_OSD_OP) {
 81     // queue it directly
 82     enqueue_op(
 83       static_cast<MOSDFastDispatchOp*>(m)->get_spg(),
 84       std::move(op),
 85       static_cast<MOSDFastDispatchOp*>(m)->get_map_epoch());
 86   } else {
 87     // legacy client, and this is an MOSDOp (the *only* fast dispatch
 88     // message that didn't have an explicit spg_t); we need to map
 89     // them to an spg_t while preserving delivery order.
 90     auto priv = m->get_connection()->get_priv();
 91     if (auto session = static_cast<Session*>(priv.get()); session) {
 92       std::lock_guard l{session->session_dispatch_lock};
 93       op->get();//加计数
 94       session->waiting_on_map.push_back(*op);
 95       OSDMapRef nextmap = service.get_nextmap_reserved();
 96       dispatch_session_waiting(session, nextmap);
 97       service.release_map(nextmap);//释放之前预留的osdmap epoch。
 98     }
 99   }
100   OID_EVENT_TRACE_WITH_MSG(m, "MS_FAST_DISPATCH_END", false); 
101 }

在enqueue_op（）中加入OSDService->op_wq队列.该队列时由OSD->SharedOpWQ队列初始化，实际上是保存到了OSD->SharedOpWQ,然后保存到ShardData中等待被处理，然后唤醒处理这个队列的线程，线程处理函数OSD::SharedOpWQ::_process().

 

 1 void OSD::enqueue_op(spg_t pg, OpRequestRef&& op, epoch_t epoch)
 2 {
 3   const utime_t stamp = op->get_req()->get_recv_stamp();
 4   const utime_t latency = ceph_clock_now() - stamp;
 5   const unsigned priority = op->get_req()->get_priority();
 6   const int cost = op->get_req()->get_cost();
 7   const uint64_t owner = op->get_req()->get_source().num();
 8 
 9   dout(15) << "enqueue_op " << op << " prio " << priority
10        << " cost " << cost
11        << " latency " << latency
12        << " epoch " << epoch
13        << " " << *(op->get_req()) << dendl;
14   op->osd_trace.event("enqueue op");
15   op->osd_trace.keyval("priority", priority);
16   op->osd_trace.keyval("cost", cost);
17   op->mark_queued_for_pg();
18   logger->tinc(l_osd_op_before_queue_op_lat, latency);
19   op_shardedwq.queue(
20     OpQueueItem(
21       unique_ptr<OpQueueItem::OpQueueable>(new PGOpItem(pg, std::move(op))),
22       cost, priority, stamp, owner, epoch));
23 }

 处理函数void OSD::ShardedOpWQ::_process(uint32_t thread_index, heartbeat_handle_d *hb)

 1 void OSD::ShardedOpWQ::_process(uint32_t thread_index, heartbeat_handle_d *hb)
 2 {
 3 ······
 4 OpQueueItem item = sdata->pqueue->dequeue();//出对列
 5 
 6 ·······
 7 
 8 ThreadPool::TPHandle tp_handle(osd->cct, hb, timeout_interval,
 9                  suicide_interval);
10 
11   // take next item
12   auto qi = std::move(slot->to_process.front()); //deque<OpQueueItem> to_process;  class OpQueueItem{}在OSD OpQueueItem.h中
13 ············
14 qi.run(osd, sdata, pg, tp_handle);//处理线程  PGOpItem::run
15 
16 ·········
17 
18 }
 1 void PGOpItem::run(
 2   OSD *osd,
 3   OSDShard *sdata,
 4   PGRef& pg,
 5   ThreadPool::TPHandle &handle)
 6 {
 7   osd->dequeue_op(pg, op, handle);
 8   pg->unlock();
 9 }
10 
11 void PGPeeringItem::run(
12   OSD *osd,
13   OSDShard *sdata,
14   PGRef& pg,
15   ThreadPool::TPHandle &handle)
16 {
17   osd->dequeue_peering_evt(sdata, pg.get(), evt, handle);
18 }
19 
20 void PGSnapTrim::run（）
21 
22 void PGScrub::run（）
23 
24 void PGRecovery::run（）
25 
26 void PGRecoveryContext::run（）
27 
28 void PGDelete::run（）

 osd->dequeue_op() 中调用pg->do_request()  //处理请求  PrimaryLogPG::do_request 【在PrimaryLogPG.cc中】

根据不用的消息类型对op进行处理

  1 void PrimaryLogPG::do_request(
  2   OpRequestRef& op,
  3   ThreadPool::TPHandle &handle)
  4 {
  5   if (op->osd_trace) {
  6     op->pg_trace.init("pg op", &trace_endpoint, &op->osd_trace);
  7     op->pg_trace.event("do request");
  8   }
  9   // make sure we have a new enough map
 10   auto p = waiting_for_map.find(op->get_source());
 11   if (p != waiting_for_map.end()) {
 12     // preserve ordering
 13     dout(20) << __func__ << " waiting_for_map "
 14          << p->first << " not empty, queueing" << dendl;
 15     p->second.push_back(op);
 16     op->mark_delayed("waiting_for_map not empty");
 17     return;
 18   }
 19   if (!have_same_or_newer_map(op->min_epoch)) {
 20     dout(20) << __func__ << " min " << op->min_epoch
 21          << ", queue on waiting_for_map " << op->get_source() << dendl;
 22     waiting_for_map[op->get_source()].push_back(op);
 23     op->mark_delayed("op must wait for map");
 24     osd->request_osdmap_update(op->min_epoch);//---更新map
 25     return;
 26   }
 27 
 28   if (can_discard_request(op)) {//条件成立 return
 29     return;
 30   }
 31 
 32   // pg-wide backoffs
 33   const Message *m = op->get_req();
 34   int msg_type = m->get_type();
 35   if (m->get_connection()->has_feature(CEPH_FEATURE_RADOS_BACKOFF)) {
 36     SessionRef session{static_cast<Session*>(m->get_connection()->get_priv().get())};
 37     if (!session)
 38       return;  // drop it.
 39 
 40     if (msg_type == CEPH_MSG_OSD_OP) {
 41       if (session->check_backoff(cct, info.pgid,
 42                  info.pgid.pgid.get_hobj_start(), m)) {
 43     return;
 44       }
 45 
 46       bool backoff =
 47     is_down() ||
 48     is_incomplete() ||
 49     (!is_active() && is_peered());
 50       if (g_conf()->osd_backoff_on_peering && !backoff) {
 51     if (is_peering()) {
 52       backoff = true;
 53     }
 54       }
 55       if (backoff) {
 56     add_pg_backoff(session);//稍后处理
 57     return;
 58       }
 59     }
 60     // pg backoff acks at pg-level
 61     if (msg_type == CEPH_MSG_OSD_BACKOFF) {
 62       const MOSDBackoff *ba = static_cast<const MOSDBackoff*>(m);
 63       if (ba->begin != ba->end) {
 64     handle_backoff(op);    //处理backoff的pg
 65     return;
 66       }
 67     }
 68   }
 69 
 70   if (!is_peered()) {
 71     // Delay unless PGBackend says it's ok
 72     if (pgbackend->can_handle_while_inactive(op)) {
 73       bool handled = pgbackend->handle_message(op);
 74       ceph_assert(handled);
 75       return;
 76     } else {
 77       waiting_for_peered.push_back(op);
 78       op->mark_delayed("waiting for peered");
 79       return;
 80     }
 81   }
 82 
 83   if (flushes_in_progress > 0) {
 84     dout(20) << flushes_in_progress
 85          << " flushes_in_progress pending "
 86          << "waiting for flush on " << op << dendl;
 87     waiting_for_flush.push_back(op);//pg处于flash状态，将op放入等待队列，等待pg变为可用状态
 88     op->mark_delayed("waiting for flush");
 89     return;
 90   }
 91 
 92   ceph_assert(is_peered() && flushes_in_progress == 0);
 93   if (pgbackend->handle_message(op))
 94     return;
 95 
 96   switch (msg_type) {
 97   case CEPH_MSG_OSD_OP:
 98   case CEPH_MSG_OSD_BACKOFF:
 99     if (!is_active()) {
100       dout(20) << " peered, not active, waiting for active on " << op << dendl;
101       waiting_for_active.push_back(op);
102       op->mark_delayed("waiting for active");
103       return;
104     }
105     switch (msg_type) {
106     case CEPH_MSG_OSD_OP:
107       // verify client features
108       if ((pool.info.has_tiers() || pool.info.is_tier()) &&
109       !op->has_feature(CEPH_FEATURE_OSD_CACHEPOOL)) {
110     osd->reply_op_error(op, -EOPNOTSUPP);
111     return;
112       }
113       do_op(op);  //处理op PrimaryLogPG::do_op(OpRequestRef& op) 【PrimaryLogPG.cc 中】
114       break;
115     case CEPH_MSG_OSD_BACKOFF:
116       // object-level backoff acks handled in osdop context
117       handle_backoff(op);
118       break;
119     }
120     break;
121 
122   case MSG_OSD_PG_SCAN:
123     do_scan(op, handle);
124     break;
125 
126   case MSG_OSD_PG_BACKFILL:
127     do_backfill(op);
128     break;
129 
130   case MSG_OSD_PG_BACKFILL_REMOVE:
131     do_backfill_remove(op);
132     break;
133 
134   case MSG_OSD_SCRUB_RESERVE:
135     {
136       const MOSDScrubReserve *m =
137     static_cast<const MOSDScrubReserve*>(op->get_req());
138       switch (m->type) {
139       case MOSDScrubReserve::REQUEST:
140     handle_scrub_reserve_request(op);
141     break;
142       case MOSDScrubReserve::GRANT:
143     handle_scrub_reserve_grant(op, m->from);
144     break;
145       case MOSDScrubReserve::REJECT:
146     handle_scrub_reserve_reject(op, m->from);
147     break;
148       case MOSDScrubReserve::RELEASE:
149     handle_scrub_reserve_release(op);
150     break;
151       }
152     }
153     break;
154 
155   case MSG_OSD_REP_SCRUB:
156     replica_scrub(op, handle);
157     break;
158 
159   case MSG_OSD_REP_SCRUBMAP:
160     do_replica_scrub_map(op);
161     break;
162 
163   case MSG_OSD_PG_UPDATE_LOG_MISSING:
164     do_update_log_missing(op);
165     break;
166 
167   case MSG_OSD_PG_UPDATE_LOG_MISSING_REPLY:
168     do_update_log_missing_reply(op);
169     break;
170 
171   default:
172     ceph_abort_msg("bad message type in do_request");
173   }
174 }

 

  1 /** do_op - do an op
  2  * pg lock will be held (if multithreaded)
  3  * osd_lock NOT held.
  4  */
  5 void PrimaryLogPG::do_op(OpRequestRef& op)
  6 {
  7   FUNCTRACE(cct);
  8   // NOTE: take a non-const pointer here; we must be careful not to
  9   // change anything that will break other reads on m (operator<<).
 10   MOSDOp *m = static_cast<MOSDOp*>(op->get_nonconst_req());
 11   ceph_assert(m->get_type() == CEPH_MSG_OSD_OP);
 12   if (m->finish_decode()) {  //解码什么？
 13     op->reset_desc();   // for TrackedOp
 14     m->clear_payload();
 15   }
 16 
 17   dout(20) << __func__ << ": op " << *m << dendl;
 18 
 19   hobject_t head = m->get_hobj();
 20   head.snap = CEPH_NOSNAP;
 21 
 22   if (!info.pgid.pgid.contains(
 23     info.pgid.pgid.get_split_bits(pool.info.get_pg_num()), head)) {//??PG 处于分裂状态
 24     derr << __func__ << " " << info.pgid.pgid << " does not contain "
 25      << head << " pg_num " << pool.info.get_pg_num() << " hash "
 26      << std::hex << head.get_hash() << std::dec << dendl;
 27     osd->clog->warn() << info.pgid.pgid << " does not contain " << head
 28               << " op " << *m;
 29     ceph_assert(!cct->_conf->osd_debug_misdirected_ops);
 30     return;
 31   }
 32 
 33   bool can_backoff =
 34     m->get_connection()->has_feature(CEPH_FEATURE_RADOS_BACKOFF);
 35   SessionRef session;
 36   if (can_backoff) {
 37     session = static_cast<Session*>(m->get_connection()->get_priv().get());
 38     if (!session.get()) {
 39       dout(10) << __func__ << " no session" << dendl;
 40       return;
 41     }
 42 
 43     if (session->check_backoff(cct, info.pgid, head, m)) {
 44       return;
 45     }
 46   }
 47 
 48   if (m->has_flag(CEPH_OSD_FLAG_PARALLELEXEC)) {
 49     // not implemented.
 50     dout(20) << __func__ << ": PARALLELEXEC not implemented " << *m << dendl;
 51     osd->reply_op_error(op, -EINVAL);  //并行执行状态，直接返回失败？
 52     return;
 53   }
 54 
 55   if (op->rmw_flags == 0) {
 56     int r = osd->osd->init_op_flags(op);
 57     if (r) {
 58       osd->reply_op_error(op, r);
 59       return;
 60     }
 61   }
 62 
 63   if ((m->get_flags() & (CEPH_OSD_FLAG_BALANCE_READS |
 64              CEPH_OSD_FLAG_LOCALIZE_READS)) &&
 65       op->may_read() &&
 66       !(op->may_write() || op->may_cache())) {//此时只有副本才能执行操作
 67     // balanced reads; any replica will do
 68     if (!(is_primary() || is_replica())) {
 69       osd->handle_misdirected_op(this, op);
 70       return;
 71     }
 72   } else {
 73     // normal case; must be primary
 74     if (!is_primary()) {
 75       osd->handle_misdirected_op(this, op);
 76       return;
 77     }
 78   }
 79 
 80   if (!op_has_sufficient_caps(op)) {  //没有足够cap，则直接返回失败
 81     osd->reply_op_error(op, -EPERM);
 82     return;
 83   }
 84 
 85   if (op->includes_pg_op()) {//op is OpRequest  对于请求中包含对PG的操作 CEPH_OSD_RMW_FLAG_PGOD
 86     return do_pg_op(op);//void PrimaryLogPG::do_pg_op(OpRequestRef op)     
 87   }
 88 
 89   // object name too long?
 90   if (m->get_oid().name.size() > cct->_conf->osd_max_object_name_len) {//对象长度，如果大于osd_max_object_name_len，则return
 91     dout(4) << "do_op name is longer than "
 92         << cct->_conf->osd_max_object_name_len
 93         << " bytes" << dendl;
 94     osd->reply_op_error(op, -ENAMETOOLONG);
 95     return;
 96   }
 97   if (m->get_hobj().get_key().size() > cct->_conf->osd_max_object_name_len) {//对象local 名称空间长度，如果大于osd_max_object_name_len,则return
 98     dout(4) << "do_op locator is longer than "
 99         << cct->_conf->osd_max_object_name_len
100         << " bytes" << dendl;
101     osd->reply_op_error(op, -ENAMETOOLONG);
102     return;
103   }
104   if (m->get_hobj().nspace.size() > cct->_conf->osd_max_object_namespace_len) {//对象local 名称空间长度，如果大于osd_max_object_name_len，则return
105     dout(4) << "do_op namespace is longer than "
106         << cct->_conf->osd_max_object_namespace_len
107         << " bytes" << dendl;
108     osd->reply_op_error(op, -ENAMETOOLONG);
109     return;
110   }
111 
112   if (int r = osd->store->validate_hobject_key(head)) {//  object的head是否有效
113     dout(4) << "do_op object " << head << " invalid for backing store: "
114         << r << dendl;
115     osd->reply_op_error(op, r);
116     return;
117   }
118 
119   // blacklisted?
120   if (get_osdmap()->is_blacklisted(m->get_source_addr())) {//检查op请求的地址是否在OSDMAP的blacklist中
121     dout(10) << "do_op " << m->get_source_addr() << " is blacklisted" << dendl;
122     osd->reply_op_error(op, -EBLACKLISTED);
123     return;
124   }
125 
126   // order this op as a write?
127   bool write_ordered = op->rwordered(); //是否是写请求
128 
129   // discard due to cluster full transition?  (we discard any op that
130   // originates before the cluster or pool is marked full; the client
131   // will resend after the full flag is removed or if they expect the
132   // op to succeed despite being full).  The except is FULL_FORCE and
133   // FULL_TRY ops, which there is no reason to discard because they
134   // bypass all full checks anyway.  If this op isn't write or
135   // read-ordered, we skip.
136   // FIXME: we exclude mds writes for now.
137   if (write_ordered && !(m->get_source().is_mds() ||
138              m->has_flag(CEPH_OSD_FLAG_FULL_TRY) ||
139              m->has_flag(CEPH_OSD_FLAG_FULL_FORCE)) &&
140       info.history.last_epoch_marked_full > m->get_map_epoch()) {
141     dout(10) << __func__ << " discarding op sent before full " << m << " "
142          << *m << dendl;
143     return;
144   }
145   // mds should have stopped writing before this point.
146   // We can't allow OSD to become non-startable even if mds
147   // could be writing as part of file removals.
148   if (write_ordered && osd->check_failsafe_full(get_dpp()) && 
149       !m->has_flag(CEPH_OSD_FLAG_FULL_TRY)) {
150     dout(10) << __func__ << " fail-safe full check failed, dropping request." << dendl;
151     return;
152   }
153   int64_t poolid = get_pgid().pool();
154   if (op->may_write()) {      //如果是写请求
155 
156     const pg_pool_t *pi = get_osdmap()->get_pg_pool(poolid);//获取对应的pool，pool获取失败，直接return？难道没有返回消息？
157     if (!pi) {
158       return;
159     }
160 
161     // invalid?
162     if (m->get_snapid() != CEPH_NOSNAP) {
163       dout(20) << __func__ << ": write to clone not valid " << *m << dendl;
164       osd->reply_op_error(op, -EINVAL);
165       return;
166     }
167 
168     // too big?
169     if (cct->_conf->osd_max_write_size &&
170         m->get_data_len() > cct->_conf->osd_max_write_size << 20) {//写请求的数据大于osd_max_write_size << 20 
171       // journal can't hold commit!
172       derr << "do_op msg data len " << m->get_data_len()
173            << " > osd_max_write_size " << (cct->_conf->osd_max_write_size << 20)
174            << " on " << *m << dendl;
175       osd->reply_op_error(op, -OSD_WRITETOOBIG);
176       return;
177     }
178   }
179 
180   dout(10) << "do_op " << *m
181        << (op->may_write() ? " may_write" : "")
182        << (op->may_read() ? " may_read" : "")
183        << (op->may_cache() ? " may_cache" : "")
184        << " -> " << (write_ordered ? "write-ordered" : "read-ordered")
185        << " flags " << ceph_osd_flag_string(m->get_flags())
186        << dendl;
187 
188   // missing object?
189   if (is_unreadable_object(head)) {//head 有效
190     if (!is_primary()) {//        如果不是主OSD则reply_op_error
191       osd->reply_op_error(op, -EAGAIN);
192       return;
193     }
194     if (can_backoff &&
195     (g_conf()->osd_backoff_on_degraded ||
196      (g_conf()->osd_backoff_on_unfound && missing_loc.is_unfound(head)))) {
197       add_backoff(session, head, head);
198       maybe_kick_recovery(head);//条件成立，add_backoff（），尝试启动recovery
199     } else {
200       wait_for_unreadable_object(head, op);
201     }
202     return;
203   }
204 
205   if (write_ordered) {
206     // degraded object?
207     if (is_degraded_or_backfilling_object(head)) {
208       if (can_backoff && g_conf()->osd_backoff_on_degraded) {
209         add_backoff(session, head, head);
210         maybe_kick_recovery(head);
211       } else {
212         wait_for_degraded_object(head, op);
213       }
214       return;
215     }
216 
217     if (scrubber.is_chunky_scrub_active() && write_blocked_by_scrub(head)) {
218       dout(20) << __func__ << ": waiting for scrub" << dendl;
219       waiting_for_scrub.push_back(op);
220       op->mark_delayed("waiting for scrub");
221       return;
222     }
223 
224     // blocked on snap?
225     //s    head在objects_blocked_on_degraded_snap 则将op放入放入waiting_for_degraded_object
226     if (auto blocked_iter = objects_blocked_on_degraded_snap.find(head);
227     blocked_iter != std::end(objects_blocked_on_degraded_snap)) {
228       hobject_t to_wait_on(head);
229       to_wait_on.snap = blocked_iter->second;
230       wait_for_degraded_object(to_wait_on, op);
231       return;
232     }
233     
234     //检查head是否在objects_blocked_on_snap_promotion，如果是则将op放入waiting_for_blocked_object
235     if (auto blocked_snap_promote_iter = objects_blocked_on_snap_promotion.find(head);
236     blocked_snap_promote_iter != std::end(objects_blocked_on_snap_promotion)) {
237       wait_for_blocked_object(blocked_snap_promote_iter->second->obs.oi.soid, op);
238       return;
239     }
240     
241     //检查head是否在objects_blocked_on_cache_full中，如果是则将op放入waiting_for_cache_not_full
242     if (objects_blocked_on_cache_full.count(head)) {
243       block_write_on_full_cache(head, op);
244       return;
245     }
246   }
247 
248   // dup/resent?
249   if (op->may_write() || op->may_cache()) {
250     // warning: we will get back *a* request for this reqid, but not
251     // necessarily the most recent.  this happens with flush and
252     // promote ops, but we can't possible have both in our log where
253     // the original request is still not stable on disk, so for our
254     // purposes here it doesn't matter which one we get.
255     eversion_t version;
256     version_t user_version;
257     int return_code = 0;
258     bool got = check_in_progress_op(
259       m->get_reqid(), &version, &user_version, &return_code);
260     if (got) {
261       dout(3) << __func__ << " dup " << m->get_reqid()
262           << " version " << version << dendl;
263       if (already_complete(version)) {
264     osd->reply_op_error(op, return_code, version, user_version);
265       } else {
266     dout(10) << " waiting for " << version << " to commit" << dendl;
267         // always queue ondisk waiters, so that we can requeue if needed
268     waiting_for_ondisk[version].emplace_back(op, user_version, return_code);
269     op->mark_delayed("waiting for ondisk");
270       }
271       return;
272     }
273   }
274 
275   ObjectContextRef obc;
276   bool can_create = op->may_write();
277   hobject_t missing_oid;
278 
279   // kludge around the fact that LIST_SNAPS sets CEPH_SNAPDIR for LIST_SNAPS
280   hobject_t _oid_head;
281   if (m->get_snapid() == CEPH_SNAPDIR) {
282     _oid_head = m->get_hobj().get_head();
283   }
284   const hobject_t& oid =
285     m->get_snapid() == CEPH_SNAPDIR ? _oid_head : m->get_hobj();
286 
287   // make sure LIST_SNAPS is on CEPH_SNAPDIR and nothing else
288   for (vector<OSDOp>::iterator p = m->ops.begin(); p != m->ops.end(); ++p) {
289     OSDOp& osd_op = *p;
290 
291     if (osd_op.op.op == CEPH_OSD_OP_LIST_SNAPS) {
292       if (m->get_snapid() != CEPH_SNAPDIR) {
293     dout(10) << "LIST_SNAPS with incorrect context" << dendl;
294     osd->reply_op_error(op, -EINVAL);
295     return;
296       }
297     } else {
298       if (m->get_snapid() == CEPH_SNAPDIR) {
299     dout(10) << "non-LIST_SNAPS on snapdir" << dendl;
300     osd->reply_op_error(op, -EINVAL);
301     return;
302       }
303     }
304   }
305 
306   // io blocked on obc?
307   if (!m->has_flag(CEPH_OSD_FLAG_FLUSH) &&
308       maybe_await_blocked_head(oid, op)) {
309     return;
310   }
311 
312   int r = find_object_context(
313     oid, &obc, can_create,
314     m->has_flag(CEPH_OSD_FLAG_MAP_SNAP_CLONE),
315     &missing_oid);
316 
317   // LIST_SNAPS needs the ssc too
318   if (obc &&
319       m->get_snapid() == CEPH_SNAPDIR &&
320       !obc->ssc) {
321     obc->ssc = get_snapset_context(oid, true);
322   }
323 
324   if (r == -EAGAIN) {
325     // If we're not the primary of this OSD, we just return -EAGAIN. Otherwise,
326     // we have to wait for the object.
327     if (is_primary()) {
328       // missing the specific snap we need; requeue and wait.
329       ceph_assert(!op->may_write()); // only happens on a read/cache
330       wait_for_unreadable_object(missing_oid, op);
331       return;
332     }
333   } else if (r == 0) {
334     if (is_unreadable_object(obc->obs.oi.soid)) {
335       dout(10) << __func__ << ": clone " << obc->obs.oi.soid
336            << " is unreadable, waiting" << dendl;
337       wait_for_unreadable_object(obc->obs.oi.soid, op);
338       return;
339     }
340 
341     // degraded object?  (the check above was for head; this could be a clone)
342     if (write_ordered &&
343     obc->obs.oi.soid.snap != CEPH_NOSNAP &&
344     is_degraded_or_backfilling_object(obc->obs.oi.soid)) {
345       dout(10) << __func__ << ": clone " << obc->obs.oi.soid
346            << " is degraded, waiting" << dendl;
347       wait_for_degraded_object(obc->obs.oi.soid, op);
348       return;
349     }
350   }
351 
352   bool in_hit_set = false;
353   if (hit_set) {
354     if (obc.get()) {
355       if (obc->obs.oi.soid != hobject_t() && hit_set->contains(obc->obs.oi.soid))
356     in_hit_set = true;
357     } else {
358       if (missing_oid != hobject_t() && hit_set->contains(missing_oid))
359         in_hit_set = true;
360     }
361     if (!op->hitset_inserted) {
362       hit_set->insert(oid);
363       op->hitset_inserted = true;
364       if (hit_set->is_full() ||
365           hit_set_start_stamp + pool.info.hit_set_period <= m->get_recv_stamp()) {
366         hit_set_persist();
367       }
368     }
369   }
370 
371   if (agent_state) {
372     if (agent_choose_mode(false, op))
373       return;
374   }
375 
376   if (obc.get() && obc->obs.exists && obc->obs.oi.has_manifest()) {
377     if (maybe_handle_manifest(op,
378                    write_ordered,
379                    obc))
380     return;
381   }
382 
383   if (maybe_handle_cache(op,
384              write_ordered,
385              obc,
386              r,
387              missing_oid,
388              false,
389              in_hit_set))
390     return;
391 
392   if (r && (r != -ENOENT || !obc)) {
393     // copy the reqids for copy get on ENOENT
394     if (r == -ENOENT &&
395     (m->ops[0].op.op == CEPH_OSD_OP_COPY_GET)) {
396       fill_in_copy_get_noent(op, oid, m->ops[0]);
397       return;
398     }
399     dout(20) << __func__ << ": find_object_context got error " << r << dendl;
400     if (op->may_write() &&
401     get_osdmap()->require_osd_release >= CEPH_RELEASE_KRAKEN) {
402       record_write_error(op, oid, nullptr, r);
403     } else {
404       osd->reply_op_error(op, r);
405     }
406     return;
407   }
408 
409   // make sure locator is consistent
410   object_locator_t oloc(obc->obs.oi.soid);
411   if (m->get_object_locator() != oloc) {
412     dout(10) << " provided locator " << m->get_object_locator() 
413          << " != object's " << obc->obs.oi.soid << dendl;
414     osd->clog->warn() << "bad locator " << m->get_object_locator() 
415              << " on object " << oloc
416               << " op " << *m;
417   }
418 
419   // io blocked on obc?
420   if (obc->is_blocked() &&
421       !m->has_flag(CEPH_OSD_FLAG_FLUSH)) {
422     wait_for_blocked_object(obc->obs.oi.soid, op);
423     return;
424   }
425 
426   dout(25) << __func__ << " oi " << obc->obs.oi << dendl;
427 
428   OpContext *ctx = new OpContext(op, m->get_reqid(), &m->ops, obc, this);//这里要创建一个OpContext结构，该结构会接管message中的所有ops的操作，ops的操作就是客户端将rbd请求拆分成object的请求。
429 
430   if (m->has_flag(CEPH_OSD_FLAG_SKIPRWLOCKS)) {
431     dout(20) << __func__ << ": skipping rw locks" << dendl;
432   } else if (m->get_flags() & CEPH_OSD_FLAG_FLUSH) {
433     dout(20) << __func__ << ": part of flush, will ignore write lock" << dendl;
434 
435     // verify there is in fact a flush in progress
436     // FIXME: we could make this a stronger test.
437     map<hobject_t,FlushOpRef>::iterator p = flush_ops.find(obc->obs.oi.soid);
438     if (p == flush_ops.end()) {
439       dout(10) << __func__ << " no flush in progress, aborting" << dendl;
440       reply_ctx(ctx, -EINVAL);
441       return;
442     }
443   } else if (!get_rw_locks(write_ordered, ctx)) {
444     dout(20) << __func__ << " waiting for rw locks " << dendl;
445     op->mark_delayed("waiting for rw locks");
446     close_op_ctx(ctx);
447     return;
448   }
449   dout(20) << __func__ << " obc " << *obc << dendl;
450 
451   if (r) {
452     dout(20) << __func__ << " returned an error: " << r << dendl;
453     close_op_ctx(ctx);
454     if (op->may_write() &&
455     get_osdmap()->require_osd_release >= CEPH_RELEASE_KRAKEN) {
456       record_write_error(op, oid, nullptr, r);
457     } else {
458       osd->reply_op_error(op, r);
459     }
460     return;
461   }
462 
463   if (m->has_flag(CEPH_OSD_FLAG_IGNORE_CACHE)) {
464     ctx->ignore_cache = true;
465   }
466 
467   if ((op->may_read()) && (obc->obs.oi.is_lost())) {
468     // This object is lost. Reading from it returns an error.
469     dout(20) << __func__ << ": object " << obc->obs.oi.soid
470          << " is lost" << dendl;
471     reply_ctx(ctx, -ENFILE);
472     return;
473   }
474   if (!op->may_write() &&
475       !op->may_cache() &&
476       (!obc->obs.exists ||
477        ((m->get_snapid() != CEPH_SNAPDIR) &&
478     obc->obs.oi.is_whiteout()))) {
479     // copy the reqids for copy get on ENOENT
480     if (m->ops[0].op.op == CEPH_OSD_OP_COPY_GET) {
481       fill_in_copy_get_noent(op, oid, m->ops[0]);
482       close_op_ctx(ctx);
483       return;
484     }
485     reply_ctx(ctx, -ENOENT);
486     return;
487   }
488 
489   op->mark_started();
490 
491   execute_ctx(ctx);
492   utime_t prepare_latency = ceph_clock_now();
493   prepare_latency -= op->get_dequeued_time();
494   osd->logger->tinc(l_osd_op_prepare_lat, prepare_latency);
495   if (op->may_read() && op->may_write()) {
496     osd->logger->tinc(l_osd_op_rw_prepare_lat, prepare_latency);
497   } else if (op->may_read()) {
498     osd->logger->tinc(l_osd_op_r_prepare_lat, prepare_latency);
499   } else if (op->may_write() || op->may_cache()) {
500     osd->logger->tinc(l_osd_op_w_prepare_lat, prepare_latency);
501   }
502 
503   // force recovery of the oldest missing object if too many logs
504   maybe_force_recovery();
505 }

 

posted @ 2019-01-17 15:52  yimuxi  阅读(2381)  评论(0)  编辑  收藏  举报