snap相关（个人总结）

看了ceph的关于snap的一些代码（snap.cc、snaprealm.cc、snapserver.cc、snapmapper.cc），作了标注，在此记录。

snap.cc主要是通过encode（序列化），将数据结构表示为二进制流的方式，decode（反序列化），以便通过网络传输或保存在磁盘等存储介质上。

以下是snaprealm.h的一些结构体的声明和注释

struct SnapRealm {
  // realm state

  sr_t srnode;

  // in-memory state
  MDCache *mdcache;
  CInode *inode;

  bool open;                        // set to true once all past_parents are opened
  SnapRealm *parent;
  set<SnapRealm*> open_children;    // active children that are currently open
  map<inodeno_t,SnapRealm*> open_past_parents;  // these are explicitly pinned.

  // cache
  snapid_t cached_seq;           // max seq over self and all past+present parents.
  snapid_t cached_last_created;  // max last_created over all past+present parents
  snapid_t cached_last_destroyed;
  set<snapid_t> cached_snaps;
  SnapContext cached_snap_context;

  bufferlist cached_snap_trace;

  elist<CInode*> inodes_with_caps;             // for efficient realm splits
  map<client_t, xlist<Capability*>* > client_caps;   // to identify clients who need snap notifications

snaprealm.cc：

#include "SnapRealm.h"
#include "MDCache.h"
#include "MDS.h"

#include "messages/MClientSnap.h"


/*
 * SnapRealm
 */

#define dout_subsys ceph_subsys_mds
#undef dout_prefix
#define dout_prefix _prefix(_dout, mdcache->mds->get_nodeid(), inode, srnode.seq, this)
static ostream& _prefix(std::ostream *_dout, int whoami, CInode *inode,
            uint64_t seq, SnapRealm *realm) {
  return *_dout << " mds." << whoami
        << ".cache.snaprealm(" << inode->ino()
        << " seq " << seq << " " << realm << ") ";
}

ostream& operator<<(ostream& out, const SnapRealm& realm) 
{
  out << "snaprealm(" << realm.inode->ino()
      << " seq " << realm.srnode.seq
      << " lc " << realm.srnode.last_created
      << " cr " << realm.srnode.created;
  if (realm.srnode.created != realm.srnode.current_parent_since)
    out << " cps " << realm.srnode.current_parent_since;
  out << " snaps=" << realm.srnode.snaps;
  if (realm.srnode.past_parents.size()) {
    out << " past_parents=(";
    for (map<snapid_t, snaplink_t>::const_iterator p = realm.srnode.past_parents.begin(); 
     p != realm.srnode.past_parents.end(); 
     ++p) {
      if (p != realm.srnode.past_parents.begin()) out << ",";
      out << p->second.first << "-" << p->first
      << "=" << p->second.ino;
    }
    out << ")";
  }
  out << " " << &realm << ")";
  return out;
}

//添加parent到open_past_parents的map容器中
void SnapRealm::add_open_past_parent(SnapRealm *parent)
{
  open_past_parents[parent->inode->ino()] = parent;
  parent->inode->get(CInode::PIN_PASTSNAPPARENT);        //标记为CInode::PIN_PASTSNAPPARENT
}

struct C_SR_RetryOpenParents : public MDSInternalContextBase {
  SnapRealm *sr;
  snapid_t first, last, parent_last;
  inodeno_t parent;
  MDSInternalContextBase* fin;
  C_SR_RetryOpenParents(SnapRealm *s, snapid_t f, snapid_t l, snapid_t pl,
            inodeno_t p, MDSInternalContextBase *c) :
    sr(s), first(f), last(l), parent_last(pl),  parent(p), fin(c) {}
  MDS *get_mds() { return sr->mdcache->mds; }
  void finish(int r) {
    if (r < 0)
      sr->_remove_missing_parent(parent_last, parent, r);
    if (sr->_open_parents(fin, first, last))
      fin->complete(0);
  }
};

//删除找不到snapid的parent
void SnapRealm::_remove_missing_parent(snapid_t snapid, inodeno_t parent, int err)
{
  map<snapid_t, snaplink_t>::iterator p = srnode.past_parents.find(snapid);
  if (p != srnode.past_parents.end()) {
    dout(10) << __func__ << " " << parent << " [" << p->second.first << ","
         << p->first << "]  errno " << err << dendl;
    srnode.past_parents.erase(p);
  } else {
    dout(10) << __func__ << " " << parent << " not found" << dendl;
  }
}

//判定parents是否open，返回true or false
bool SnapRealm::_open_parents(MDSInternalContextBase *finish, snapid_t first, snapid_t last)
{
  dout(10) << "open_parents [" << first << "," << last << "]" << dendl;
  if (open) 
    return true;

  // make sure my current parents' parents are open...
  if (parent) {
    dout(10) << " current parent [" << srnode.current_parent_since << ",head] is " << *parent
         << " on " << *parent->inode << dendl;
    if (last >= srnode.current_parent_since &&
    !parent->_open_parents(finish, MAX(first, srnode.current_parent_since), last))
      return false;
  }

  // and my past parents too!
  assert(srnode.past_parents.size() >= open_past_parents.size());
  if (srnode.past_parents.size() > open_past_parents.size()) {
    for (map<snapid_t, snaplink_t>::iterator p = srnode.past_parents.begin();
     p != srnode.past_parents.end();
     ++p) {    
      dout(10) << " past_parent [" << p->second.first << "," << p->first << "] is "
           << p->second.ino << dendl;
      CInode *parent = mdcache->get_inode(p->second.ino);
      if (!parent) {
    C_SR_RetryOpenParents *fin = new C_SR_RetryOpenParents(this, first, last, p->first,
                                   p->second.ino, finish);
    mdcache->open_ino(p->second.ino, mdcache->mds->mdsmap->get_metadata_pool(), fin);
    return false;
      }
      assert(parent->snaprealm);  // hmm!
      if (!parent->snaprealm->_open_parents(finish, p->second.first, p->first))
    return false;
      if (!open_past_parents.count(p->second.ino)) {
    add_open_past_parent(parent->snaprealm);
      }
    }
  }

  open = true;
  return true;
}

//判断从first到last的snap的past_parents是否open
bool SnapRealm::have_past_parents_open(snapid_t first, snapid_t last)    
{
  dout(10) << "have_past_parents_open [" << first << "," << last << "]" << dendl;
  if (open)
    return true;

  for (map<snapid_t, snaplink_t>::iterator p = srnode.past_parents.lower_bound(first);
       p != srnode.past_parents.end();
       ++p) {
    if (p->second.first > last)
      break;
    dout(10) << " past parent [" << p->second.first << "," << p->first << "] was "
         << p->second.ino << dendl;
    if (open_past_parents.count(p->second.ino) == 0) {
      dout(10) << " past parent " << p->second.ino << " is not open" << dendl;
      return false;
    }
    if (!open_past_parents[p->second.ino]->have_past_parents_open(MAX(first, p->second.first),
                                  MIN(last, p->first)))
      return false;
  }

  open = true;
  return true;
}


void SnapRealm::close_parents()
{
  for (map<inodeno_t,SnapRealm*>::iterator p = open_past_parents.begin();
       p != open_past_parents.end();
       ++p)
    p->second->inode->put(CInode::PIN_PASTSNAPPARENT);   //解除标记CInode::PIN_PASTSNAPPARENT
  open_past_parents.clear();  //清空
}


/*
 * get list of snaps for this realm.  we must include parents' snaps
 * for the intervals during which they were our parent.
 */
 //将realm中的snaps取出并存入第一个参数set容器中
void SnapRealm::build_snap_set(set<snapid_t> &s,
                   snapid_t& max_seq, snapid_t& max_last_created, snapid_t& max_last_destroyed,
                   snapid_t first, snapid_t last)
{
  dout(10) << "build_snap_set [" << first << "," << last << "] on " << *this << dendl;

  if (srnode.seq > max_seq)
    max_seq = srnode.seq;
  if (srnode.last_created > max_last_created)
    max_last_created = srnode.last_created;
  if (srnode.last_destroyed > max_last_destroyed)
    max_last_destroyed = srnode.last_destroyed;

  // include my snaps within interval [first,last]
  for (map<snapid_t, SnapInfo>::iterator p = srnode.snaps.lower_bound(first); // first element >= first
       p != srnode.snaps.end() && p->first <= last;
       ++p)
    s.insert(p->first);

  // include snaps for parents during intervals that intersect [first,last]
  for (map<snapid_t, snaplink_t>::iterator p = srnode.past_parents.lower_bound(first);
       p != srnode.past_parents.end() && p->first >= first && p->second.first <= last;
       ++p) {
    CInode *oldparent = mdcache->get_inode(p->second.ino);
    assert(oldparent);  // call open_parents first!
    assert(oldparent->snaprealm);
    oldparent->snaprealm->build_snap_set(s, max_seq, max_last_created, max_last_destroyed,
                     MAX(first, p->second.first),
                     MIN(last, p->first));           //递归，将past_parent的snapid加入set容器
  }
  if (srnode.current_parent_since <= last && parent)
    parent->build_snap_set(s, max_seq, max_last_created, max_last_destroyed,
               MAX(first, srnode.current_parent_since), last);         //递归，将current_parent的snapid加入set容器
}


//检查cache中的数据结构并更新cached_snaps和cached_snap_trace
void SnapRealm::check_cache()
{
  if (cached_seq >= srnode.seq)
    return;

  cached_snaps.clear();
  cached_snap_context.clear();

  cached_last_created = srnode.last_created;
  cached_last_destroyed = srnode.last_destroyed;
  cached_seq = srnode.seq;
  build_snap_set(cached_snaps, cached_seq, cached_last_created, cached_last_destroyed,
         0, CEPH_NOSNAP);

  cached_snap_trace.clear();
  build_snap_trace(cached_snap_trace);
  
  dout(10) << "check_cache rebuilt " << cached_snaps
       << " seq " << srnode.seq
       << " cached_seq " << cached_seq
       << " cached_last_created " << cached_last_created
       << " cached_last_destroyed " << cached_last_destroyed
       << ")" << dendl;
}


const set<snapid_t>& SnapRealm::get_snaps()
{
  check_cache();
  dout(10) << "get_snaps " << cached_snaps
       << " (seq " << srnode.seq << " cached_seq " << cached_seq << ")"
       << dendl;
  return cached_snaps;
}

/*
 * build vector in reverse sorted order
 */
 //循环将存储snapid的set中的内容放入返回值cached_snap_context中
const SnapContext& SnapRealm::get_snap_context()
{
  check_cache();

  if (!cached_snap_context.seq) {
    cached_snap_context.seq = cached_seq;
    cached_snap_context.snaps.resize(cached_snaps.size());
    unsigned i = 0;
    for (set<snapid_t>::reverse_iterator p = cached_snaps.rbegin();
     p != cached_snaps.rend();
     ++p)
      cached_snap_context.snaps[i++] = *p;
  }

  return cached_snap_context;
}

//得到cache_snap，并将其info存入infomap的map容器中
void SnapRealm::get_snap_info(map<snapid_t,SnapInfo*>& infomap, snapid_t first, snapid_t last)
{
  const set<snapid_t>& snaps = get_snaps();
  dout(10) << "get_snap_info snaps " << snaps << dendl;

  // include my snaps within interval [first,last]
  for (map<snapid_t, SnapInfo>::iterator p = srnode.snaps.lower_bound(first); // first element >= first
       p != srnode.snaps.end() && p->first <= last;
       ++p)
    infomap[p->first] = &p->second;    //令键为p->first的infomap的值等于&p->second

  // include snaps for parents during intervals that intersect [first,last]
  for (map<snapid_t, snaplink_t>::iterator p = srnode.past_parents.lower_bound(first);
       p != srnode.past_parents.end() && p->first >= first && p->second.first <= last;
       ++p) {
    CInode *oldparent = mdcache->get_inode(p->second.ino);
    assert(oldparent);  // call open_parents first!
    assert(oldparent->snaprealm);
    oldparent->snaprealm->get_snap_info(infomap,
                    MAX(first, p->second.first),
                    MIN(last, p->first));
  }
  if (srnode.current_parent_since <= last && parent)
    parent->get_snap_info(infomap, MAX(first, srnode.current_parent_since), last);
}

const string& SnapRealm::get_snapname(snapid_t snapid, inodeno_t atino)
{
  if (srnode.snaps.count(snapid)) {
    if (atino == inode->ino())
      return srnode.snaps[snapid].name;
    else
      return srnode.snaps[snapid].get_long_name();
  }

  map<snapid_t,snaplink_t>::iterator p = srnode.past_parents.lower_bound(snapid);
  if (p != srnode.past_parents.end() && p->second.first <= snapid) {
    CInode *oldparent = mdcache->get_inode(p->second.ino);
    assert(oldparent);  // call open_parents first!
    assert(oldparent->snaprealm);    
    return oldparent->snaprealm->get_snapname(snapid, atino);
  }

  assert(srnode.current_parent_since <= snapid);
  assert(parent);
  return parent->get_snapname(snapid, atino);
}

//判断第一个参数n，在srnode.snaps从first到last范围内找出对应的snapid
snapid_t SnapRealm::resolve_snapname(const string& n, inodeno_t atino, snapid_t first, snapid_t last)
{
  // first try me
  dout(10) << "resolve_snapname '" << n << "' in [" << first << "," << last << "]" << dendl;

  //snapid_t num;
  //if (n[0] == '~') num = atoll(n.c_str()+1);

  bool actual = (atino == inode->ino());
  string pname;
  inodeno_t pino;
  if (!actual) {
    if (!n.length() ||
    n[0] != '_') return 0;
    int next_ = n.find('_', 1);
    if (next_ < 0) return 0;
    pname = n.substr(1, next_ - 1);
    pino = atoll(n.c_str() + next_ + 1);
    dout(10) << " " << n << " parses to name '" << pname << "' dirino " << pino << dendl;
  }

//根据snapinfo得到snapid
  for (map<snapid_t, SnapInfo>::iterator p = srnode.snaps.lower_bound(first); // first element >= first
       p != srnode.snaps.end() && p->first <= last;
       ++p) {
    dout(15) << " ? " << p->second << dendl;
    //if (num && p->second.snapid == num)
    //return p->first;
    if (actual && p->second.name == n)
    return p->first;
    if (!actual && p->second.name == pname && p->second.ino == pino)
      return p->first;
  }

    // include snaps for parents during intervals that intersect [first,last]
  for (map<snapid_t, snaplink_t>::iterator p = srnode.past_parents.lower_bound(first);
       p != srnode.past_parents.end() && p->first >= first && p->second.first <= last;
       ++p) {
    CInode *oldparent = mdcache->get_inode(p->second.ino);
    assert(oldparent);  // call open_parents first!
    assert(oldparent->snaprealm);
    snapid_t r = oldparent->snaprealm->resolve_snapname(n, atino,
                            MAX(first, p->second.first),
                            MIN(last, p->first));
    if (r)
      return r;
  }
  if (parent && srnode.current_parent_since <= last)
    return parent->resolve_snapname(n, atino, MAX(first, srnode.current_parent_since), last);
  return 0;
}


//调整parent令变量parent等于当前parent
void SnapRealm::adjust_parent()
{
  SnapRealm *newparent = inode->get_parent_dn()->get_dir()->get_inode()->find_snaprealm();
  if (newparent != parent) {
    dout(10) << "adjust_parent " << parent << " -> " << newparent << dendl;
    if (parent)
      parent->open_children.erase(this);
    parent = newparent;
    if (parent)
      parent->open_children.insert(this);
    
    invalidate_cached_snaps();
  }
}

//将child分离出当前realm
void SnapRealm::split_at(SnapRealm *child)
{
  dout(10) << "split_at " << *child 
       << " on " << *child->inode << dendl;

  if (inode->is_mdsdir() || !child->inode->is_dir()) {
    // it's not a dir.
    if (child->inode->containing_realm) {
      //  - no open children.
      //  - only need to move this child's inode's caps.
      child->inode->move_to_realm(child);           //脱离containing_realm，加入到child realm
    } else {
      // no caps, nothing to move/split.
      dout(20) << " split no-op, no caps to move on file " << *child->inode << dendl;
      assert(!child->inode->is_any_caps());
    }
    return;
  }

  // it's a dir.

  // split open_children
  dout(10) << " open_children are " << open_children << dendl;
  for (set<SnapRealm*>::iterator p = open_children.begin();
       p != open_children.end(); ) {
    SnapRealm *realm = *p;
    if (realm != child &&
    child->inode->is_projected_ancestor_of(realm->inode)) {
      dout(20) << " child gets child realm " << *realm << " on " << *realm->inode << dendl;
      realm->parent = child;
      child->open_children.insert(realm);
      open_children.erase(p++);
    } else {
      dout(20) << "    keeping child realm " << *realm << " on " << *realm->inode << dendl;
      ++p;
    }
  }

  // split inodes_with_caps
  elist<CInode*>::iterator p = inodes_with_caps.begin(member_offset(CInode, item_caps));
  while (!p.end()) {
    CInode *in = *p;
    ++p;

    // does inode fall within the child realm?
    bool under_child = false;

    if (in == child->inode) {
      under_child = true;
    } else {
      CInode *t = in;
      while (t->get_parent_dn()) {
    t = t->get_parent_dn()->get_dir()->get_inode();
    if (t == child->inode) {
      under_child = true;
      break;
    }
    if (t == in)
      break;
      }
    }
    if (under_child) {
      dout(20) << " child gets " << *in << dendl;
      in->move_to_realm(child);         //脱离containing_realm，加入到child realm
    } else {
      dout(20) << "    keeping " << *in << dendl;
    }
  }

}

const bufferlist& SnapRealm::get_snap_trace()
{
  check_cache();
  return cached_snap_trace;
}

void SnapRealm::build_snap_trace(bufferlist& snapbl)    //将snaps和所有parents存储在snapbl中
{
  SnapRealmInfo info(inode->ino(), srnode.created, srnode.seq, srnode.current_parent_since);

  if (parent) {
    info.h.parent = parent->inode->ino();
    if (!srnode.past_parents.empty()) {
      snapid_t last = srnode.past_parents.rbegin()->first;
      set<snapid_t> past;
      snapid_t max_seq, max_last_created, max_last_destroyed;
      build_snap_set(past, max_seq, max_last_created, max_last_destroyed, 0, last);
      info.prior_parent_snaps.reserve(past.size());
      for (set<snapid_t>::reverse_iterator p = past.rbegin(); p != past.rend(); ++p)
    info.prior_parent_snaps.push_back(*p);
      dout(10) << "build_snap_trace prior_parent_snaps from [1," << last << "] "
           << info.prior_parent_snaps << dendl;
    }
  } else 
    info.h.parent = 0;

  info.my_snaps.reserve(srnode.snaps.size());
  for (map<snapid_t,SnapInfo>::reverse_iterator p = srnode.snaps.rbegin();
       p != srnode.snaps.rend();
       ++p)
    info.my_snaps.push_back(p->first);
  dout(10) << "build_snap_trace my_snaps " << info.my_snaps << dendl;

  ::encode(info, snapbl);

  if (parent)
    parent->build_snap_trace(snapbl);
}


//删除past_parents中有但cached_snaps中没有的
void SnapRealm::prune_past_parents()
{
  dout(10) << "prune_past_parents" << dendl;
  check_cache();
  assert(open);

  map<snapid_t, snaplink_t>::iterator p = srnode.past_parents.begin();
  while (p != srnode.past_parents.end()) {
    set<snapid_t>::iterator q = cached_snaps.lower_bound(p->second.first);
    if (q == cached_snaps.end() ||   //若cached_snaps中没有p指向的值则erase
    *q > p->first) {
      dout(10) << "prune_past_parents pruning [" << p->second.first << "," << p->first 
           << "] " << p->second.ino << dendl;
      srnode.past_parents.erase(p++);
    } else {
      dout(10) << "prune_past_parents keeping [" << p->second.first << "," << p->first 
           << "] " << p->second.ino << dendl;
      ++p;
    }
  }
}

snapserver.cc：

#include "SnapServer.h"
#include "MDS.h"
#include "osd/OSDMap.h"
#include "osdc/Objecter.h"
#include "mon/MonClient.h"

#include "include/types.h"
#include "messages/MMDSTableRequest.h"
#include "messages/MRemoveSnaps.h"

#include "msg/Messenger.h"

#include "common/config.h"
#include "include/assert.h"

#define dout_subsys ceph_subsys_mds
#undef dout_prefix
#define dout_prefix *_dout << "mds." << rank << ".snap "


void SnapServer::reset_state()  //重置状态,将snaps和need_to_purge中的内容清空
{
  last_snap = 1;  /* snapid 1 reserved for initial root snaprealm */
  snaps.clear();
  need_to_purge.clear();

  // find any removed snapshot in data pools
  snapid_t first_free = 0;
  const OSDMap *osdmap = mds->objecter->get_osdmap_read();
  for (set<int64_t>::const_iterator p = mds->mdsmap->get_data_pools().begin();
       p != mds->mdsmap->get_data_pools().end();
       ++p) {
    const pg_pool_t *pi = osdmap->get_pg_pool(*p);
    if (!pi->removed_snaps.empty() &&
        pi->removed_snaps.range_end() > first_free)
      first_free = pi->removed_snaps.range_end();
  }
  mds->objecter->put_osdmap_read();
  if (first_free > last_snap)
    last_snap = first_free;        //扩展last_snap范围
}


// SERVER
//根据操作执行:创建-将bl中的内容加入到pending_creat中，销毁-将bl中的内容加入到pending_destroy中
void SnapServer::_prepare(bufferlist &bl, uint64_t reqid, mds_rank_t bymds)
{
  bufferlist::iterator p = bl.begin();
  __u32 op;
  ::decode(op, p);

  switch (op) {
  case TABLE_OP_CREATE:
    {
      version++;

      SnapInfo info;
      ::decode(info.ino, p);
      if (!p.end()) {
    ::decode(info.name, p);
    ::decode(info.stamp, p);
    info.snapid = ++last_snap;
    pending_create[version] = info;
    dout(10) << "prepare v" << version << " create " << info << dendl;
      } else {
    pending_noop.insert(version);
    dout(10) << "prepare v" << version << " noop" << dendl;
      }
      bl.clear();
      ::encode(last_snap, bl);
    }
    break;

  case TABLE_OP_DESTROY:
    {
      inodeno_t ino;
      snapid_t snapid;
      ::decode(ino, p);    // not used, currently.
      ::decode(snapid, p);
      version++;

      // bump last_snap... we use it as a version value on the snaprealm.
      ++last_snap;

      pending_destroy[version] = pair<snapid_t,snapid_t>(snapid, last_snap);
      dout(10) << "prepare v" << version << " destroy " << snapid << " seq " << last_snap << dendl;

      bl.clear();
      ::encode(last_snap, bl);
    }
    break;

  default:
    assert(0);
  }
  //dump();
}

//返回容器中此id出现的次数
bool SnapServer::_is_prepared(version_t tid)
{
  return 
    pending_create.count(tid) ||
    pending_destroy.count(tid);
}

//将操作pending_create或pending_destroy的内容存入对应的结构中
bool SnapServer::_commit(version_t tid, MMDSTableRequest *req)
{
  if (pending_create.count(tid)) {    //若此tid在pending_creat中有值，清空并将此结构中对应snapid中的内容存入snaps中
    dout(7) << "commit " << tid << " create " << pending_create[tid] << dendl;
    snaps[pending_create[tid].snapid] = pending_create[tid];
    pending_create.erase(tid);
  } 

  else if (pending_destroy.count(tid)) {      //若此tid在pending_creat中有值，清空并将对应内容加入到need_to_purge
    snapid_t sn = pending_destroy[tid].first;    //removed_snap
    snapid_t seq = pending_destroy[tid].second;   //seq
    dout(7) << "commit " << tid << " destroy " << sn << " seq " << seq << dendl;
    snaps.erase(sn);

    for (set<int64_t>::const_iterator p = mds->mdsmap->get_data_pools().begin();
     p != mds->mdsmap->get_data_pools().end();
     ++p) {
      need_to_purge[*p].insert(sn);
      need_to_purge[*p].insert(seq);
    }

    pending_destroy.erase(tid);
  }
  else if (pending_noop.count(tid)) {     //清空pending_noop
    dout(7) << "commit " << tid << " noop" << dendl;
    pending_noop.erase(tid);
  }
  else
    assert(0);

  // bump version.
  version++;
  //dump();
  return true;
}

//输出要执行的回滚命令并清空pending_create等结构中的内容，不执行具体操作
void SnapServer::_rollback(version_t tid) 
{
  if (pending_create.count(tid)) {
    dout(7) << "rollback " << tid << " create " << pending_create[tid] << dendl;
    pending_create.erase(tid);
  } 

  else if (pending_destroy.count(tid)) {
    dout(7) << "rollback " << tid << " destroy " << pending_destroy[tid] << dendl;
    pending_destroy.erase(tid);
  }
  
  else if (pending_noop.count(tid)) {
    dout(7) << "rollback " << tid << " noop" << dendl;
    pending_noop.erase(tid);
  }    

  else
    assert(0);

  // bump version.
  version++;
  //dump();
}

//将传入参数bl中的项在need_to_purge的对应项擦除
void SnapServer::_server_update(bufferlist& bl)
{
  bufferlist::iterator p = bl.begin();
  map<int, vector<snapid_t> > purge;
  ::decode(purge, p);

  dout(7) << "_server_update purged " << purge << dendl;
  for (map<int, vector<snapid_t> >::iterator p = purge.begin();
       p != purge.end();
       ++p) {
    for (vector<snapid_t>::iterator q = p->second.begin();
     q != p->second.end();
     ++q)
      need_to_purge[p->first].erase(*q);
    if (need_to_purge[p->first].empty())
      need_to_purge.erase(p->first);
  }

  version++;
}

void SnapServer::handle_query(MMDSTableRequest *req)
{
  req->put();
}


//遍历need_to_purge，执行remove的操作
void SnapServer::check_osd_map(bool force)
{
  if (!force && version == last_checked_osdmap) {
    dout(10) << "check_osd_map - version unchanged" << dendl;
    return;
  }
  dout(10) << "check_osd_map need_to_purge=" << need_to_purge << dendl;

  map<int, vector<snapid_t> > all_purge;
  map<int, vector<snapid_t> > all_purged;

  const OSDMap *osdmap = mds->objecter->get_osdmap_read();
  for (map<int, set<snapid_t> >::iterator p = need_to_purge.begin();       //遍历need_to_purge
       p != need_to_purge.end();
       ++p) {
    int id = p->first;
    const pg_pool_t *pi = osdmap->get_pg_pool(id);
    for (set<snapid_t>::iterator q = p->second.begin();
     q != p->second.end();
     ++q) {
      if (pi->is_removed_snap(*q)) {     //已经remove，存入all_purged，还没有remove则存入all_purge
    dout(10) << " osdmap marks " << *q << " as removed" << dendl;
    all_purged[id].push_back(*q);
      } else {
    all_purge[id].push_back(*q);
      }
    }
  }
  mds->objecter->put_osdmap_read();

  if (!all_purged.empty()) {  //已经purge，调用do_server_update更新mdlog
    // prepare to remove from need_to_purge list
    bufferlist bl;
    ::encode(all_purged, bl);
    do_server_update(bl);
  }

  if (!all_purge.empty()) {        //需要purge，创建消息存储all_purge并发送给mon
    dout(10) << "requesting removal of " << all_purge << dendl;
    MRemoveSnaps *m = new MRemoveSnaps(all_purge);
    mds->monc->send_mon_message(m);
  }

  last_checked_osdmap = version;
}



void SnapServer::dump(Formatter *f) const
{
  f->open_object_section("snapserver");

  f->dump_int("last_snap", last_snap.val);

  f->open_array_section("pending_noop");
  for(set<version_t>::const_iterator i = pending_noop.begin(); i != pending_noop.end(); ++i) {
    f->dump_unsigned("version", *i);
  }
  f->close_section();

  f->open_array_section("snaps");
  for (map<snapid_t, SnapInfo>::const_iterator i = snaps.begin(); i != snaps.end(); ++i) {
    f->open_object_section("snap");
    i->second.dump(f);
    f->close_section();
  }
  f->close_section();

  f->open_object_section("need_to_purge");
  for (map<int, set<snapid_t> >::const_iterator i = need_to_purge.begin(); i != need_to_purge.end(); ++i) {
    stringstream pool_id;
    pool_id << i->first;
    f->open_array_section(pool_id.str().c_str());
    for (set<snapid_t>::const_iterator s = i->second.begin(); s != i->second.end(); ++s) {
      f->dump_unsigned("snapid", s->val);
    }
    f->close_section();
  }
  f->close_section();

  f->open_array_section("pending_create");
  for(map<version_t, SnapInfo>::const_iterator i = pending_create.begin(); i != pending_create.end(); ++i) {
    f->open_object_section("snap");
    f->dump_unsigned("version", i->first);
    f->open_object_section("snapinfo");
    i->second.dump(f);
    f->close_section();
    f->close_section();
  }
  f->close_section();

  f->open_array_section("pending_destroy");
  for(map<version_t, pair<snapid_t, snapid_t> >::const_iterator i = pending_destroy.begin(); i != pending_destroy.end(); ++i) {
    f->open_object_section("snap");
    f->dump_unsigned("version", i->first);
    f->dump_unsigned("removed_snap", i->second.first);
    f->dump_unsigned("seq", i->second.second);
    f->close_section();
  }
  f->close_section();

  f->close_section();
}

void SnapServer::generate_test_instances(list<SnapServer*>& ls)
{
  list<SnapInfo*> snapinfo_instances;
  SnapInfo::generate_test_instances(snapinfo_instances);
  SnapInfo populated_snapinfo = *(snapinfo_instances.back());
  for (list<SnapInfo*>::iterator i = snapinfo_instances.begin(); i != snapinfo_instances.end(); ++i) {
    delete *i;
  }

  SnapServer *blank = new SnapServer();
  ls.push_back(blank);
  SnapServer *populated = new SnapServer();
  populated->last_snap = 123;
  populated->snaps[456] = populated_snapinfo;
  populated->need_to_purge[2].insert(012);
  populated->pending_create[234] = populated_snapinfo;
  populated->pending_destroy[345].first = 567;
  populated->pending_destroy[345].second = 768;
  populated->pending_noop.insert(890);

  ls.push_back(populated);

}

snapmapper.h中有一段注释如下，

/**
 * SnapMapper
 *
 * Manages two mappings:
 *  1) hobject_t -> {snapid}    //为每个克隆对象存储快照集
 *  2) snapid -> {hobject_t}    //存储快照集的对象作为其快照
 *
 * We accomplish this using two sets of keys:
 *  1) OBJECT_PREFIX + obj.str() -> encoding of object_snaps
 *  2) MAPPING_PREFIX + snapid_t + obj.str() -> encoding of pair<snapid_t, obj>
 *
 * The on disk strings and encodings are implemented in to_raw, to_raw_key,
 * from_raw, to_object_key.
 *
 * The object -> {snapid} mapping is primarily included so that the
 * SnapMapper state can be verified against the external PG state during
 * scrub etc.
 *
 * The 2) mapping is arranged such that all objects in a particular
 * snap will sort together, and so that all objects in a pg for a
 * particular snap will group under up to 8 prefixes.    //最多8前缀需要进行检查以确定用于特定pg在一个特定的单元的所有对象
 */

snapmapper.cc:

#include "SnapMapper.h"

#define dout_subsys ceph_subsys_osd
#undef dout_prefix
#define dout_prefix *_dout << "snap_mapper."

using std::string;

const string SnapMapper::MAPPING_PREFIX = "MAP_";
const string SnapMapper::OBJECT_PREFIX = "OBJ_";

int OSDriver::get_keys(
  const std::set<std::string> &keys,
  std::map<std::string, bufferlist> *out)
{
  return os->omap_get_values(cid, hoid, keys, out);
}

int OSDriver::get_next(
  const std::string &key,
  pair<std::string, bufferlist> *next)
{
  ObjectMap::ObjectMapIterator iter =
    os->get_omap_iterator(cid, hoid);
  if (!iter) {
    assert(0);
    return -EINVAL;
  }
  iter->upper_bound(key);
  if (iter->valid()) {
    if (next)
      *next = make_pair(iter->key(), iter->value());
    return 0;
  } else {
    return -ENOENT;
  }
}

struct Mapping {
  snapid_t snap;
  hobject_t hoid;
  Mapping(const pair<snapid_t, hobject_t> &in)
    : snap(in.first), hoid(in.second) {}
  Mapping() : snap(0) {}
  void encode(bufferlist &bl) const {
    ENCODE_START(1, 1, bl);
    ::encode(snap, bl);
    ::encode(hoid, bl);
    ENCODE_FINISH(bl);
  }
  void decode(bufferlist::iterator &bl) {
    DECODE_START(1, bl);
    ::decode(snap, bl);
    ::decode(hoid, bl);
    DECODE_FINISH(bl);
  }
};
WRITE_CLASS_ENCODER(Mapping)

string SnapMapper::get_prefix(snapid_t snap)
{
  char buf[100];
  int len = snprintf(
    buf, sizeof(buf),
    "%.*X_", (int)(sizeof(snap)*2),
    static_cast<unsigned>(snap));
  return MAPPING_PREFIX + string(buf, len);
}

string SnapMapper::to_raw_key(            //实现mapping有两种key，此为raw_key
  const pair<snapid_t, hobject_t> &in)
{
  return get_prefix(in.first) + shard_prefix + in.second.to_str();
}

pair<string, bufferlist> SnapMapper::to_raw(    //返回键值和对应map
  const pair<snapid_t, hobject_t> &in)
{
  bufferlist bl;
  ::encode(Mapping(in), bl);
  return make_pair(
    to_raw_key(in),
    bl);
}

pair<snapid_t, hobject_t> SnapMapper::from_raw(    //根据map返回snapid和hobject对应的snap和hoid 与to_raw相反
  const pair<std::string, bufferlist> &image)
{
  Mapping map;
  bufferlist bl(image.second);
  bufferlist::iterator bp(bl.begin());
  ::decode(map, bp);
  return make_pair(map.snap, map.hoid);
}

bool SnapMapper::is_mapping(const string &to_test)
{
  return to_test.substr(0, MAPPING_PREFIX.size()) == MAPPING_PREFIX;
}

string SnapMapper::to_object_key(const hobject_t &hoid)          //实现mapping有两种key，此为object_key
{
  return OBJECT_PREFIX + shard_prefix + hoid.to_str();
}

void SnapMapper::object_snaps::encode(bufferlist &bl) const
{
  ENCODE_START(1, 1, bl);
  ::encode(oid, bl);
  ::encode(snaps, bl);
  ENCODE_FINISH(bl);
}

void SnapMapper::object_snaps::decode(bufferlist::iterator &bl)
{
  DECODE_START(1, bl);
  ::decode(oid, bl);
  ::decode(snaps, bl);
  DECODE_FINISH(bl);
}

int SnapMapper::get_snaps(      //根据iod从backend中找到对应的snaps并输出
  const hobject_t &oid,
  object_snaps *out)
{
  assert(check(oid));
  set<string> keys;
  map<string, bufferlist> got;
  keys.insert(to_object_key(oid));
  int r = backend.get_keys(keys, &got);
  if (r < 0)
    return r;
  if (got.empty())
    return -ENOENT;
  if (out) {
    bufferlist::iterator bp = got.begin()->second.begin();
    ::decode(*out, bp);
    dout(20) << __func__ << " " << oid << " " << out->snaps << dendl;
    assert(!out->snaps.empty());
  } else {
    dout(20) << __func__ << " " << oid << " (out == NULL)" << dendl;
  }
  return 0;
}

void SnapMapper::clear_snaps(         //remove to_object_key
  const hobject_t &oid,
  MapCacher::Transaction<std::string, bufferlist> *t)
{
  assert(check(oid));
  set<string> to_remove;
  to_remove.insert(to_object_key(oid));
  backend.remove_keys(to_remove, t);
}

void SnapMapper::set_snaps(           //将object_key和snap设置为一个map，并调用set_keys加入到Transaction结构中
  const hobject_t &oid,
  const object_snaps &in,
  MapCacher::Transaction<std::string, bufferlist> *t)
{
  assert(check(oid));
  map<string, bufferlist> to_set;
  bufferlist bl;
  ::encode(in, bl);
  to_set[to_object_key(oid)] = bl;
  backend.set_keys(to_set, t);
}

int SnapMapper::update_snaps(          //将new_snaps更新加入backend中
  const hobject_t &oid,
  const set<snapid_t> &new_snaps,
  const set<snapid_t> *old_snaps_check,
  MapCacher::Transaction<std::string, bufferlist> *t)
{
  dout(20) << __func__ << " " << oid << " " << new_snaps
       << " was " << (old_snaps_check ? *old_snaps_check : set<snapid_t>())
       << dendl;
  assert(check(oid));
  if (new_snaps.empty())
    return remove_oid(oid, t);

  object_snaps out;
  int r = get_snaps(oid, &out);
  if (r < 0)
    return r;
  if (old_snaps_check)
    assert(out.snaps == *old_snaps_check);

  object_snaps in(oid, new_snaps);
  set_snaps(oid, in, t);

  set<string> to_remove;
  for (set<snapid_t>::iterator i = out.snaps.begin();
       i != out.snaps.end();
       ++i) {
    if (!new_snaps.count(*i)) {
      to_remove.insert(to_raw_key(make_pair(*i, oid)));
    }
  }
  backend.remove_keys(to_remove, t);
  return 0;
}

void SnapMapper::add_oid(
  const hobject_t &oid,
  const set<snapid_t>& snaps,
  MapCacher::Transaction<std::string, bufferlist> *t)
{
  dout(20) << __func__ << " " << oid << " " << snaps << dendl;
  assert(check(oid));
  {
    object_snaps out;
    int r = get_snaps(oid, &out);
    assert(r == -ENOENT);
  }

  object_snaps _snaps(oid, snaps);
  set_snaps(oid, _snaps, t);

  map<string, bufferlist> to_add;
  for (set<snapid_t>::iterator i = snaps.begin();
       i != snaps.end();
       ++i) {
    to_add.insert(to_raw(make_pair(*i, oid)));
  }
  backend.set_keys(to_add, t);  //将to_add和t打包加入backend
}

int SnapMapper::get_next_object_to_trim(
  snapid_t snap,
  hobject_t *hoid)
{
  for (set<string>::iterator i = prefixes.begin();
       i != prefixes.end();
       ++i) {
    string list_after(get_prefix(snap) + *i);

    pair<string, bufferlist> next;
    int r = backend.get_next(list_after, &next);
    if (r < 0) {
      break; // Done
    }

    if (next.first.substr(0, list_after.size()) !=
    list_after) {
      continue; // Done with this prefix
    }

    assert(is_mapping(next.first));

    pair<snapid_t, hobject_t> next_decoded(from_raw(next));
    assert(next_decoded.first == snap);
    assert(check(next_decoded.second));

    if (hoid)
      *hoid = next_decoded.second;
    return 0;
  }
  return -ENOENT;
}


int SnapMapper::remove_oid(
  const hobject_t &oid,
  MapCacher::Transaction<std::string, bufferlist> *t)
{
  dout(20) << __func__ << " " << oid << dendl;
  assert(check(oid));
  return _remove_oid(oid, t);
}

int SnapMapper::_remove_oid(
  const hobject_t &oid,
  MapCacher::Transaction<std::string, bufferlist> *t)
{
  object_snaps out;
  int r = get_snaps(oid, &out);
  if (r < 0)
    return r;

  clear_snaps(oid, t);       //remove to_object_key

  set<string> to_remove;
  for (set<snapid_t>::iterator i = out.snaps.begin();
       i != out.snaps.end();
       ++i) {
    to_remove.insert(to_raw_key(make_pair(*i, oid)));
  }
  backend.remove_keys(to_remove, t);    //remove to_raw_key
  return 0;
}

int SnapMapper::get_snaps(         //根据oid找到对应snaps并存在set容器snaps中
  const hobject_t &oid,
  std::set<snapid_t> *snaps)
{
  assert(check(oid));
  object_snaps out;
  int r = get_snaps(oid, &out);
  if (r < 0)
    return r;
  if (snaps)
    snaps->swap(out.snaps);
  return 0;
}