学习Faster R-CNN代码rpn（六）

代码文件结构

bbox_transform.py # bounding box变换。
generate_anchors.py # 生成anchor，根据几种尺度和比例生成的anchor。
proposal_layer.py # 通过将估计的边界框变换应用于一组常规框（称为“锚点”）来输出对象检测候选区域。选出合适的ROIS。
anchor_target_layer.py # 将anchor对应ground truth。生成anchor分类标签和边界框回归目标。为anchor找到训练所需的ground truth类别和坐标变换信息。
proposal_target_layer_cascade.py # 将对象检测候选分配给ground truth目标。生成候选分类标签和边界框回归目标。为选择出的rois找到训练所需的ground truth类别和坐标变换信息
rpn.py # RPN网络定义。

参考

详细的Faster R-CNN源码解析之RPN源码解析 https://blog.csdn.net/jiongnima/article/details/79781792

Faster R-CNN 入坑之源码阅读 https://www.jianshu.com/p/a223853f8402?tdsourcetag=s_pcqq_aiomsg 对RPN部分代码进行注释。

1 rpn.py

  1 class _RPN(nn.Module):
  2     """ region proposal network """
  3     def __init__(self, din):
  4         super(_RPN, self).__init__()
  5         
  6         #得到输入特征图的深度
  7         self.din = din  # get depth of input feature map, e.g., 512
  8         #anchor的尺度 __C.ANCHOR_SCALES = [8,16,32]
  9         self.anchor_scales = cfg.ANCHOR_SCALES
 10         #anchor的比例 __C.ANCHOR_RATIOS = [0.5,1,2]
 11         self.anchor_ratios = cfg.ANCHOR_RATIOS
 12         #特征步长 __C.FEAT_STRIDE = [16, ]
 13         self.feat_stride = cfg.FEAT_STRIDE[0]
 14 
 15         # define the convrelu layers processing input feature map
 16         #定义处理输入要素图的convrelu层
 17         #nn.Conv2d(in_channels, out_channels, kernel_size, stride=1, padding=0, dilation=1, groups=1, bias=True)
 18         self.RPN_Conv = nn.Conv2d(self.din, 512, 3, 1, 1, bias=True)
 19 
 20         # define bg/fg classifcation score layer
 21         #定义背景和前景分类得分
 22         #对每个anchor都要进行背景或前景的分类得分，个数就是尺度个数乘以比例个数再乘以2
 23         self.nc_score_out = len(self.anchor_scales) * len(self.anchor_ratios) * 2 # 2(bg/fg) * 9 (anchors)
 24         #上面是RPN卷积 这里是分类， 网络输入是512 输出是参数个数
 25         self.RPN_cls_score = nn.Conv2d(512, self.nc_score_out, 1, 1, 0)
 26 
 27         # define anchor box offset prediction layer
 28         #定义anchor的偏移层
 29         #偏移的输出个数是anchor个数乘以4
 30         self.nc_bbox_out = len(self.anchor_scales) * len(self.anchor_ratios) * 4 # 4(coords) * 9 (anchors)
 31         #网络输入是512 输出是参数个数
 32         self.RPN_bbox_pred = nn.Conv2d(512, self.nc_bbox_out, 1, 1, 0)
 33 
 34         # define proposal layer
 35         #定义候选区域层 _ProposalLayer
 36         # 参数是 特征步长 尺度 比例
 37         self.RPN_proposal = _ProposalLayer(self.feat_stride, self.anchor_scales, self.anchor_ratios)
 38 
 39         # define anchor target layer
 40         #定义anchor目标层 _AnchorTargetLayer
 41         # 参数是 特征步长 尺度 比例
 42         self.RPN_anchor_target = _AnchorTargetLayer(self.feat_stride, self.anchor_scales, self.anchor_ratios)
 43 
 44         self.rpn_loss_cls = 0 #分类损失
 45         self.rpn_loss_box = 0 #回归损失
 46 
 47     @staticmethod #静态方法
 48     #将x reshape
 49     def reshape(x, d):
 50         input_shape = x.size()
 51         x = x.view(
 52             input_shape[0],
 53             int(d),
 54             int(float(input_shape[1] * input_shape[2]) / float(d)),
 55             input_shape[3]
 56         )
 57         return x
 58 
 59     def forward(self, base_feat, im_info, gt_boxes, num_boxes):
 60 
 61         #features信息包括batch_size，data_height，data_width，num_channels
 62         #即批尺寸，特征数据高度，特征数据宽度，特征的通道数。
 63         batch_size = base_feat.size(0)#特征的第一维
 64 
 65         # return feature map after convrelu layer
 66         # 在卷积之后返回特征图
 67         rpn_conv1 = F.relu(self.RPN_Conv(base_feat), inplace=True)
 68         # get rpn classification score
 69         #得到RPN分类得分
 70         rpn_cls_score = self.RPN_cls_score(rpn_conv1)
 71 
 72         ##将rpn_cls_score转化为rpn_cls_score_reshape
 73         rpn_cls_score_reshape = self.reshape(rpn_cls_score, 2)
 74         #用softmax函数得到概率
 75         rpn_cls_prob_reshape = F.softmax(rpn_cls_score_reshape, 1)
 76         #前景背景分类，2个参数
 77         rpn_cls_prob = self.reshape(rpn_cls_prob_reshape, self.nc_score_out)
 78 
 79         # get rpn offsets to the anchor boxes
 80         #4个参数的偏移
 81         rpn_bbox_pred = self.RPN_bbox_pred(rpn_conv1)
 82 
 83         # proposal layer
 84         cfg_key = 'TRAIN' if self.training else 'TEST'
 85 
 86         #用anchor提取候选区域
 87         #参数有分类概率 四个参数偏移 图片信息
 88         rois = self.RPN_proposal((rpn_cls_prob.data, rpn_bbox_pred.data,
 89                                  im_info, cfg_key))
 90 
 91         self.rpn_loss_cls = 0#分类损失
 92         self.rpn_loss_box = 0#回归损失
 93 
 94         # generating training labels and build the rpn loss
 95         #生成训练标签并构建rpn损失
 96         if self.training:#训练
 97             assert gt_boxes is not None
 98 
 99             #anchor的目标
100             rpn_data = self.RPN_anchor_target((rpn_cls_score.data, gt_boxes, im_info, num_boxes))
101 
102             # compute classification loss
103             #计算分类损失
104             #permute(多维数组,[维数的组合]) 该函数是改变维数
105             #contiguous：view只能用在contiguous的variable上。
106             #如果在view之前用了transpose, permute等，需要用contiguous()来返回一个contiguous copy。
107             ##返回rpn网络判断的anchor前后景分数
108             rpn_cls_score = rpn_cls_score_reshape.permute(0, 2, 3, 1).contiguous().view(batch_size, -1, 2)
109             ##返回每个anchor属于前景还是后景的ground truth
110             rpn_label = rpn_data[0].view(batch_size, -1)
111 
112             rpn_keep = Variable(rpn_label.view(-1).ne(-1).nonzero().view(-1))
113             rpn_cls_score = torch.index_select(rpn_cls_score.view(-1,2), 0, rpn_keep)
114             rpn_label = torch.index_select(rpn_label.view(-1), 0, rpn_keep.data)
115             rpn_label = Variable(rpn_label.long())
116             self.rpn_loss_cls = F.cross_entropy(rpn_cls_score, rpn_label)
117             fg_cnt = torch.sum(rpn_label.data.ne(0))
118 
119             rpn_bbox_targets, rpn_bbox_inside_weights, rpn_bbox_outside_weights = rpn_data[1:]
120 
121             # compute bbox regression loss
122             #计算回归损失
123             
124             ##在训练计算边框误差时有用，仅对未超出图像边界的anchor有用
125             rpn_bbox_inside_weights = Variable(rpn_bbox_inside_weights)
126             ##在训练计算边框误差时有用，仅对未超出图像边界的anchor有用
127             rpn_bbox_outside_weights = Variable(rpn_bbox_outside_weights)
128             ##返回每个anchor对应的事实的四个偏移值
129             rpn_bbox_targets = Variable(rpn_bbox_targets)
130 
131             ##计算rpn的边界损失loss，请注意在这里用到了inside和outside_weights
132             self.rpn_loss_box = _smooth_l1_loss(rpn_bbox_pred, rpn_bbox_targets, rpn_bbox_inside_weights,
133                                                             rpn_bbox_outside_weights, sigma=3, dim=[1,2,3])
134 
135         return rois, self.rpn_loss_cls, self.rpn_loss_box

2 generate_anchors.py

这一部分比较简单，就是把几种尺度几种比例（这里是3种）的anchor合起来用anchors来存储所有的anchor。
详细注释如下：

  1 # Verify that we compute the same anchors as Shaoqing's matlab implementation:
  2 #
  3 #    >> load output/rpn_cachedir/faster_rcnn_VOC2007_ZF_stage1_rpn/anchors.mat
  4 #    >> anchors
  5 #
  6 #    anchors =     %9种anchor
  7 #
  8 #       -83   -39   100    56
  9 #      -175   -87   192   104
 10 #      -359  -183   376   200
 11 #       -55   -55    72    72
 12 #      -119  -119   136   136
 13 #      -247  -247   264   264
 14 #       -35   -79    52    96
 15 #       -79  -167    96   184
 16 #      -167  -343   184   360
 17 
 18 #array([[ -83.,  -39.,  100.,   56.],
 19 #       [-175.,  -87.,  192.,  104.],
 20 #       [-359., -183.,  376.,  200.],
 21 #       [ -55.,  -55.,   72.,   72.],
 22 #       [-119., -119.,  136.,  136.],
 23 #       [-247., -247.,  264.,  264.],
 24 #       [ -35.,  -79.,   52.,   96.],
 25 #       [ -79., -167.,   96.,  184.],
 26 #       [-167., -343.,  184.,  360.]])
 27 
 28 try:
 29     xrange          # Python 2
 30 except NameError:
 31     xrange = range  # Python 3
 32 
 33 
 34 def generate_anchors(base_size=16, ratios=[0.5, 1, 2],
 35                      scales=2**np.arange(3, 6)):#arange函数用于创建等差数组3 4 5 
 36     """
 37     Generate anchor (reference) windows by enumerating aspect ratios X
 38     scales wrt a reference (0, 0, 15, 15) window.
 39     """
 40 
 41     base_anchor = np.array([1, 1, base_size, base_size]) - 1
 42     ratio_anchors = _ratio_enum(base_anchor, ratios)
 43     #vstack(tup) ，参数tup可以是元组，列表，或者numpy数组，返回结果为numpy的数组，
 44     #就是横着排起来
 45     anchors = np.vstack([_scale_enum(ratio_anchors[i, :], scales)
 46                          for i in xrange(ratio_anchors.shape[0])])
 47     return anchors
 48 
 49 #Return width, height, x center, and y center for an anchor (window).
 50 #得到anchor宽 高 中点坐标
 51 def _whctrs(anchor):
 52     """
 53     Return width, height, x center, and y center for an anchor (window).
 54     """
 55 
 56     w = anchor[2] - anchor[0] + 1
 57     h = anchor[3] - anchor[1] + 1
 58     x_ctr = anchor[0] + 0.5 * (w - 1)
 59     y_ctr = anchor[1] + 0.5 * (h - 1)
 60     return w, h, x_ctr, y_ctr
 61 
 62 #把给的anchor合在一起，按列排
 63 def _mkanchors(ws, hs, x_ctr, y_ctr):
 64     """
 65     Given a vector of widths (ws) and heights (hs) around a center
 66     (x_ctr, y_ctr), output a set of anchors (windows).
 67     """
 68 
 69     ws = ws[:, np.newaxis]#np.newaxis 在使用和功能上等价于 None，其实就是 None 的一个别名。
 70     hs = hs[:, np.newaxis]
 71     anchors = np.hstack((x_ctr - 0.5 * (ws - 1),
 72                          y_ctr - 0.5 * (hs - 1),
 73                          x_ctr + 0.5 * (ws - 1),
 74                          y_ctr + 0.5 * (hs - 1)))
 75     return anchors
 76 
 77 #每个比例下有一组anchor
 78 def _ratio_enum(anchor, ratios):
 79     """
 80     Enumerate a set of anchors for each aspect ratio wrt an anchor.
 81     """
 82 
 83     w, h, x_ctr, y_ctr = _whctrs(anchor)#上面定义的函数 得到anchor的宽高中心
 84     size = w * h
 85     size_ratios = size / ratios#该比例下anchor的大小
 86     ws = np.round(np.sqrt(size_ratios))
 87     hs = np.round(ws * ratios)
 88     anchors = _mkanchors(ws, hs, x_ctr, y_ctr)#把这个比例下的anchor保留下来
 89     return anchors
 90 
 91 #每个尺度下有一组anchor
 92 def _scale_enum(anchor, scales):
 93     """
 94     Enumerate a set of anchors for each scale wrt an anchor.
 95     """
 96 
 97     w, h, x_ctr, y_ctr = _whctrs(anchor)
 98     ws = w * scales
 99     hs = h * scales
100     anchors = _mkanchors(ws, hs, x_ctr, y_ctr)#把这个比例下的anchor保留下来
101     return anchors
102 
103 if __name__ == '__main__':
104     import time
105     t = time.time()
106     a = generate_anchors()#得到的anchor
107     print(time.time() - t)
108     print(a)
109     from IPython import embed; embed()

3 proposal_layer.py

根据anchor得到候选区域，这里有NMS，在后面再介绍。详细注释如下：

  1 #通过将估计的边界框变换应用于一组常规框（称为“锚点”）来输出对象检测候选区域。
  2 class _ProposalLayer(nn.Module):
  3     """
  4     Outputs object detection proposals by applying estimated bounding-box
  5     transformations to a set of regular boxes (called "anchors").
  6     """
  7     #参数是 特征步长 尺度 比例
  8     def __init__(self, feat_stride, scales, ratios):
  9         super(_ProposalLayer, self).__init__()
 10         #得到特征步长
 11         self._feat_stride = feat_stride
 12         #得到所有的anchor
 13         self._anchors = torch.from_numpy(generate_anchors(scales=np.array(scales), 
 14             ratios=np.array(ratios))).float()
 15         #anchors的行数就是所有anchor的个数
 16         self._num_anchors = self._anchors.size(0)
 17 
 18         # rois blob: holds R regions of interest, each is a 5-tuple #一个索引和四个矩形参数
 19         # (n, x1, y1, x2, y2) specifying an image batch index n and a
 20         # rectangle (x1, y1, x2, y2)
 21         # top[0].reshape(1, 5)
 22         #
 23         # # scores blob: holds scores for R regions of interest
 24         # if len(top) > 1:
 25         #     top[1].reshape(1, 1, 1, 1)
 26 
 27     def forward(self, input):
 28 
 29         # Algorithm:
 30         #
 31         # for each (H, W) location i
 32         #   generate A anchor boxes centered on cell i
 33         #   apply predicted bbox deltas at cell i to each of the A anchors
 34         # clip predicted boxes to image
 35         # remove predicted boxes with either height or width < threshold
 36         # sort all (proposal, score) pairs by score from highest to lowest
 37         # take top pre_nms_topN proposals before NMS
 38         # apply NMS with threshold 0.7 to remaining proposals
 39         # take after_nms_topN proposals after NMS
 40         # return the top proposals (-> RoIs top, scores top)
 41         #在NMS后得到最佳的
 42 
 43 
 44         # the first set of _num_anchors channels are bg probs
 45         #_num_anchors通道的第一组是背景概率
 46         # the second set are the fg probs
 47         #第二组是前景概率
 48         scores = input[0][:, self._num_anchors:, :, :]#分类概率
 49         bbox_deltas = input[1]#偏移
 50         im_info = input[2]#图像信息
 51         cfg_key = input[3]#是training还是test
 52 
 53         #设置一些参数
 54         pre_nms_topN  = cfg[cfg_key].RPN_PRE_NMS_TOP_N
 55         post_nms_topN = cfg[cfg_key].RPN_POST_NMS_TOP_N
 56         nms_thresh    = cfg[cfg_key].RPN_NMS_THRESH
 57         min_size      = cfg[cfg_key].RPN_MIN_SIZE
 58 
 59         #批尺寸
 60         batch_size = bbox_deltas.size(0)
 61         #下面是在原图上生成anchor
 62         feat_height, feat_width = scores.size(2), scores.size(3)
 63         shift_x = np.arange(0, feat_width) * self._feat_stride#shape: [width,] 特征图相对于原图的偏移
 64         shift_y = np.arange(0, feat_height) * self._feat_stride#shape: [height,]
 65         shift_x, shift_y = np.meshgrid(shift_x, shift_y) #生成网格 shift_x shape: [height, width], shift_y shape: [height, width]
 66         shifts = torch.from_numpy(np.vstack((shift_x.ravel(), shift_y.ravel(),
 67                                   shift_x.ravel(), shift_y.ravel())).transpose()) #shape[height*width, 4]
 68         shifts = shifts.contiguous().type_as(scores).float()
 69 
 70         A = self._num_anchors
 71         K = shifts.size(0)
 72 
 73         self._anchors = self._anchors.type_as(scores)
 74         # anchors = self._anchors.view(1, A, 4) + shifts.view(1, K, 4).permute(1, 0, 2).contiguous()
 75         anchors = self._anchors.view(1, A, 4) + shifts.view(K, 1, 4)
 76         anchors = anchors.view(1, K * A, 4).expand(batch_size, K * A, 4)
 77 
 78         # Transpose and reshape predicted bbox transformations to get them
 79         # into the same order as the anchors:
 80         #转置和重塑预测的bbox转换，使它们与锚点的顺序相同：
 81 
 82         bbox_deltas = bbox_deltas.permute(0, 2, 3, 1).contiguous()
 83         bbox_deltas = bbox_deltas.view(batch_size, -1, 4)
 84 
 85         # Same story for the scores:
 86         scores = scores.permute(0, 2, 3, 1).contiguous()
 87         scores = scores.view(batch_size, -1)
 88 
 89         # Convert anchors into proposals via bbox transformations
 90         #通过bbox转换将锚点转换为候选区域
 91         proposals = bbox_transform_inv(anchors, bbox_deltas, batch_size)
 92 
 93         # 2. clip predicted boxes to image
 94         #裁剪预测框到图像
 95         #严格限制proposal的四个角在图像边界内
 96         proposals = clip_boxes(proposals, im_info, batch_size)
 97         # proposals = clip_boxes_batch(proposals, im_info, batch_size)
 98 
 99         # assign the score to 0 if it's non keep.
100         # keep = self._filter_boxes(proposals, min_size * im_info[:, 2])
101 
102         # trim keep index to make it euqal over batch
103         # keep_idx = torch.cat(tuple(keep_idx), 0)
104 
105         # scores_keep = scores.view(-1)[keep_idx].view(batch_size, trim_size)
106         # proposals_keep = proposals.view(-1, 4)[keep_idx, :].contiguous().view(batch_size, trim_size, 4)
107         
108         # _, order = torch.sort(scores_keep, 1, True)
109         
110         scores_keep = scores
111         proposals_keep = proposals
112         _, order = torch.sort(scores_keep, 1, True)
113 
114         output = scores.new(batch_size, post_nms_topN, 5).zero_()
115         for i in range(batch_size):
116             # # 3. remove predicted boxes with either height or width < threshold
117             # # (NOTE: convert min_size to input image scale stored in im_info[2])
118             #删除高度或宽度<阈值的预测框（注意：将min_size转换为存储在im_info [2]中的输入图像比例）
119             proposals_single = proposals_keep[i]
120             scores_single = scores_keep[i]
121 
122             # # 4. sort all (proposal, score) pairs by score from highest to lowest
123             #按分数从最高到最低排序所有（h候选区域，得分）对
124             # # 5. take top pre_nms_topN (e.g. 6000)
125             #取顶部pre_nms_topN
126             order_single = order[i]
127 
128             if pre_nms_topN > 0 and pre_nms_topN < scores_keep.numel():
129                 order_single = order_single[:pre_nms_topN]
130 
131             proposals_single = proposals_single[order_single, :]
132             scores_single = scores_single[order_single].view(-1,1)
133 
134             # 6. apply nms (e.g. threshold = 0.7)
135             # 7. take after_nms_topN (e.g. 300)
136             # 8. return the top proposals (-> RoIs top)
137 
138             keep_idx_i = nms(torch.cat((proposals_single, scores_single), 1), nms_thresh, force_cpu=not cfg.USE_GPU_NMS)
139             keep_idx_i = keep_idx_i.long().view(-1)
140 
141             if post_nms_topN > 0:
142                 keep_idx_i = keep_idx_i[:post_nms_topN]
143             proposals_single = proposals_single[keep_idx_i, :]
144             scores_single = scores_single[keep_idx_i, :]
145 
146             # padding 0 at the end.
147             num_proposal = proposals_single.size(0)
148             output[i,:,0] = i
149             output[i,:num_proposal,1:] = proposals_single
150 
151         return output
152 
153     def backward(self, top, propagate_down, bottom):
154         """This layer does not propagate gradients."""
155         pass
156 
157     def reshape(self, bottom, top):
158         """Reshaping happens during the call to forward."""
159         pass
160 
161     #删除任何小于min_size的边框
162     def _filter_boxes(self, boxes, min_size):
163         """Remove all boxes with any side smaller than min_size."""
164         ws = boxes[:, :, 2] - boxes[:, :, 0] + 1
165         hs = boxes[:, :, 3] - boxes[:, :, 1] + 1
166         #expand_as(ws) 将tensor扩展为参数ws的大小
167         keep = ((ws >= min_size.view(-1,1).expand_as(ws)) & (hs >= min_size.view(-1,1).expand_as(hs)))
168         return keep

4 bbox_transform.py

就是一些变换，注释如下：

  1 #在计算anchor的坐标变换值的时候，使用到了bbox_transform函数，
  2 #注意在计算坐标变换的时候是将anchor的表示形式变成中心坐标与长宽
  3 def bbox_transform(ex_rois, gt_rois):
  4     ex_widths = ex_rois[:, 2] - ex_rois[:, 0] + 1.0
  5     ex_heights = ex_rois[:, 3] - ex_rois[:, 1] + 1.0
  6     ex_ctr_x = ex_rois[:, 0] + 0.5 * ex_widths
  7     ex_ctr_y = ex_rois[:, 1] + 0.5 * ex_heights#计算得到每个anchor的中心坐标和长宽
  8 
  9     gt_widths = gt_rois[:, 2] - gt_rois[:, 0] + 1.0
 10     gt_heights = gt_rois[:, 3] - gt_rois[:, 1] + 1.0
 11     gt_ctr_x = gt_rois[:, 0] + 0.5 * gt_widths
 12     gt_ctr_y = gt_rois[:, 1] + 0.5 * gt_heights#计算每个anchor对应的ground truth box对应的中心坐标和长宽
 13 
 14     targets_dx = (gt_ctr_x - ex_ctr_x) / ex_widths#计算四个坐标变换值
 15     targets_dy = (gt_ctr_y - ex_ctr_y) / ex_heights
 16     targets_dw = torch.log(gt_widths / ex_widths)
 17     targets_dh = torch.log(gt_heights / ex_heights)
 18 
 19     targets = torch.stack(
 20         (targets_dx, targets_dy, targets_dw, targets_dh),1)#对于每一个anchor，得到四个关系值 shape: [4, num_anchor]
 21 
 22     return targets
 23 
 24 def bbox_transform_batch(ex_rois, gt_rois):
 25 
 26     if ex_rois.dim() == 2:
 27         ex_widths = ex_rois[:, 2] - ex_rois[:, 0] + 1.0
 28         ex_heights = ex_rois[:, 3] - ex_rois[:, 1] + 1.0
 29         ex_ctr_x = ex_rois[:, 0] + 0.5 * ex_widths
 30         ex_ctr_y = ex_rois[:, 1] + 0.5 * ex_heights
 31 
 32         gt_widths = gt_rois[:, :, 2] - gt_rois[:, :, 0] + 1.0
 33         gt_heights = gt_rois[:, :, 3] - gt_rois[:, :, 1] + 1.0
 34         gt_ctr_x = gt_rois[:, :, 0] + 0.5 * gt_widths
 35         gt_ctr_y = gt_rois[:, :, 1] + 0.5 * gt_heights
 36 
 37         targets_dx = (gt_ctr_x - ex_ctr_x.view(1,-1).expand_as(gt_ctr_x)) / ex_widths
 38         targets_dy = (gt_ctr_y - ex_ctr_y.view(1,-1).expand_as(gt_ctr_y)) / ex_heights
 39         targets_dw = torch.log(gt_widths / ex_widths.view(1,-1).expand_as(gt_widths))
 40         targets_dh = torch.log(gt_heights / ex_heights.view(1,-1).expand_as(gt_heights))
 41 
 42     elif ex_rois.dim() == 3:
 43         ex_widths = ex_rois[:, :, 2] - ex_rois[:, :, 0] + 1.0
 44         ex_heights = ex_rois[:,:, 3] - ex_rois[:,:, 1] + 1.0
 45         ex_ctr_x = ex_rois[:, :, 0] + 0.5 * ex_widths
 46         ex_ctr_y = ex_rois[:, :, 1] + 0.5 * ex_heights
 47 
 48         gt_widths = gt_rois[:, :, 2] - gt_rois[:, :, 0] + 1.0
 49         gt_heights = gt_rois[:, :, 3] - gt_rois[:, :, 1] + 1.0
 50         gt_ctr_x = gt_rois[:, :, 0] + 0.5 * gt_widths
 51         gt_ctr_y = gt_rois[:, :, 1] + 0.5 * gt_heights
 52 
 53         targets_dx = (gt_ctr_x - ex_ctr_x) / ex_widths
 54         targets_dy = (gt_ctr_y - ex_ctr_y) / ex_heights
 55         targets_dw = torch.log(gt_widths / ex_widths)
 56         targets_dh = torch.log(gt_heights / ex_heights)
 57     else:
 58         raise ValueError('ex_roi input dimension is not correct.')
 59 
 60     targets = torch.stack(
 61         (targets_dx, targets_dy, targets_dw, targets_dh),2)
 62 
 63     return targets
 64 
 65 #bbox_transform_inv函数结合RPN的输出对所有初始框进行了坐标变换
 66 def bbox_transform_inv(boxes, deltas, batch_size):
 67 
 68     ##获得初始proposal的中心和长宽信息
 69     widths = boxes[:, :, 2] - boxes[:, :, 0] + 1.0
 70     heights = boxes[:, :, 3] - boxes[:, :, 1] + 1.0
 71     ctr_x = boxes[:, :, 0] + 0.5 * widths
 72     ctr_y = boxes[:, :, 1] + 0.5 * heights
 73     
 74     ##获得坐标变换信息
 75     dx = deltas[:, :, 0::4]
 76     dy = deltas[:, :, 1::4]
 77     dw = deltas[:, :, 2::4]
 78     dh = deltas[:, :, 3::4]
 79 
 80     # #得到改变后的proposal的中心和长宽信息
 81     pred_ctr_x = dx * widths.unsqueeze(2) + ctr_x.unsqueeze(2)
 82     pred_ctr_y = dy * heights.unsqueeze(2) + ctr_y.unsqueeze(2)
 83     pred_w = torch.exp(dw) * widths.unsqueeze(2)
 84     pred_h = torch.exp(dh) * heights.unsqueeze(2)
 85 
 86     #将改变后的proposal的中心和长宽信息还原成左上角和右下角的版本
 87     pred_boxes = deltas.clone()
 88     # x1
 89     pred_boxes[:, :, 0::4] = pred_ctr_x - 0.5 * pred_w
 90     # y1
 91     pred_boxes[:, :, 1::4] = pred_ctr_y - 0.5 * pred_h
 92     # x2
 93     pred_boxes[:, :, 2::4] = pred_ctr_x + 0.5 * pred_w
 94     # y2
 95     pred_boxes[:, :, 3::4] = pred_ctr_y + 0.5 * pred_h
 96 
 97     return pred_boxes
 98 
 99 def clip_boxes_batch(boxes, im_shape, batch_size):
100     """
101     Clip boxes to image boundaries.
102     """
103     num_rois = boxes.size(1)
104 
105     boxes[boxes < 0] = 0
106     # batch_x = (im_shape[:,0]-1).view(batch_size, 1).expand(batch_size, num_rois)
107     # batch_y = (im_shape[:,1]-1).view(batch_size, 1).expand(batch_size, num_rois)
108 
109     batch_x = im_shape[:, 1] - 1
110     batch_y = im_shape[:, 0] - 1
111 
112     boxes[:,:,0][boxes[:,:,0] > batch_x] = batch_x
113     boxes[:,:,1][boxes[:,:,1] > batch_y] = batch_y
114     boxes[:,:,2][boxes[:,:,2] > batch_x] = batch_x
115     boxes[:,:,3][boxes[:,:,3] > batch_y] = batch_y
116 
117     return boxes
118 
119 #严格限制proposal的四个角在图像边界内
120 def clip_boxes(boxes, im_shape, batch_size):
121 
122     for i in range(batch_size):
123         boxes[i,:,0::4].clamp_(0, im_shape[i, 1]-1)
124         boxes[i,:,1::4].clamp_(0, im_shape[i, 0]-1)
125         boxes[i,:,2::4].clamp_(0, im_shape[i, 1]-1)
126         boxes[i,:,3::4].clamp_(0, im_shape[i, 0]-1)
127 
128     return boxes
129 
130 
131 ##计算重合程度，两个框之间的重合区域的面积 / 两个区域一共加起来的面
132 def bbox_overlaps(anchors, gt_boxes):
133     """
134     anchors: (N, 4) ndarray of float
135     gt_boxes: (K, 4) ndarray of float
136 
137     overlaps: (N, K) ndarray of overlap between boxes and query_boxes
138     """
139     N = anchors.size(0)
140     K = gt_boxes.size(0)
141 
142     gt_boxes_area = ((gt_boxes[:,2] - gt_boxes[:,0] + 1) *
143                 (gt_boxes[:,3] - gt_boxes[:,1] + 1)).view(1, K)
144 
145     anchors_area = ((anchors[:,2] - anchors[:,0] + 1) *
146                 (anchors[:,3] - anchors[:,1] + 1)).view(N, 1)
147 
148     boxes = anchors.view(N, 1, 4).expand(N, K, 4)
149     query_boxes = gt_boxes.view(1, K, 4).expand(N, K, 4)
150 
151     iw = (torch.min(boxes[:,:,2], query_boxes[:,:,2]) -
152         torch.max(boxes[:,:,0], query_boxes[:,:,0]) + 1)
153     iw[iw < 0] = 0
154 
155     ih = (torch.min(boxes[:,:,3], query_boxes[:,:,3]) -
156         torch.max(boxes[:,:,1], query_boxes[:,:,1]) + 1)
157     ih[ih < 0] = 0
158 
159     ua = anchors_area + gt_boxes_area - (iw * ih)
160     overlaps = iw * ih / ua
161 
162     return overlaps
163 
164 def bbox_overlaps_batch(anchors, gt_boxes):
165     """
166     anchors: (N, 4) ndarray of float
167     gt_boxes: (b, K, 5) ndarray of float
168 
169     overlaps: (N, K) ndarray of overlap between boxes and query_boxes
170     """
171     batch_size = gt_boxes.size(0)
172 
173 
174     if anchors.dim() == 2:
175 
176         N = anchors.size(0)
177         K = gt_boxes.size(1)
178 
179         anchors = anchors.view(1, N, 4).expand(batch_size, N, 4).contiguous()
180         gt_boxes = gt_boxes[:,:,:4].contiguous()
181 
182 
183         gt_boxes_x = (gt_boxes[:,:,2] - gt_boxes[:,:,0] + 1)
184         gt_boxes_y = (gt_boxes[:,:,3] - gt_boxes[:,:,1] + 1)
185         gt_boxes_area = (gt_boxes_x * gt_boxes_y).view(batch_size, 1, K)
186 
187         anchors_boxes_x = (anchors[:,:,2] - anchors[:,:,0] + 1)
188         anchors_boxes_y = (anchors[:,:,3] - anchors[:,:,1] + 1)
189         anchors_area = (anchors_boxes_x * anchors_boxes_y).view(batch_size, N, 1)
190 
191         gt_area_zero = (gt_boxes_x == 1) & (gt_boxes_y == 1)
192         anchors_area_zero = (anchors_boxes_x == 1) & (anchors_boxes_y == 1)
193 
194         boxes = anchors.view(batch_size, N, 1, 4).expand(batch_size, N, K, 4)
195         query_boxes = gt_boxes.view(batch_size, 1, K, 4).expand(batch_size, N, K, 4)
196 
197         iw = (torch.min(boxes[:,:,:,2], query_boxes[:,:,:,2]) -
198             torch.max(boxes[:,:,:,0], query_boxes[:,:,:,0]) + 1)
199         iw[iw < 0] = 0
200 
201         ih = (torch.min(boxes[:,:,:,3], query_boxes[:,:,:,3]) -
202             torch.max(boxes[:,:,:,1], query_boxes[:,:,:,1]) + 1)
203         ih[ih < 0] = 0
204         ua = anchors_area + gt_boxes_area - (iw * ih)
205         overlaps = iw * ih / ua
206 
207         # mask the overlap here.
208         overlaps.masked_fill_(gt_area_zero.view(batch_size, 1, K).expand(batch_size, N, K), 0)
209         overlaps.masked_fill_(anchors_area_zero.view(batch_size, N, 1).expand(batch_size, N, K), -1)
210 
211     elif anchors.dim() == 3:
212         N = anchors.size(1)
213         K = gt_boxes.size(1)
214 
215         if anchors.size(2) == 4:
216             anchors = anchors[:,:,:4].contiguous()
217         else:
218             anchors = anchors[:,:,1:5].contiguous()
219 
220         gt_boxes = gt_boxes[:,:,:4].contiguous()
221 
222         gt_boxes_x = (gt_boxes[:,:,2] - gt_boxes[:,:,0] + 1)
223         gt_boxes_y = (gt_boxes[:,:,3] - gt_boxes[:,:,1] + 1)
224         gt_boxes_area = (gt_boxes_x * gt_boxes_y).view(batch_size, 1, K)
225 
226         anchors_boxes_x = (anchors[:,:,2] - anchors[:,:,0] + 1)
227         anchors_boxes_y = (anchors[:,:,3] - anchors[:,:,1] + 1)
228         anchors_area = (anchors_boxes_x * anchors_boxes_y).view(batch_size, N, 1)
229 
230         gt_area_zero = (gt_boxes_x == 1) & (gt_boxes_y == 1)
231         anchors_area_zero = (anchors_boxes_x == 1) & (anchors_boxes_y == 1)
232 
233         boxes = anchors.view(batch_size, N, 1, 4).expand(batch_size, N, K, 4)
234         query_boxes = gt_boxes.view(batch_size, 1, K, 4).expand(batch_size, N, K, 4)
235 
236         iw = (torch.min(boxes[:,:,:,2], query_boxes[:,:,:,2]) -
237             torch.max(boxes[:,:,:,0], query_boxes[:,:,:,0]) + 1)
238         iw[iw < 0] = 0
239 
240         ih = (torch.min(boxes[:,:,:,3], query_boxes[:,:,:,3]) -
241             torch.max(boxes[:,:,:,1], query_boxes[:,:,:,1]) + 1)
242         ih[ih < 0] = 0
243         ua = anchors_area + gt_boxes_area - (iw * ih)
244 
245         overlaps = iw * ih / ua
246 
247         # mask the overlap here.
248         overlaps.masked_fill_(gt_area_zero.view(batch_size, 1, K).expand(batch_size, N, K), 0)
249         overlaps.masked_fill_(anchors_area_zero.view(batch_size, N, 1).expand(batch_size, N, K), -1)
250     else:
251         raise ValueError('anchors input dimension is not correct.')
252 
253     return overlaps