手势识别控制pygame精灵

步骤：

1. 编写简易pygame精灵游戏（只实现键盘上下左右控制） 
2. 解决opencv手势识别核心问题
3. 上述2部分对接上

 

 pygame部分我们只加载个背景，然后里面放1只乌龟精灵，用键盘的上下左右键来控制，直接给出代码：

乌龟精灵代码（DemoSpirit.py）：

?

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31

import pygame     class DemoSpirit(pygame.sprite.Sprite):     def __init__(self, target, screen_size, position):         pygame.sprite.Sprite.__init__(self)         self.target_surface = target         self.screen_size = screen_size         self.position = position         self.image = pygame.image.load("resources\\wugui.png").convert_alpha()         self.image = pygame.transform.smoothscale(self.image, (50, 50))       def draw(self):         # random_text = font_200.render('***', True, white_color)         self.target_surface.blit(self.image, self.position)       def move_left(self):         if self.position[0]-10 > 0:             self.position=(self.position[0]-10, self.position[1])       def move_right(self):         if self.position[0]+10 < self.screen_size[0]:             self.position=(self.position[0]+10, self.position[1])       def move_up(self):         if self.position[1] - 10 > 0:             self.position=(self.position[0], self.position[1]-10)       def move_down(self):         if self.position[1] + 10 < self.screen_size[1]:             self.position=(self.position[0], self.position[1]+10)

　　

游戏主循环代码(game-main.py)：

 

?

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39

import pygame from pygame.locals import *   background_image_filename = 'resources/back.jpg'   pygame.init()  # 2、初始化init() 及设置 screen_list = pygame.display.list_modes() screen_size = screen_list[16] screen = pygame.display.set_mode(screen_size) background = pygame.image.load(background_image_filename).convert() background = pygame.transform.scale(background, screen_size)   clock = pygame.time.Clock()   pos = (screen_size[0] * 0.6, screen_size[1] * 0.3)   from cvgame.DemoSpirit import DemoSpirit   s1 = DemoSpirit(screen, screen_size, pos)   # 开始游戏循环 while True:     for event in pygame.event.get():         if event.type == KEYDOWN:             if event.key == K_UP:                 s1.move_up()             if event.key == K_DOWN:                 s1.move_down()             if event.key == K_LEFT:                 s1.move_left()             if event.key == K_RIGHT:                 s1.move_right()             elif event.key == K_q:                 exit()     screen.blit(background, (0, 0))     s1.draw()       pygame.display.update()  # 6、update 更新屏幕显示     clock.tick(100)

 

 效果图：

 

 

接下来，进入手势识别领域

我们是做了个小技巧来侧面绕过手跟踪问题，如下图（直接指定了左手右手监控区域，这样就不需要动态跟踪手的rect了）：

 

 

 

?

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15

while success:     success, img = cap.read()     frame = imutils.resize(img, width=700)       cv2.rectangle(frame, (50, 0), (264, 250), (170, 170, 0))                          #左手区域     cv2.rectangle(frame, (426, 0), (640, 250), (170, 170, 0))                         #右手区域       cv2.imshow("Frame_Original", frame)                                               #显示       rightHand = frame[0:250, 50:264]     leftHand = frame[0:210, 426:640]       left_hand_event = grdetect(leftHand, fgbg_left, verbose=True)                     #检测左手手势识别事件     right_hand_event = grdetect(rightHand, fgbg_right, verbose=True)                  #检测右手手势识别事件     print('left hand: ', left_hand_event, 'right hand: ', right_hand_event)           #打印出来检测结果

　　

主要看看grdetect和fgbg_left/fgbg_right：

 

 看上图，除了手之外，还有个大背景，首先得把背景去掉，才能识别出前景色-手，fgbg_left/fgbg_right其实就是用来干着活的，分别为左手、右手的背景减噪用的

?

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23

fgbg_left = cv2.createBackgroundSubtractorMOG2() fgbg_right = cv2.createBackgroundSubtractorMOG2()   def train_bg(fgbg, roi):     fgbg.apply(roi)     def start():     global fgbg_left     global fgbg_right     trainingBackgroundCount = 200                           #200次来训练背景减噪训练     while trainingBackgroundCount>0:         success, img = cap.read()         frame = imutils.resize(img, width=700)         cv2.imshow("Frame_Original", frame)         rightHand = frame[0:250, 50:264]         leftHand = frame[0:210, 426:640]           train_bg(fgbg_left, leftHand)                      #训练左手区域         train_bg(fgbg_right, rightHand)                    #训练右手区域           key = cv2.waitKey(1) & 0xFF         trainingBackgroundCount -= 1

 

再来看看核心函数

?

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24

def grdetect(array, fgbg, verbose=False):     event = {'type': 'none'}     copy = array.copy()     array = _remove_background(array, fgbg)  #移除背景，会用到背景减噪（上述提到）     thresh = _bodyskin_detetc(array)         #高斯+二值化     contours = _get_contours(thresh.copy())  #计算图像的轮廓点，可能会返回多个轮廓     largecont = max(contours, key=lambda contour: cv2.contourArea(contour))   #选择面积最大的轮廓     hull = cv2.convexHull(largecont, returnPoints=False)                      #根据轮廓点计算凸点     defects = cv2.convexityDefects(largecont, hull)                           #计算轮廓的凹点（凸缺陷）     if defects is not None:         # 利用凹陷点坐标, 根据余弦定理计算图像中锐角个数         copy, ndefects = _get_defects_count(copy, largecont, defects, verbose=verbose)         # 根据锐角个数判断手势, 会有一定的误差         if ndefects == 0:             event['type'] = '0'         elif ndefects == 1:             event['type'] = '2'         elif ndefects == 2:             event['type'] = '3'         elif ndefects == 3:             event['type'] = '4'         elif ndefects == 4:             event['type'] = '5'     return event

　　

剩下的就是上述的支持函数了

?

1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60

def _remove_background(frame, fgbg):     fgmask = fgbg.apply(frame, learningRate=0)                       #learningRate=0代表不更新背景噪声，也就是不学习     kernel = np.ones((3, 3), np.uint8)     fgmask = cv2.erode(fgmask, kernel, iterations=1)     res = cv2.bitwise_and(frame, frame, mask=fgmask)     return res   def _bodyskin_detetc(frame):     # 肤色检测: YCrCb之Cr分量 + OTSU二值化     ycrcb = cv2.cvtColor(frame, cv2.COLOR_BGR2YCrCb)  # 分解为YUV图像,得到CR分量     (_, cr, _) = cv2.split(ycrcb)     cr1 = cv2.GaussianBlur(cr, (5, 5), 0)  # 高斯滤波     _, skin = cv2.threshold(cr1, 0, 255, cv2.THRESH_BINARY + cv2.THRESH_OTSU)  # OTSU图像二值化     return skin     # 检测图像中的凸点(手指)个数 def _get_contours(array):     # 利用findContours检测图像中的轮廓, 其中返回值contours包含了图像中所有轮廓的坐标点     contours, _ = cv2.findContours(array, cv2.RETR_TREE, cv2.CHAIN_APPROX_NONE)     return contours     _COLOR_RED = (0, 0, 255)     def _get_eucledian_distance(beg, end):  # 计算两点之间的坐标     i = str(beg).split(',')     j = i[0].split('(')     x1 = int(j[1])     k = i[1].split(')')     y1 = int(k[0])     i = str(end).split(',')     j = i[0].split('(')     x2 = int(j[1])     k = i[1].split(')')     y2 = int(k[0])     d = math.sqrt((x1 - x2) * (x1 - x2) + (y1 - y2) * (y1 - y2))     return d     # 根据图像中凹凸点中的 (开始点, 结束点, 远点)的坐标, 利用余弦定理计算两根手指之间的夹角, 其必为锐角, 根据锐角的个数判别手势. def _get_defects_count(array, contour, defects, verbose=False):     ndefects = 0     for i in range(defects.shape[0]):         s, e, f, _ = defects[i, 0]         beg = tuple(contour[s][0])         end = tuple(contour[e][0])         far = tuple(contour[f][0])         a = _get_eucledian_distance(beg, end)         b = _get_eucledian_distance(beg, far)         c = _get_eucledian_distance(end, far)         angle = math.acos((b ** 2 + c ** 2 - a ** 2) / (2 * b * c))  # * 57         if angle <= math.pi / 2:  # 90:             ndefects = ndefects + 1             if verbose:                 cv2.circle(array, far, 3, _COLOR_RED, -1)         if verbose:             cv2.line(array, beg, end, _COLOR_RED, 1)     return array, ndefects