人脸对齐过程实现 c++和python
c++ 实现版本：
1 人脸检测
1.1 使用mtcnn-ncnn进行人脸检测,会输出face box和landmark5
face box: [x1,y1,x2,y2]
landmark5: [left_eye,right_eye, nose, month_left, month_right]
2 图像变换(使用Egien矩阵库)
2.1 下载Egien
2.2 创建一个项目, 引用Egien,
2.3 使用SVD分解, 计算变换矩阵,
2.4 进行图像的仿射变换
=================================================
代码如下：
#pragma once
#include<Dense>
#include"opencv_base.h"
using namespace std;
using namespace Eigen;
/*
使用五点人脸对齐
left_eye, right_eye, nose, left_month, right_month
*/
static Mat face_align_bypoint5(Mat src_frame, vector<Point2d> landmark5, vector<Point2d> box) {
MatrixXf src_landmark_mtx(5, 2);
int i = 0;
for (size_t r = 0; r < landmark5.size(); r++)
{
src_landmark_mtx(r, 0) = landmark5.at(r).x;
src_landmark_mtx(r, 1) = landmark5.at(r).y;
i++;
}
Matrix<float, 5, 2> dst_landmark_mtx;
dst_landmark_mtx << 30.2946, 51.6963,
65.5318, 51.6963,
48.0252, 71.7366,
33.5493, 92.3655,
62.7299, 92.3655;
int rows = src_landmark_mtx.rows();
int cols = src_landmark_mtx.cols();
MatrixXf mean1 = src_landmark_mtx.colwise().mean(); // [1,2]
MatrixXf mean2 = dst_landmark_mtx.colwise().mean(); // [1,2]
float col_mean_1 = mean1(0, 0); // 列的均值
float col_mean_2 = mean2(0, 0);
//cout << src_landmark_mtx << endl;
//cout << dst_landmark_mtx << endl;
// std = sqrt(mean(abs(x - x.mean())**2))
Matrix<float, 5, 2> m1, m2;
for (size_t r = 0; r < rows; r++)
{
for (size_t c = 0; c < cols; c++)
{
src_landmark_mtx(r, c) -= mean1(0, c);
dst_landmark_mtx(r, c) -= mean2(0, c);
auto abs_v1 = std::abs(src_landmark_mtx(r, c));
auto abs_v2 = std::abs(dst_landmark_mtx(r, c));
m1(r, c) = pow(abs_v1, 2);
m2(r, c) = pow(abs_v2, 2);
}
}
float std1 = sqrt(m1.mean());
float std2 = sqrt(m2.mean());
src_landmark_mtx /= std1;
dst_landmark_mtx /= std2;
//printf_s(" ============= std norm =============\n");
//cout << src_landmark_mtx << endl;
//cout << dst_landmark_mtx << endl;
MatrixXf m = src_landmark_mtx.transpose() * dst_landmark_mtx; // [2,2]
JacobiSVD<MatrixXf> svd(m, ComputeThinU | ComputeThinV);
MatrixXf VT = svd.matrixV().transpose();
MatrixXf U = svd.matrixU();
MatrixXf A = svd.singularValues();
MatrixXf R = (U*VT).transpose(); // [2,2]
//printf_s(" ============= svd =============\n");
//cout << VT << endl;
//cout << U << endl;
//cout << A << endl;
//cout << R << endl;
MatrixXf M(3, 3);
M.row(2) << 0., 0., 1.;
MatrixXf t1 = (std1 / std2)*R; // [2,2]
MatrixXf t2 = mean2.transpose() - (std1 / std2)*R*mean1.transpose(); // [2,1]
M.row(0) << t1(0, 0), t1(0, 1), t2(0, 0);
M.row(1) << t1(1, 0), t1(1, 1), t2(1, 0);
//printf_s(" ============= transformer =============\n");
//cout << M << endl;
//printf_s(" ============= face transformer =============\n");
Mat cv_transformMat = (Mat_<double>(2, 3) << \
M(0, 0), M(0, 1), M(0, 2), \
M(1, 0), M(1, 1), M(1, 2));
Mat align_img = Mat::zeros(src_frame.rows * 3, src_frame.cols * 3, src_frame.type());
src_frame.copyTo(align_img(Rect(100, 100, src_frame.cols, src_frame.rows)));
for (size_t k = 0; k < box.size(); k++)
{
box.at(k).x += 100;
box.at(k).y += 100;
}
//rectangle(align_img,
// Rect(box[0].x, box[0].y,
// box[3].x - box[0].x,
// box[3].y - box[0].y),
// Scalar(0, 255, 0), 1, 1);
for (size_t k = 0; k < landmark5.size(); k++)
{
landmark5.at(k).x += 100;
landmark5.at(k).y += 100;
//circle(align_img, landmark5.at(k), 2, Scalar(0, 255, 0), 1);
}
//imshow("align", align_img);
warpAffine(align_img, align_img, cv_transformMat, align_img.size());
//printf_s(" ============= point transformer =============\n");
Rect roi(0, 0, 0, 0);
for (size_t r = 0; r < box.size(); r++)
{
auto x = box.at(r).x, y = box.at(r).y;
box[r].x = x * M(0, 0) + y * M(0, 1) + M(0, 2);
box[r].y = x * M(1, 0) + y * M(1, 1) + M(1, 2);
}
for (size_t r = 0; r < landmark5.size(); r++)
{
auto x = landmark5.at(r).x, y = landmark5.at(r).y;
landmark5[r].x = x * M(0, 0) + y * M(0, 1) + M(0, 2);
landmark5[r].y = x * M(1, 0) + y * M(1, 1) + M(1, 2);
}
// 由于box的点对齐后会出现 脸不全的情况，需要使用对齐后的关键点来矫正box
auto dis_left_eye_right_eye = landmark5[1].x - landmark5[0].x;
auto dis_left_month_left_eye = landmark5[landmark5.size() - 2].y - landmark5[0].y;
box[0].x = min(box[0].x, landmark5[0].x - dis_left_eye_right_eye / 2); // left top X;
box[0].y = min(box[0].y, landmark5[0].y - dis_left_month_left_eye / 1.1); // left top Y;
box[box.size() - 1].x = max(box[box.size() - 1].x, landmark5[1].x + dis_left_eye_right_eye / 2); // right bottom X;
box[box.size() - 1].y = max(box[box.size() - 1].y, landmark5[landmark5.size() - 1].y + dis_left_month_left_eye / 1.1); // right bottom y;
roi.x = max(0, int(box[0].x));
roi.y = max(0, int(box[0].y));
roi.width = min(int(box.at(box.size() - 1).x - roi.x) + 10, align_img.cols - roi.x);
roi.height = min(int(box.at(box.size() - 1).y - roi.y) + 10, align_img.rows - roi.y);
Mat align_face = align_img(roi);
//// draw
//rectangle(align_img, roi, Scalar(0, 0, 255), 1, 1);
//for (size_t k = 0; k < landmark5.size(); k++)
//{
// circle(align_img, landmark5.at(k), 2, Scalar(0, 0, 255), 1);
//}
//resize(align_face, align_face, Size(300, 350));
//imshow("src", src_frame);
//imshow("align2", align_img);
//imshow("align3", align_face);
//waitKey(0);
return align_face;
}
static Mat face_align_bypoint72(Mat src_frame, vector<Point2d> landmark72, vector<Point2d> box) {
int rows = landmark72.size();
int cols = 2;
MatrixXf src_landmark_mtx(rows, 2);
int i = 0;
for (size_t r = 0; r < landmark72.size(); r++)
{
src_landmark_mtx(r, 0) = landmark72.at(r).x;
src_landmark_mtx(r, 1) = landmark72.at(r).y;
i++;
}
MatrixXf dst_landmark_mtx(rows, 2);
dst_landmark_mtx << 32.00, 210.00,
42.00, 281.00,
54.00, 353.00,
73.00, 424.00,
118.00, 497.00,
189.00, 558.00,
265.00, 579.00,
335.00, 553.00,
400.00, 494.00,
444.00, 425.00,
465.00, 353.00,
477.00, 282.00,
484.00, 211.00,
107.00, 256.00,
132.00, 241.00,
158.00, 238.00,
185.00, 245.00,
205.00, 267.00,
181.00, 274.00,
154.00, 275.00,
128.00, 268.00,
160.00, 254.00,
73.000, 212.00,
107.00, 190.00,
148.00, 194.00,
182.00, 207.00,
214.00, 231.00,
179.00, 226.00,
143.00, 219.00,
107.00, 213.00,
312.00, 266.00,
334.00, 243.00,
361.00, 235.00,
387.00, 239.00,
412.00, 254.00,
392.00, 268.00,
364.00, 275.00,
336.00, 273.00,
361.00, 253.00,
298.00, 230.00,
334.00, 207.00,
369.00, 196.00,
411.00, 192.00,
447.00, 213.00,
412.00, 215.00,
374.00, 221.00,
337.00, 228.00,
229.00, 270.00,
224.00, 314.00,
220.00, 357.00,
204.00, 402.00,
225.00, 409.00,
295.00, 410.00,
320.00, 404.00,
303.00, 359.00,
296.00, 315.00,
287.00, 270.00,
260.00, 392.00,
188.00, 465.00,
227.00, 464.00,
264.00, 467.00,
302.00, 461.00,
340.00, 466.00,
305.00, 487.00,
264.00, 492.00,
223.00, 485.00,
226.00, 475.00,
263.00, 481.00,
302.00, 476.00,
300.00, 467.00,
263.00, 467.00,
229.00, 465.00;
MatrixXf mean1 = src_landmark_mtx.colwise().mean(); // [1,2]
MatrixXf mean2 = dst_landmark_mtx.colwise().mean(); // [1,2]
float col_mean_1 = mean1(0, 0); // 列的均值
float col_mean_2 = mean2(0, 0);
cout << src_landmark_mtx << endl;
cout << dst_landmark_mtx << endl;
// std = sqrt(mean(abs(x - x.mean())**2))
MatrixXf m1(rows, 2), m2(rows, 2);
for (size_t r = 0; r < rows; r++)
{
for (size_t c = 0; c < cols; c++)
{
src_landmark_mtx(r, c) -= mean1(0, c);
dst_landmark_mtx(r, c) -= mean2(0, c);
auto abs_v1 = std::abs(src_landmark_mtx(r, c));
auto abs_v2 = std::abs(dst_landmark_mtx(r, c));
m1(r, c) = pow(abs_v1, 2);
m2(r, c) = pow(abs_v2, 2);
}
}
float std1 = sqrt(m1.mean());
float std2 = sqrt(m2.mean());
src_landmark_mtx /= std1;
dst_landmark_mtx /= std2;
printf_s(" ============= std norm =============\n");
cout << src_landmark_mtx << endl;
cout << dst_landmark_mtx << endl;
MatrixXf m = src_landmark_mtx.transpose() * dst_landmark_mtx; // [2,2]
JacobiSVD<MatrixXf> svd(m, ComputeThinU | ComputeThinV);
MatrixXf VT = svd.matrixV().transpose();
MatrixXf U = svd.matrixU();
MatrixXf A = svd.singularValues();
MatrixXf R = (U*VT).transpose(); // [2,2]
printf_s(" ============= svd =============\n");
cout << VT << endl;
cout << U << endl;
cout << A << endl;
cout << R << endl;
MatrixXf M(3, 3);
M.row(2) << 0., 0., 1.;
MatrixXf t1 = (std1 / std2)*R; // [2,2]
MatrixXf t2 = mean2.transpose() - (std1 / std2)*R*mean1.transpose(); // [2,1]
M.row(0) << t1(0, 0), t1(0, 1), t2(0, 0);
M.row(1) << t1(1, 0), t1(1, 1), t2(1, 0);
printf_s(" ============= transformer =============\n");
cout << M << endl;
printf_s(" ============= face transformer =============\n");
Mat cv_transformMat = (Mat_<double>(2, 3) << \
M(0, 0), M(0, 1), M(0, 2), \
M(1, 0), M(1, 1), M(1, 2));
Mat align_img = Mat::zeros(src_frame.rows * 3, src_frame.cols * 3, CV_8UC3);
src_frame.copyTo(align_img(Rect(100, 100, src_frame.cols, src_frame.rows)));
for (size_t k = 0; k < box.size(); k++)
{
box.at(k).x += 100;
box.at(k).y += 100;
}
//rectangle(align_img, Rect(box[0].x, box[0].y, box[3].x - box[0].x, box[3].y - box[0].y),
// Scalar(0, 255, 0), 1, 1);
for (size_t k = 0; k < landmark72.size(); k++)
{
landmark72.at(k).x += 100;
landmark72.at(k).y += 100;
//circle(align_img, landmark72.at(k), 2, Scalar(0, 255, 0), 1);
}
//imshow("align", align_img);
warpAffine(align_img, align_img, cv_transformMat, align_img.size());
printf_s(" ============= point transformer =============\n");
Rect roi(0, 0, 0, 0);
for (size_t r = 0; r < box.size(); r++)
{
auto x = box.at(r).x, y = box.at(r).y;
box[r].x = x * M(0, 0) + y * M(0, 1) + M(0, 2);
box[r].y = x * M(1, 0) + y * M(1, 1) + M(1, 2);
}
for (size_t r = 0; r < landmark72.size(); r++)
{
auto x = landmark72.at(r).x, y = landmark72.at(r).y;
landmark72[r].x = x * M(0, 0) + y * M(0, 1) + M(0, 2);
landmark72[r].y = x * M(1, 0) + y * M(1, 1) + M(1, 2);
}
// 由于box的点对齐后会出现 脸不全的情况，需要使用对齐后的关键点来矫正box
//auto dis_left_eye_right_eye = landmark72[38].x - landmark72[21].x;
//auto dis_left_month_left_eye = landmark72[58].y - landmark72[21].y;
//box[0].x = min(box[0].x, landmark72[21].x - dis_left_eye_right_eye / 2); // left top X;
//box[0].y = min(box[0].y, landmark72[21].y - dis_left_month_left_eye / 1.1); // left top Y;
//box[box.size() - 1].x = max(box[box.size() - 1].x, landmark72[38].x + dis_left_eye_right_eye / 2); // right bottom X;
//box[box.size() - 1].y = max(box[box.size() - 1].y, landmark72[58].y + dis_left_month_left_eye / 1.1); // right bottom y;
//roi.x = max(0, int(box[0].x));
//roi.y = max(0, int(box[0].y));
//roi.width = min(int(box.at(box.size() - 1).x - roi.x) + 10, align_img.cols - roi.x);
//roi.height = min(int(box.at(box.size() - 1).y - roi.y) + 10, align_img.rows - roi.y);
// 使用脸部外围的极值点来矫正
double min_x = landmark72[0].x, min_y = landmark72[41].y;
double max_x = landmark72[12].x, max_y = landmark72[6].y;
box[0].x = min(box[0].x, min_x);
box[0].y = min(box[0].y, min_y);
roi.x = max(0, int(box[0].x));
roi.y = max(0, int(box[0].y));
roi.width = min(int(max_x - roi.x) + 5, align_img.cols - roi.x);
roi.height = min(int(max_y - roi.y) + 5, align_img.rows - roi.y);
if (roi.width <= 0 || roi.height <= 0)
{
return align_img;
}
Mat align_face = align_img(roi);
//// draw box & keypoints
//rectangle(align_img, roi, Scalar(0, 0, 255), 1, 1);
//for (size_t k = 0; k < landmark72.size(); k++)
//{
// circle(align_img, landmark72.at(k), 2, Scalar(0, 0, 255), 1);
//}
//resize(align_face, align_face, Size(300, 350));
//imshow("src", src_frame);
//imshow("align2", align_img);
//imshow("align3", align_face);
//waitKey(0);
return align_face;
}
=================================================
3 输出
3.1 输出的图像就是垂直的人脸图像
Python 实现过程：
1 人脸检测
1.1 使用mtcnn-tensorflow进行人脸检测,会输出face box和landmark5
face box: [x1,y1,x2,y2]
landmark5: [left_eye,right_eye, nose, month_left, month_right]
2 图像变换(使用numpy矩阵库)
2.1 使用SVD分解, 计算变换矩阵,
2.2 进行图像的仿射变换
==========================================
# 计算旋转矩阵
def transformation_from_points(points1, points2):
points1 = points1.astype(np.float64)
points2 = points2.astype(np.float64)
c1 = np.mean(points1, axis=0)
c2 = np.mean(points2, axis=0)
points1 -= c1
points2 -= c2
s1 = np.std(points1)
s2 = np.std(points2)
points1 /= s1
points2 /= s2
U, S, Vt = np.linalg.svd(points1.T * points2)
R = (U * Vt).T
M = np.vstack([
np.hstack(((s2 / s1) * R, c2.T - (s2 / s1) * R * c1.T)),
np.asmatrix([0., 0., 1.])])
return M
# 关键点对齐
def landmark_alignment(src_landmark, M):
M_matrix = np.asmatrix(M)
assert M_matrix.shape == (3, 3)
rows, cols = src_landmark.shape
src_landmark2 = src_landmark.copy()
for i in range(rows):
x, y = src_landmark[i, 0], src_landmark[i, 1]
x1 = x * M_matrix[0, 0] + y * M_matrix[0, 1] + M_matrix[0, 2]
y1 = x * M_matrix[1, 0] + y * M_matrix[1, 1] + M_matrix[1, 2]
src_landmark2[i] = [x1, y1]
src_landmark2 = src_landmark2.astype(np.int)
return src_landmark2
# 人脸对齐
def face_alignment(src_landmark5, image_numpy):
h, w, c = image_numpy.shape
# 5点对齐后的基准点
dst_landmark5 = np.asmatrix([[30.2946, 51.6963], # left eye
[65.5318, 51.6963], # right eye
[48.0252, 71.7366], # nose
[33.5493, 92.3655], # left month
[62.7299, 92.3655]]) # right month
# 68点对齐后的基准点
dst_landmark68 = np.asmatrix([[282, 156], [278, 176], [277, 198], [276, 219],
[279, 241], [287, 262], [299, 280], [315, 293],
[333, 300], [352, 300], [372, 293], [391, 282],
[406, 270], [418, 253], [426, 236], [434, 217],
[439, 197], [301, 138], [313, 128], [328, 127],
[343, 130], [356, 138], [384, 144], [400, 142],
[417, 145], [431, 156], [437, 171], [365, 160],
[362, 174], [359, 188], [357, 202], [337, 211],
[344, 215], [352, 219], [360, 219], [368, 218],
[312, 152], [322, 147], [332, 149], [340, 159],
[330, 159], [320, 157], [385, 169], [396, 164],
[407, 167], [414, 176], [405, 177], [394, 174],
[317, 242], [329, 237], [341, 235], [347, 238],
[356, 237], [364, 243], [371, 253], [361, 258],
[351, 259], [343, 259], [335, 257], [326, 251],
[322, 242], [339, 243], [346, 245], [354, 245],
[366, 251], [353, 247], [345, 247], [338, 245]])
# 计算旋转矩阵
M = transformation_from_points(src_landmark5, dst_landmark5)
# 旋转图像
align_image = cv2.warpAffine(image_numpy, M[:2], (w, h))
# M = cv2.getPerspectiveTransform(np.asarray(src_landmark5, np.float32),
# np.asarray(dst_landmark5, np.float32))
# align_image = cv2.warpPerspective(image_numpy, M, (w, h))
return align_image, M
==========================================
3 输出
3.1 输出的图像就是垂直的人脸图像
