OpenCvSharp 通过特征点匹配图片

现在的手游基本都是重复操作,一个动作要等好久,结束之后继续另一个动作.很麻烦,所以动起了自己写一个游戏辅助的心思.

这个辅助本身没什么难度,就是通过不断的截图,然后从这个截图中找出预先截好的能代表相应动作的按钮或者触发条件的小图.

找到之后获取该子区域的左上角坐标,然后通过windows API调用鼠标或者键盘做操作就行了.

这里面最难的也就是找图了,因为要精准找图,而且最好能适应不同的分辨率下找图,所以在模板匹配的基础上,就有了SIFT和SURF的特征点找图方式.

在写的过程中查找资料,大都是C++ 或者python的, 很少有原生的C#实现, 所以我就直接拿来翻译过来了(稍作改动).

SIFT算法

public Point2d Point2fToPoint2d(Point2f point) => new Point2d((double)point.X, (double)point.Y);

public static Bitmap MatchPicBySift(Bitmap imgSrc, Bitmap imgSub)
        {
            using (Mat matSrc = imgSrc.ToMat())
            using (Mat matTo = imgSub.ToMat())
            using (Mat matSrcRet = new Mat())
            using (Mat matToRet = new Mat())
            {
                KeyPoint[] keyPointsSrc, keyPointsTo;
                using (var sift = OpenCvSharp.XFeatures2D.SIFT.Create())
                {
                    sift.DetectAndCompute(matSrc, null, out keyPointsSrc, matSrcRet);
                    sift.DetectAndCompute(matTo, null, out keyPointsTo, matToRet);
                }
                using (var bfMatcher = new OpenCvSharp.BFMatcher())
                {
                    var matches = bfMatcher.KnnMatch(matSrcRet, matToRet, k: 2);

                    var pointsSrc = new List<Point2f>();
                    var pointsDst = new List<Point2f>();
                    var goodMatches = new List<DMatch>();
                    foreach (DMatch[] items in matches.Where(x => x.Length > 1))
                    {
                        if (items[0].Distance < 0.5 * items[1].Distance)
                        {
                            pointsSrc.Add(keyPointsSrc[items[0].QueryIdx].Pt);
                            pointsDst.Add(keyPointsTo[items[0].TrainIdx].Pt);
                            goodMatches.Add(items[0]);
                            Console.WriteLine($"{keyPointsSrc[items[0].QueryIdx].Pt.X}, {keyPointsSrc[items[0].QueryIdx].Pt.Y}");
                        }
                    }

                    var outMat = new Mat();

                    // 算法RANSAC对匹配的结果做过滤
                    var pSrc = pointsSrc.ConvertAll(Point2fToPoint2d);
                    var pDst = pointsDst.ConvertAll(Point2fToPoint2d);
                    var outMask = new Mat();
                    // 如果原始的匹配结果为空, 则跳过过滤步骤
                    if (pSrc.Count > 0 && pDst.Count > 0)
                        Cv2.FindHomography(pSrc, pDst, HomographyMethods.Ransac, mask: outMask);
                    // 如果通过RANSAC处理后的匹配点大于10个,才应用过滤. 否则使用原始的匹配点结果(匹配点过少的时候通过RANSAC处理后,可能会得到0个匹配点的结果).
                    if (outMask.Rows > 10)
                    {
                        byte[] maskBytes = new byte[outMask.Rows * outMask.Cols];
                        outMask.GetArray(0, 0, maskBytes);
                        Cv2.DrawMatches(matSrc, keyPointsSrc, matTo, keyPointsTo, goodMatches, outMat, matchesMask: maskBytes, flags: DrawMatchesFlags.NotDrawSinglePoints);
                    }
                    else
                        Cv2.DrawMatches(matSrc, keyPointsSrc, matTo, keyPointsTo, goodMatches, outMat, flags: DrawMatchesFlags.NotDrawSinglePoints);
                    return OpenCvSharp.Extensions.BitmapConverter.ToBitmap(outMat);
                }
            }
        }

SURF算法

  public static Point2d Point2fToPoint2d(Point2f point) => new Point2d((double)point.X, (double)point.Y);

public static Bitmap MatchPicBySurf(Bitmap imgSrc, Bitmap imgSub, double threshold = 400)
        {
            using (Mat matSrc = imgSrc.ToMat())
            using (Mat matTo = imgSub.ToMat())
            using (Mat matSrcRet = new Mat())
            using (Mat matToRet = new Mat())
            {
                KeyPoint[] keyPointsSrc, keyPointsTo;
                using (var surf = OpenCvSharp.XFeatures2D.SURF.Create(threshold,4,3,true,true))
                {
                    surf.DetectAndCompute(matSrc, null, out keyPointsSrc, matSrcRet);
                    surf.DetectAndCompute(matTo, null, out keyPointsTo, matToRet);
                }
               
                using (var flnMatcher = new OpenCvSharp.FlannBasedMatcher())
                {
                    var matches = flnMatcher.Match(matSrcRet, matToRet);
                    //求最小最大距离
                    double minDistance = 1000;//反向逼近
                    double maxDistance = 0;
                    for (int i = 0; i < matSrcRet.Rows; i++)
                    {
                        double distance = matches[i].Distance;
                        if (distance > maxDistance)
                        {
                            maxDistance = distance;
                        }
                        if (distance < minDistance)
                        {
                            minDistance = distance;
                        }
                    }
                    Console.WriteLine($"max distance : {maxDistance}");
                    Console.WriteLine($"min distance : {minDistance}");

                    var pointsSrc = new List<Point2f>();
                    var pointsDst = new List<Point2f>();
                    //筛选较好的匹配点
                    var goodMatches = new List<DMatch>();
                    for (int i = 0; i < matSrcRet.Rows; i++)
                    {
                        double distance = matches[i].Distance;
                        if (distance < Math.Max(minDistance * 2, 0.02))
                        {
                            pointsSrc.Add(keyPointsSrc[matches[i].QueryIdx].Pt);
                            pointsDst.Add(keyPointsTo[matches[i].TrainIdx].Pt);
                            //距离小于范围的压入新的DMatch
                            goodMatches.Add(matches[i]);
                        }
                    }

                    var outMat = new Mat();

                    // 算法RANSAC对匹配的结果做过滤
                    var pSrc = pointsSrc.ConvertAll(Point2fToPoint2d);
                    var pDst = pointsDst.ConvertAll(Point2fToPoint2d);
                    var outMask = new Mat();
                    // 如果原始的匹配结果为空, 则跳过过滤步骤
                    if (pSrc.Count > 0 && pDst.Count > 0)
                        Cv2.FindHomography(pSrc, pDst, HomographyMethods.Ransac, mask: outMask);
                    // 如果通过RANSAC处理后的匹配点大于10个,才应用过滤. 否则使用原始的匹配点结果(匹配点过少的时候通过RANSAC处理后,可能会得到0个匹配点的结果).
                    if (outMask.Rows > 10)
                    {
                        byte[] maskBytes = new byte[outMask.Rows * outMask.Cols];
                        outMask.GetArray(0, 0, maskBytes);
                        Cv2.DrawMatches(matSrc, keyPointsSrc, matTo, keyPointsTo, goodMatches, outMat, matchesMask: maskBytes, flags: DrawMatchesFlags.NotDrawSinglePoints);
                    }
                    else
                        Cv2.DrawMatches(matSrc, keyPointsSrc, matTo, keyPointsTo, goodMatches, outMat, flags: DrawMatchesFlags.NotDrawSinglePoints);
                    return OpenCvSharp.Extensions.BitmapConverter.ToBitmap(outMat);
                }
            }
        }

模板匹配

 public static System.Drawing.Point FindPicFromImage(Bitmap imgSrc, Bitmap imgSub, double threshold = 0.9)
        {
            OpenCvSharp.Mat srcMat = null;
            OpenCvSharp.Mat dstMat = null;
            OpenCvSharp.OutputArray outArray = null;
            try
            {
                srcMat = imgSrc.ToMat();
                dstMat = imgSub.ToMat();
                outArray = OpenCvSharp.OutputArray.Create(srcMat);
               
                OpenCvSharp.Cv2.MatchTemplate(srcMat, dstMat, outArray, Common.templateMatchModes);
                double minValue, maxValue;
                OpenCvSharp.Point location, point;
                OpenCvSharp.Cv2.MinMaxLoc(OpenCvSharp.InputArray.Create(outArray.GetMat()), out minValue, out maxValue, out location, out point);
                Console.WriteLine(maxValue);
                if (maxValue >= threshold)
                    return new System.Drawing.Point(point.X, point.Y);
                return System.Drawing.Point.Empty;
            }
            catch(Exception ex)
            {
                return System.Drawing.Point.Empty;
            }
            finally
            {
                if (srcMat != null)
                    srcMat.Dispose();
                if (dstMat != null)
                    dstMat.Dispose();
                if (outArray != null)
                    outArray.Dispose();
            }
        }