海康威视监控设备的Yv12视频在XNA框架中播放
最近使用海康威视的5寸一体化智能球，查阅些SDK, 在Winform中海康威视能较好的集成，但是对于Yv12编解码和实时视频流的支持未找到较好的SDK介绍。
因为项目需要是用在XNA游戏框架中，本身有自己的消息处理机制，集成海康威视有些复杂。在结合Goblin XNA、DirectShow的基础上，查询和测试了相关的算法，给出下面的解码方式，分享给遇到同样问题的博友。
具体的SDK引用和DecFunctionEvent事件触发比较简单，请查阅海康威视官方样例中自定义视频流的RealDataCallBack中的PlayCtrl.DECCBFUN。
（CH-HCNetSDK(Windows32)V4.2.8.1_5目录中的“实时预览示例代码”等）
以下两个类的重点在GetImageTexture和_yv12ToRgb.ConvertToInt解码方法，以获取到Goblin Scene中int[] returnImage（每个int为color的packedValue，由RGB移位生成， rgb24[index++] = (int)(rgb[0]<<16 | rgb[1]<<8 | rgb[2]);）
各值 rgb[2] = py + rdif; //R
rgb[1] = py - invgdif; //G
rgb[0] = py + bdif; //B
1. 设备类-HkwsCapture.cs
使用Goblin的scene.AddVideoCaptureDevice(captureDevice)加载
using System;
using System.Collections;
using System.Collections.Generic;
using System.Drawing;
using System.Text;
using System.IO;
using System.Reflection;
using CameraWrapper.VideoCapture;
using GoblinXNA.Device.Capture;
using Microsoft.Xna.Framework;
using Microsoft.Xna.Framework.Graphics;
using System.Runtime.InteropServices;
using DShowNET;
using DShowNET.Device;
using GoblinXNA;
using GoblinXNA.Device;
namespace MkcXNALibrary.VideoLib
{
public class HkwsCapture : ISampleGrabberCB, IVideoCapture
{
private enum PlaybackState
{
Stopped,
Paused,
Running,
Init
}
#region Member Fields
private int cameraWidth;
private int cameraHeight;
private bool grayscale;
private bool cameraInitialized;
private Resolution resolution;
private FrameRate frameRate;
private ImageFormat format;
private ImageReadyCallback imageReadyCallback;
private volatile PlaybackState playbackState = PlaybackState.Stopped;
private IntPtr grabbedImage;
private String selectedVideoDeviceName;
/// <summary>
/// Used to count the number of times it failed to capture an image
/// If it fails more than certain times, it will assume that the video
/// capture device can not be accessed
/// </summary>
private int failureCount;
private const int FAILURE_THRESHOLD = 1000;
private bool _processing = false, _previewUpdating = false;
private YV12ToRGB _yv12ToRgb;
readonly CameraInitInfo _initInfo;
ICameraCapture _currentCamera = new HikCapture();
#endregion
#region Constructors
/// <summary>
/// Creates a video capture using the DirectShow library.
/// </summary>
public HkwsCapture(CameraInitInfo cameraInfo)
{
cameraInitialized = false;
grabbedImage = IntPtr.Zero;
cameraWidth = 0;
cameraHeight = 0;
grayscale = false;
_initInfo = cameraInfo;
failureCount = 0;
}
#endregion
#region Properties
public int Width
{
get { return cameraWidth; }
}
public int Height
{
get { return cameraHeight; }
}
public int VideoDeviceID { get; private set; }
public bool GrayScale
{
get { return grayscale; }
}
public bool Initialized
{
get { return cameraInitialized; }
}
public ImageFormat Format
{
get { return format; }
}
public IResizer MarkerTrackingImageResizer { get; set; }
public SpriteEffects RenderFormat
{
get { return SpriteEffects.None; }
}
/// <summary>
/// Sets the callback function to be called when a new image becomes ready.
/// </summary>
public ImageReadyCallback CaptureCallback
{
set { imageReadyCallback = value; }
}
#endregion
#region Public Methods
public void InitVideoCapture(int videoDeviceID, FrameRate framerate, Resolution resolution,
ImageFormat format, bool grayscale)
{
if (cameraInitialized)
return;
this.resolution = resolution;
this.grayscale = grayscale;
this.frameRate = framerate;
this.format = format;
switch (resolution)
{
case Resolution._160x120:
cameraWidth = 160;
cameraHeight = 120;
break;
case Resolution._320x240:
cameraWidth = 320;
cameraHeight = 240;
break;
case Resolution._640x480:
cameraWidth = 640;
cameraHeight = 480;
break;
case Resolution._800x600:
cameraWidth = 800;
cameraHeight = 600;
break;
case Resolution._1024x768:
cameraWidth = 1024;
cameraHeight = 768;
break;
case Resolution._1280x1024:
cameraWidth = 1280;
cameraHeight = 1024;
break;
case Resolution._1600x1200:
cameraWidth = 1600;
cameraHeight = 1200;
break;
case Resolution._1280x720:
cameraWidth = 1280;
cameraHeight = 720;
break;
}
if (!DsUtils.IsCorrectDirectXVersion())
throw new GoblinException("DirectX 8.1 NOT installed!");
var result = _currentCamera.Init(_initInfo);
if (!string.IsNullOrEmpty(result))
throw new GoblinException(result);
_currentCamera.StartPreview(IntPtr.Zero);
_currentCamera.DecFunctionEvent += new EventHandler<CameraWrapper.HkwsAPI.HkwsIPC.EventArgsStreamVideo>(_currentCamera_DecFunctionEvent);
_yv12ToRgb = new YV12ToRGB(cameraWidth, cameraHeight);
cameraInitialized = true;
}
void _currentCamera_DecFunctionEvent(object sender, CameraWrapper.HkwsAPI.HkwsIPC.EventArgsStreamVideo e)
{
if (_processing) return;
grabbedImage = e.Buffer;
}
private int count = 0;
public void GetImageTexture(int[] returnImage, ref IntPtr imagePtr)
{
if (grabbedImage != IntPtr.Zero)
{
failureCount = 0;
_processing = true;
bool replaceBackground = false;
if (imageReadyCallback != null)
replaceBackground = imageReadyCallback(grabbedImage, returnImage);
if ((returnImage != null) && !replaceBackground)
{
if (!_previewUpdating)
{
_previewUpdating = true;
_yv12ToRgb.ConvertToInt(grabbedImage, ref returnImage);
_previewUpdating = !_previewUpdating;
count++;
}
}
_processing = false;
}
else
{
failureCount++;
if (failureCount > FAILURE_THRESHOLD)
{
throw new GoblinException("Video capture device ID: HK IP " + ", Name: " +
selectedVideoDeviceName + " is used by " +
"other application, and can not be accessed");
}
}
}
public void Dispose()
{
}
/// <summary>
/// When you are ready to convert the byte array back
/// to an image, you can include the following code in your method.
/// </summary>
/// <param name="myByteArray"></param>
/// <returns></returns>
public static Image ConvertByteArrayToImage(byte[] myByteArray)
{
System.Drawing.Image newImage;
using (MemoryStream ms = new MemoryStream(myByteArray, 0, myByteArray.Length))
{
ms.Write(myByteArray, 0, myByteArray.Length);
newImage = Image.FromStream(ms, true);
// work with image here.
// You'll need to keep the MemoryStream open for
// as long as you want to work with your new image.
return newImage;
}
}
#endregion
#region ISampleGrabberCB Members
/// <summary>
/// Buffer Callback method from the DirectShow.NET ISampleGrabberCB interface. This method is called
/// when a new frame is grabbed by the SampleGrabber.
/// </summary>
/// <param name="SampleTime">The sample time.</param>
/// <param name="pBuffer">A pointer to the image buffer that contains the grabbed sample.</param>
/// <param name="BufferLen">The length of the image buffer containing the grabbed sample.</param>
/// <returns>0 = success.</returns>
public int BufferCB(double SampleTime, IntPtr pBuffer, int BufferLen)
{
while (_processing) { }
grabbedImage = pBuffer;
return 0;
}
/// <summary>
/// Sample CallBack method from the ISampleGrabberCB interface (DirectShow.NET). Not used.
/// </summary>
/// <param name="SampleTime"></param>
/// <param name="pSample"></param>
/// <returns></returns>
public int SampleCB(double SampleTime, IMediaSample pSample)
{
throw new GoblinException("The method or operation is not implemented.");
}
#endregion
}
}
2-YV12ToRGB.cs
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Drawing;
using System.IO;
namespace MkcXNALibrary.VideoLib
{
public class YV12ToRGB
{
private int width;
private int height;
private int length;
private int v; //v值的起始位置
private int u; //u值的起始位置
private int rdif, invgdif, bdif;
private int[] Table_fv1;
private int[] Table_fv2;
private int[] Table_fu1;
private int[] Table_fu2;
private int[] rgb = new int[3];
private int m, n, i, j, hfWidth;
private bool addHalf = true;
private int py;
private int pos, pos1;//图像调整
private byte temp;
static double[,] YUV2RGB_CONVERT_MATRIX = new double[3, 3] { { 1, 0, 1.4022 }, { 1, -0.3456, -0.7145 }, { 1, 1.771, 0 } };
public YV12ToRGB(int iWidth, int iHeight)
{
Table_fv1 = new int[256] { -180, -179, -177, -176, -174, -173, -172, -170, -169, -167, -166, -165, -163, -162, -160, -159, -158, -156, -155, -153, -152, -151, -149, -148, -146, -145, -144, -142, -141, -139, -138, -137, -135, -134, -132, -131, -130, -128, -127, -125, -124, -123, -121, -120, -118, -117, -115, -114, -113, -111, -110, -108, -107, -106, -104, -103, -101, -100, -99, -97, -96, -94, -93, -92, -90, -89, -87, -86, -85, -83, -82, -80, -79, -78, -76, -75, -73, -72, -71, -69, -68, -66, -65, -64, -62, -61, -59, -58, -57, -55, -54, -52, -51, -50, -48, -47, -45, -44, -43, -41, -40, -38, -37, -36, -34, -33, -31, -30, -29, -27, -26, -24, -23, -22, -20, -19, -17, -16, -15, -13, -12, -10, -9, -8, -6, -5, -3, -2, 0, 1, 2, 4, 5, 7, 8, 9, 11, 12, 14, 15, 16, 18, 19, 21, 22, 23, 25, 26, 28, 29, 30, 32, 33, 35, 36, 37, 39, 40, 42, 43, 44, 46, 47, 49, 50, 51, 53, 54, 56, 57, 58, 60, 61, 63, 64, 65, 67, 68, 70, 71, 72, 74, 75, 77, 78, 79, 81, 82, 84, 85, 86, 88, 89, 91, 92, 93, 95, 96, 98, 99, 100, 102, 103, 105, 106, 107, 109, 110, 112, 113, 114, 116, 117, 119, 120, 122, 123, 124, 126, 127, 129, 130, 131, 133, 134, 136, 137, 138, 140, 141, 143, 144, 145, 147, 148, 150, 151, 152, 154, 155, 157, 158, 159, 161, 162, 164, 165, 166, 168, 169, 171, 172, 173, 175, 176, 178 };
Table_fv2 = new int[256] { -92, -91, -91, -90, -89, -88, -88, -87, -86, -86, -85, -84, -83, -83, -82, -81, -81, -80, -79, -78, -78, -77, -76, -76, -75, -74, -73, -73, -72, -71, -71, -70, -69, -68, -68, -67, -66, -66, -65, -64, -63, -63, -62, -61, -61, -60, -59, -58, -58, -57, -56, -56, -55, -54, -53, -53, -52, -51, -51, -50, -49, -48, -48, -47, -46, -46, -45, -44, -43, -43, -42, -41, -41, -40, -39, -38, -38, -37, -36, -36, -35, -34, -33, -33, -32, -31, -31, -30, -29, -28, -28, -27, -26, -26, -25, -24, -23, -23, -22, -21, -21, -20, -19, -18, -18, -17, -16, -16, -15, -14, -13, -13, -12, -11, -11, -10, -9, -8, -8, -7, -6, -6, -5, -4, -3, -3, -2, -1, 0, 0, 1, 2, 2, 3, 4, 5, 5, 6, 7, 7, 8, 9, 10, 10, 11, 12, 12, 13, 14, 15, 15, 16, 17, 17, 18, 19, 20, 20, 21, 22, 22, 23, 24, 25, 25, 26, 27, 27, 28, 29, 30, 30, 31, 32, 32, 33, 34, 35, 35, 36, 37, 37, 38, 39, 40, 40, 41, 42, 42, 43, 44, 45, 45, 46, 47, 47, 48, 49, 50, 50, 51, 52, 52, 53, 54, 55, 55, 56, 57, 57, 58, 59, 60, 60, 61, 62, 62, 63, 64, 65, 65, 66, 67, 67, 68, 69, 70, 70, 71, 72, 72, 73, 74, 75, 75, 76, 77, 77, 78, 79, 80, 80, 81, 82, 82, 83, 84, 85, 85, 86, 87, 87, 88, 89, 90, 90 };
Table_fu1 = new int[256] { -44, -44, -44, -43, -43, -43, -42, -42, -42, -41, -41, -41, -40, -40, -40, -39, -39, -39, -38, -38, -38, -37, -37, -37, -36, -36, -36, -35, -35, -35, -34, -34, -33, -33, -33, -32, -32, -32, -31, -31, -31, -30, -30, -30, -29, -29, -29, -28, -28, -28, -27, -27, -27, -26, -26, -26, -25, -25, -25, -24, -24, -24, -23, -23, -22, -22, -22, -21, -21, -21, -20, -20, -20, -19, -19, -19, -18, -18, -18, -17, -17, -17, -16, -16, -16, -15, -15, -15, -14, -14, -14, -13, -13, -13, -12, -12, -11, -11, -11, -10, -10, -10, -9, -9, -9, -8, -8, -8, -7, -7, -7, -6, -6, -6, -5, -5, -5, -4, -4, -4, -3, -3, -3, -2, -2, -2, -1, -1, 0, 0, 0, 1, 1, 1, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 5, 5, 6, 6, 6, 7, 7, 7, 8, 8, 8, 9, 9, 9, 10, 10, 11, 11, 11, 12, 12, 12, 13, 13, 13, 14, 14, 14, 15, 15, 15, 16, 16, 16, 17, 17, 17, 18, 18, 18, 19, 19, 19, 20, 20, 20, 21, 21, 22, 22, 22, 23, 23, 23, 24, 24, 24, 25, 25, 25, 26, 26, 26, 27, 27, 27, 28, 28, 28, 29, 29, 29, 30, 30, 30, 31, 31, 31, 32, 32, 33, 33, 33, 34, 34, 34, 35, 35, 35, 36, 36, 36, 37, 37, 37, 38, 38, 38, 39, 39, 39, 40, 40, 40, 41, 41, 41, 42, 42, 42, 43, 43 };
Table_fu2 = new int[256] { -227, -226, -224, -222, -220, -219, -217, -215, -213, -212, -210, -208, -206, -204, -203, -201, -199, -197, -196, -194, -192, -190, -188, -187, -185, -183, -181, -180, -178, -176, -174, -173, -171, -169, -167, -165, -164, -162, -160, -158, -157, -155, -153, -151, -149, -148, -146, -144, -142, -141, -139, -137, -135, -134, -132, -130, -128, -126, -125, -123, -121, -119, -118, -116, -114, -112, -110, -109, -107, -105, -103, -102, -100, -98, -96, -94, -93, -91, -89, -87, -86, -84, -82, -80, -79, -77, -75, -73, -71, -70, -68, -66, -64, -63, -61, -59, -57, -55, -54, -52, -50, -48, -47, -45, -43, -41, -40, -38, -36, -34, -32, -31, -29, -27, -25, -24, -22, -20, -18, -16, -15, -13, -11, -9, -8, -6, -4, -2, 0, 1, 3, 5, 7, 8, 10, 12, 14, 15, 17, 19, 21, 23, 24, 26, 28, 30, 31, 33, 35, 37, 39, 40, 42, 44, 46, 47, 49, 51, 53, 54, 56, 58, 60, 62, 63, 65, 67, 69, 70, 72, 74, 76, 78, 79, 81, 83, 85, 86, 88, 90, 92, 93, 95, 97, 99, 101, 102, 104, 106, 108, 109, 111, 113, 115, 117, 118, 120, 122, 124, 125, 127, 129, 131, 133, 134, 136, 138, 140, 141, 143, 145, 147, 148, 150, 152, 154, 156, 157, 159, 161, 163, 164, 166, 168, 170, 172, 173, 175, 177, 179, 180, 182, 184, 186, 187, 189, 191, 193, 195, 196, 198, 200, 202, 203, 205, 207, 209, 211, 212, 214, 216, 218, 219, 221, 223, 225 };
width = iWidth;
height = iHeight;
length = iWidth * iHeight;
v = length;
u = (length * 5) >> 2;
hfWidth = iWidth >> 1;
}
public bool Convert(byte[] yv12, ref byte[] rgb24)
{
if (yv12.Length == 0 || rgb24.Length == 0)
return false;
m = -width;
n = -hfWidth;
for (int y = 0; y < height; y++)
{
m += width;
for (int x = 0; x < width; x++)
{
i = m + x;
j = n + (x >> 1);
py = (int)yv12[i];
rdif = Table_fv1[(int)yv12[v + j]];
invgdif = Table_fu1[(int)yv12[u + j]] + Table_fv2[(int)yv12[v + j]];
bdif = Table_fu2[(int)yv12[u + j]];
rgb[2] = py + rdif;
rgb[1] = py - invgdif;
rgb[0] = py + bdif;
j = m + x;
i = (j << 1) + j;
for (j = 0; j < 3; j++)
{
rgb24[i + j] = (byte)((rgb[j] < 0) ? 0 : ((rgb[j] > 255) ? 255 : rgb[j]));
}
if (x % 4 == 3)
{
pos = (m + x - 1) * 3;
pos1 = (m + x) * 3;
temp = rgb24[pos];
rgb24[pos] = rgb24[pos1];
rgb24[pos1] = temp;
temp = rgb24[pos + 1];
rgb24[pos + 1] = rgb24[pos1 + 1];
rgb24[pos1 + 1] = temp;
temp = rgb24[pos + 2];
rgb24[pos + 2] = rgb24[pos1 + 2];
rgb24[pos1 + 2] = temp;
}
}
}
return true;
}
public bool ConvertToInt(IntPtr yv12, ref int[] rgb24)
{
if (rgb24.Length == 0)
return false;
m = -width;
n = -hfWidth;
int temp1 = 0, temp2= 0;
int index = 0;
unsafe
{
byte* src = (byte*)yv12;
for (int y = 0; y < height; y++)
{
m += width;
if (addHalf)
{
n += hfWidth;
addHalf = false;
}
else
{
addHalf = true;
}
for (int x = 0; x < width; x++)
{
i = m + x;
j = n + (x >> 1);
py = *(src + i);
temp1 = (int) *(src + v + j);
temp2 = (int)*(src + u + j);
rdif = Table_fv1[temp1];
invgdif = Table_fu1[temp2] + Table_fv2[temp1];
bdif = Table_fu2[temp2];
rgb[2] = py + rdif; //R
rgb[1] = py - invgdif; //G
rgb[0] = py + bdif; //B
for (int colorIndex = 0; colorIndex < 3; colorIndex++)
{
if (rgb[colorIndex] < 0)
rgb[colorIndex] = 0;
if (rgb[colorIndex] > 255)
rgb[colorIndex] = 255;
}
rgb24[index++] = (int)(rgb[0]<<16 | rgb[1]<<8 | rgb[2]);
j = i;
i = (j << 1) + j;
}
}
}
return true;
}
/// <summary>
/// 将转换后的 RGB 图像数据按照 BMP 格式写入文件。
/// </summary>
/// <param name="rgbFrame">RGB 格式图像数据。</param>
/// <param name="width">图像宽（单位：像素）。</param>
/// <param name="height">图像高（单位：像素）。</param>
/// <param name="bmpFile"> BMP 文件名。</param>
internal static void WriteBMP(byte[] rgbFrame, int width, int height, string bmpFile)
{
// 写 BMP 图像文件。
int yu = width * 3 % 4;
int bytePerLine = 0;
yu = yu != 0 ? 4 - yu : yu;
bytePerLine = width * 3 + yu;
using (FileStream fs = File.Open(bmpFile, FileMode.Create))
{
using (BinaryWriter bw = new BinaryWriter(fs))
{
bw.Write('B');
bw.Write('M');
bw.Write(bytePerLine * height + 54);
bw.Write(0);
bw.Write(54);
bw.Write(40);
bw.Write(width);
bw.Write(height);
bw.Write((ushort)1);
bw.Write((ushort)24);
bw.Write(0);
bw.Write(bytePerLine * height);
bw.Write(0);
bw.Write(0);
bw.Write(0);
bw.Write(0);
byte[] data = new byte[bytePerLine * height];
int gIndex = width * height;
int bIndex = gIndex * 2;
for (int y = height - 1, j = 0; y >= 0; y--, j++)
{
for (int x = 0, i = 0; x < width; x++)
{
data[y * bytePerLine + i++] = rgbFrame[bIndex + j * width + x]; // B
data[y * bytePerLine + i++] = rgbFrame[gIndex + j * width + x]; // G
data[y * bytePerLine + i++] = rgbFrame[j * width + x]; // R
}
}
bw.Write(data, 0, data.Length);
bw.Flush();
}
}
}
/// <summary>
/// 将一桢 YUV 格式的图像转换为一桢 RGB 格式图像并转换成BMP图片。
/// </summary>
/// <param name="yuvFrame">YUV 格式图像数据。</param>
/// <param name="rgbFrame">RGB 格式图像数据。</param>
/// <param name="width">图像宽（单位：像素）。</param>
/// <param name="height">图像高（单位：像素）。</param>
/// <returns>Bitmap图片</returns>
internal Bitmap ConvertYUV2RGB(byte[] yuvFrame, byte[] rgbFrame, int width, int height)
{
int uIndex = width * height;
int vIndex = uIndex + ((width * height) >> 2);
int gIndex = width * height;
int bIndex = gIndex * 2;
int temp = 0;
//图片为pic1,RGB颜色的二进制数据转换得的int r,g,b;
Bitmap bm = new Bitmap(width, height);
int r = 0;
int g = 0;
int b = 0;
for (int y = 0; y < height; y++)
{
for (int x = 0; x < width; x++)
{
// R分量
temp = (int)(yuvFrame[y * width + x] + (yuvFrame[vIndex + (y / 2) * (width / 2) + x / 2] - 128) * YUV2RGB_CONVERT_MATRIX[0, 2]);
rgbFrame[y * width + x] = (byte)(temp < 0 ? 0 : (temp > 255 ? 255 : temp));
// G分量
temp = (int)(yuvFrame[y * width + x] + (yuvFrame[uIndex + (y / 2) * (width / 2) + x / 2] - 128) * YUV2RGB_CONVERT_MATRIX[1, 1] + (yuvFrame[vIndex + (y / 2) * (width / 2) + x / 2] - 128) * YUV2RGB_CONVERT_MATRIX[1, 2]);
rgbFrame[gIndex + y * width + x] = (byte)(temp < 0 ? 0 : (temp > 255 ? 255 : temp));
// B分量
temp = (int)(yuvFrame[y * width + x] + (yuvFrame[uIndex + (y / 2) * (width / 2) + x / 2] - 128) * YUV2RGB_CONVERT_MATRIX[2, 1]);
rgbFrame[bIndex + y * width + x] = (byte)(temp < 0 ? 0 : (temp > 255 ? 255 : temp));
Color c = Color.FromArgb(rgbFrame[y * width + x], rgbFrame[gIndex + y * width + x], rgbFrame[bIndex + y * width + x]);
bm.SetPixel(x, y, c);
}
}
return bm;
}
}
}
