蛙蛙推荐：用winsock和iocp api打造一个echo server

蛙蛙推荐：用winsock和iocp api打造一个echo server
摘要：上次给大家演示了在c#里如何使用iocp的相关api，这次我们结合winsock和iocp来做一个实际的例子，就是一个回显服务器。本示例是根据《windows网络编程技术》里的一个c++例子改变而成，其中费了不少功夫，请教了不少人，在此表示感谢，希望大家也能从中有所收获，尤其是和非托管代码打交道方面。当然本文只是一个示例程序，还有好多需要考虑的地方，比如有的地方可能会造成句柄泄漏，内存泄漏等。

这里用到了一些技术点，比如平台调用、反射，多线程等，当然还有iocp和winsock的api，及GCHandle,SafeHandle，Marshal类的使用等，不过相当多的东西，我上篇帖子讲的都很细了，如果对winsock api不了解可以查阅MSDN。也没什么技术难点，说几个细节的地方吧。

1、.net自带的System.Threading.NativeOverlapped类型是完全按照win32的Overlapped结构实现的，因为我们在WSASend和WSAReceive的时候想要传递更多的数据，而不只是一个重叠结构，所以我自己定义了一个WaOverlapped，在原有结构的末尾加了一个指针，指向一个自定义类的GC句柄，这样在工作线程里就可以拿到自定义的单IO数据了，这个是我想了N种办法不行后的一个可行的办法。
2、注意GCHandle在取到数据后不用的话记着Free掉，否则就有可能造成内存泄漏。
3、如果调用WSASend或者WSAReceive返回6的话，多半是你准备的单IO数据不对,6表示无效的句柄。
4、如果传递给WSASend或者WSAReceive的Overlapped没pin住，会抛异常的，等不到GetLastWin32Error，所以用GCHandle.Alloc(PerIoData.Overlapped, GCHandleType.Pinned)把它pin住。
5、这个类还没有进行各方面的优化，其中的单IO数据，socket等都可以做成对象池来重用，Accept还可以替换成AcceptEx来用一个现成的Socket来接受新的连接，而不是自动创建一个新的，还有缓冲区可以做成环状的，关于性能方面的优化，下次有机会再给大家做实验。
完整代码如下，windows2008打开不安全代码进行编译，然后可以用telnet进行测试。

using System;
using System.Net;
using System.Net.Sockets;
using System.Reflection;
using System.Runtime.ConstrainedExecution;
using System.Runtime.InteropServices;
using System.Threading;
using Microsoft.Win32.SafeHandles;

namespace WawaSocket.Net.Iocp
{
    用IOCP和winsock api实现一个echo服务器#region 用IOCP和winsock api实现一个echo服务器
    class IocpTest
    {
        private static readonly IntPtr INVALID_HANDLE_VALUE = new IntPtr(-1); //无效句柄
        const int PORT = 5150; //要监听的端口
        const int DATA_BUFSIZE = 8192;  //默认缓冲区
        const int ERROR_IO_PENDING = 997; //表示数据正在接受或者发送中
        const uint INIFINITE = 0xffffffff; //表示等待无限时长
        private static readonly Logger _logger = Logger.GetLogger(typeof(IocpTest));

        单IO数据#region 单IO数据
        [StructLayout(LayoutKind.Sequential)]
        class PerIoOperationData
        {
            public WaOverlapped Overlapped;
            public WSABuffer DataBuf;
            public readonly byte[] Buffer = new byte[DATA_BUFSIZE];
            public uint BytesSEND;
            public uint BytesRECV;
        }
        #endregion

        单句柄数据#region 单句柄数据
        [StructLayout(LayoutKind.Sequential)]
        class PerHandleData
        {
            public SafeSocketHandle Socket;
        }
        #endregion

        public static void Run()
        {
            WSAData wsaData;
            SocketError Ret;

            初始化套接字#region 初始化套接字
            _logger.Log("初始化socket");
            if ((Ret = Win32Api.WSAStartup(0x0202, out wsaData)) != SocketError.Success)
            {
                _logger.Error("WSAStartup failed with error {0}\n", Ret);
                return;
            }
            #endregion

            创建一个完成端口内核对象#region 创建一个完成端口内核对象
            _logger.Log("创建完成端口");
            // Setup an I/O completion port.
            SafeFileHandle CompletionPort = Win32Api.CreateIoCompletionPort(INVALID_HANDLE_VALUE, IntPtr.Zero, IntPtr.Zero, 0);
            if (CompletionPort.IsInvalid)
            {
                _logger.Error("CreateIoCompletionPort failed with error: {0}\n", Marshal.GetLastWin32Error());
                Marshal.ThrowExceptionForHR(Marshal.GetLastWin32Error());
                return;
            }
            #endregion

            创建工作线程#region 创建工作线程
            int processorCount = Environment.ProcessorCount;
            _logger.Log("创建{0}个工作线程", processorCount);
            for (int i = 0; i < processorCount; i++)
            {
                // Create a server worker thread and pass the completion port to the thread.
                var thread = new Thread(ThreadProc);
                thread.Start(CompletionPort);
            }
            #endregion

            创建监听用的套接字#region 创建监听用的套接字
            _logger.Log("创建监听套接字");
            // Create a listening socket
            SafeSocketHandle Listen = Win32Api.WSASocket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp, IntPtr.Zero, 0, SocketConstructorFlags.WSA_FLAG_OVERLAPPED);
            if (Listen.IsInvalid)
            {
                Listen.SetHandleAsInvalid();
                _logger.Error("WSASocket() failed with error {0}\n", Win32Api.WSAGetLastError());
                Marshal.ThrowExceptionForHR(Win32Api.WSAGetLastError());
                return;
            }
            #endregion

            将套接字与本地端口绑定#region 将套接字与本地端口绑定
            IPEndPoint InternetAddr = new IPEndPoint(IPAddress.Any, PORT);
            SocketAddress socketAddress = InternetAddr.Serialize();
            byte[] adress_buffer;
            int adress_size;
            _logger.Log("进行套接字绑定");
            if (!DoBind(Listen, socketAddress, out adress_buffer, out adress_size))
            {
                _logger.Error("bind() failed with error {0}\n", Win32Api.WSAGetLastError());
                Marshal.ThrowExceptionForHR(Win32Api.WSAGetLastError());
                return;
            }
            #endregion

            开始监听端口#region 开始监听端口
            _logger.Log("开始监听:{0}-{1}", InternetAddr.Address, InternetAddr.Port);
            // Prepare socket for listening
            if (Win32Api.listen(Listen, 5) == SocketError.SocketError)
            {
                _logger.Error("listen() failed with error {0}\n", Win32Api.WSAGetLastError());
                Marshal.ThrowExceptionForHR(Win32Api.WSAGetLastError());
                return;
            }
            #endregion

            起一个循环来接受新连接#region 起一个循环来接受新连接
            // Accept connections and assign to the completion port.
            while (true)
                unsafe
                {
                    接受新连接#region 接受新连接
                    _logger.Log("开始接受入站连接");
                    SafeSocketHandle Accept = Win32Api.accept(Listen.DangerousGetHandle(), adress_buffer, ref adress_size);
                    if (Accept.IsInvalid)
                    {
                        _logger.Error("WSAAccept() failed with error {0}\n", Win32Api.WSAGetLastError());
                        Marshal.ThrowExceptionForHR(Win32Api.WSAGetLastError());
                    }
                    _logger.Log("有新连接进入:{0}", Accept.GetHashCode());
                    #endregion

                    创建单句柄数据#region 创建单句柄数据
                    // Create a socket information structure to associate with the socket
                    PerHandleData PerHandleData = new PerHandleData();
                    GCHandle gch_PerHandleData = GCHandle.Alloc(PerHandleData);
                    // Associate the accepted socket with the original completion port.
                    PerHandleData.Socket = Accept;
                    #endregion

                    把新接受的套接字与完成端口绑定#region 把新接受的套接字与完成端口绑定
                    SafeFileHandle iocp = Win32Api.CreateIoCompletionPort(Accept.DangerousGetHandle(),
                                                                 CompletionPort.DangerousGetHandle(),
                                                                GCHandle.ToIntPtr(gch_PerHandleData), 0);
                    if (iocp == null)
                    {
                        _logger.Error("CreateIoCompletionPort failed with error {0}\n", Marshal.GetLastWin32Error());
                        Marshal.ThrowExceptionForHR(Marshal.GetLastWin32Error());
                        return;
                    }
                    #endregion

                    准备单IO数据#region 准备单IO数据
                    // Create per I/O socket information structure to associate with the 
                    // WSARecv call below.
                    PerIoOperationData PerIoData = new PerIoOperationData();
                    GCHandle gchPerIoData = GCHandle.Alloc(PerIoData);
                    PerIoData.Overlapped = new WaOverlapped { State = ((IntPtr)gchPerIoData) };
                    GCHandle gcHandle = GCHandle.Alloc(PerIoData.Overlapped, GCHandleType.Pinned);
                    PerIoData.BytesSEND = 0;
                    PerIoData.BytesRECV = 0;
                    PerIoData.DataBuf.Length = DATA_BUFSIZE;
                    PerIoData.DataBuf.Pointer = Marshal.UnsafeAddrOfPinnedArrayElement(PerIoData.Buffer, 0);
                    #endregion

                    开始投递异步接受数据的请求#region 开始投递异步接受数据的请求
                    SocketFlags Flags = SocketFlags.None;
                    _logger.Log("开始异步接受数据");
                    int RecvBytes;
                    SocketError error = Win32Api.WSARecv(Accept, ref PerIoData.DataBuf,
                        1, out RecvBytes, ref Flags, gcHandle.AddrOfPinnedObject(),
                        IntPtr.Zero);
                    if (error == SocketError.SocketError)
                    {
                        if (Win32Api.WSAGetLastError() != ERROR_IO_PENDING)
                        {
                            _logger.Error("WSARecv() failed with error {0}\n", Win32Api.WSAGetLastError());
                            Marshal.ThrowExceptionForHR(Win32Api.WSAGetLastError());
                            //其实在主线程退出之前都应该用PostQueuedCompletionStatus通知工作线程退出
                            return;
                        }
                    }
                    #endregion
                }
            #endregion
        }

        把一个套接字绑定在一个端口上的工具方法#region 把一个套接字绑定在一个端口上的工具方法
        private static bool DoBind(SafeSocketHandle Listen, SocketAddress address, out byte[] buffer, out int size)
        {
            FieldInfo socketAddress_m_Buffer = typeof(SocketAddress).GetField("m_Buffer",
                                                                              BindingFlags.Instance | BindingFlags.NonPublic);
            FieldInfo socketAddress_m_Size = typeof(SocketAddress).GetField("m_Size",
                                                                            BindingFlags.Instance | BindingFlags.NonPublic);
            var m_buffer = (byte[])socketAddress_m_Buffer.GetValue(address);
            var m_Size = (int)socketAddress_m_Size.GetValue(address);
            buffer = m_buffer;
            size = m_Size;

            if (Win32Api.bind(Listen, m_buffer, m_Size) != SocketError.Success)
            {
                return false;
            }
            return true;
        }
        #endregion

        工作线程#region 工作线程
        static unsafe void ThreadProc(object CompletionPortID)
        {
            var CompletionPort = (SafeFileHandle)CompletionPortID; //接受通知的完成端口
            SocketFlags Flags;
            IntPtr intptr_per_io_data, intptr_per_handle_data; //单句柄数据，单实例数据的指针
            GCHandle gcHandle_per_io_data, gcHandle_per_handle_data;//单句柄数据，单实例数据的gc句柄
            uint BytesTransferred; //接受或发送的数据
            PerHandleData PerHandleData; //单据并数据
            PerIoOperationData PerIoData; //单IO数据
            int SendBytes; //发送出的字节
            int RecvBytes; //接受到的字节
            在循环里接受和发送数据#region 在循环里接受和发送数据
            while (true)
            {

                在完成端口上等消息#region 在完成端口上等消息
                if (!Win32Api.GetQueuedCompletionStatus(CompletionPort, out BytesTransferred,
             out intptr_per_handle_data, out intptr_per_io_data, INIFINITE))
                {
                    _logger.Error("GetQueuedCompletionStatus failed with error {0}\n",
                        Marshal.GetLastWin32Error());
                    return;
                }
                #endregion

                拿到单据并数据#region 拿到单据并数据
                gcHandle_per_handle_data = GCHandle.FromIntPtr(intptr_per_handle_data);
                PerHandleData = (PerHandleData)gcHandle_per_handle_data.Target;
                #endregion

                拿到单IO数据#region 拿到单IO数据
                WaOverlapped o = new WaOverlapped();
                Marshal.PtrToStructure(intptr_per_io_data, o);
                gcHandle_per_io_data = GCHandle.FromIntPtr(o.State);
                PerIoData = (PerIoOperationData)gcHandle_per_io_data.Target;
                #endregion

                判断是否为断开请求#region 判断是否为断开请求
                if (BytesTransferred == 0)
                {
                    _logger.Log("断开连接 {0}", PerHandleData.Socket.GetHashCode());
                    PerHandleData.Socket.Close();
                    gcHandle_per_handle_data.Free();
                    gcHandle_per_io_data.Free();
                    continue;
                }
                #endregion

                根据异步操作的类型来更新单IO数据#region 根据异步操作的类型来更新单IO数据
                // Check to see if the BytesRECV field equals zero. If this is so, then
                // this means a WSARecv call just completed so update the BytesRECV field
                // with the BytesTransferred value from the completed WSARecv() call.
                if (PerIoData.BytesRECV == 0)
                {
                    PerIoData.BytesRECV = BytesTransferred;
                    PerIoData.BytesSEND = 0;
                }
                else
                {
                    PerIoData.BytesSEND += BytesTransferred;
                }
                #endregion

                try
                {
                    if (PerIoData.BytesRECV > PerIoData.BytesSEND)
                    {
                        如果收到消息就原封不动发给发送者#region 如果收到消息就原封不动发给发送者
                        _logger.Log("开始异步发送数据：{0}-{1}", PerHandleData.Socket.GetHashCode(),
                            PerIoData.Overlapped.GetHashCode());

                        更新单IO数据#region 更新单IO数据
                        // Post another WSASend() request.
                        // Since WSASend() is not gauranteed to send all of the bytes requested,
                        // continue posting WSASend() calls until all received bytes are sent.
                        GCHandle gchPerIoData = GCHandle.Alloc(PerIoData);
                        PerIoData.Overlapped = new WaOverlapped { State = ((IntPtr)gchPerIoData) };
                        GCHandle gchOverlapped = GCHandle.Alloc(PerIoData.Overlapped, GCHandleType.Pinned);
                        PerIoData.DataBuf.Pointer = Marshal.UnsafeAddrOfPinnedArrayElement(
                            PerIoData.Buffer, (int)PerIoData.BytesSEND);
                        PerIoData.DataBuf.Length = PerIoData.BytesRECV - PerIoData.BytesSEND;
                        #endregion

                        投递异步发送数据请求#region 投递异步发送数据请求
                        SocketError error = Win32Api.WSASend(PerHandleData.Socket, ref PerIoData.DataBuf,
                    1, out SendBytes, 0, gchOverlapped.AddrOfPinnedObject(), IntPtr.Zero);
                        if (error == SocketError.SocketError)
                        {
                            if (Win32Api.WSAGetLastError() != ERROR_IO_PENDING)
                            {
                                _logger.Error("WSASend() failed with error {0}", Win32Api.WSAGetLastError());
                                return;
                            }
                        }
                        #endregion
                        #endregion
                    }
                    else
                    {
                        如果没有需要发送的数据，就投递一个异步接受数据请求#region 如果没有需要发送的数据，就投递一个异步接受数据请求
                        _logger.Log("开始异步接受数据：{0}-{1}", PerHandleData.Socket.GetHashCode(),
    PerIoData.Overlapped.GetHashCode());
                        更新单IO数据#region 更新单IO数据

                        PerIoData.BytesRECV = 0;
                        // Now that there are no more bytes to send post another WSARecv() request.
                        Flags = SocketFlags.None;
                        GCHandle gchPerIoData = GCHandle.Alloc(PerIoData);
                        PerIoData.Overlapped = new WaOverlapped { State = ((IntPtr)gchPerIoData) };
                        GCHandle gchOverlapped = GCHandle.Alloc(PerIoData.Overlapped, GCHandleType.Pinned);
                        PerIoData.DataBuf.Length = DATA_BUFSIZE;
                        PerIoData.DataBuf.Pointer = Marshal.UnsafeAddrOfPinnedArrayElement(PerIoData.Buffer, 0);
                        #endregion

                        投递异步接受请求#region 投递异步接受请求
                        SocketError error = Win32Api.WSARecv(PerHandleData.Socket, ref PerIoData.DataBuf,
                    1, out RecvBytes, ref Flags, gchOverlapped.AddrOfPinnedObject(),
                    IntPtr.Zero);
                        if (error == SocketError.SocketError)
                        {
                            if (Win32Api.WSAGetLastError() != ERROR_IO_PENDING)
                            {
                                _logger.Error("WSARecv() failed with error{0}", Win32Api.WSAGetLastError());
                                return;
                            }
                        }
                        #endregion
                        #endregion
                    }
                }
                finally
                {
                    if (gcHandle_per_handle_data.IsAllocated)
                        gcHandle_per_io_data.Free();
                }

            }
            #endregion
        }
        #endregion
    } 
    #endregion

    封装原生的socket对象#region 封装原生的socket对象
    public class SafeSocketHandle : SafeHandleMinusOneIsInvalid
    {
        private Logger _logger = Logger.GetLogger(typeof(SafeSocketHandle));
        public SafeSocketHandle()
            : base(true)
        {
        }


        public SafeSocketHandle(bool ownsHandle)
            : base(ownsHandle)
        {
        }

        protected override bool ReleaseHandle()
        {
            if (Win32Api.closesocket(base.handle) == SocketError.SocketError)
            {
                _logger.Error("closesocket() failed with error {0}\n", Win32Api.WSAGetLastError());
            }
            return true;
        }
    } 
    #endregion

    日志类#region 日志类
    class Logger
    {
        public static Logger GetLogger(Type type)
        {
            return new Logger();
        }

        public void Log(object o)
        {
            Console.WriteLine(o);
        }
        public void Log(string format, params object[] objects)
        {
            Console.WriteLine(format, objects);
        }
        public void Error(object o)
        {
            Console.ForegroundColor = ConsoleColor.Red;
            Console.WriteLine(o);
            Console.ForegroundColor = ConsoleColor.White;
        }
        public void Error(string format, params object[] objects)
        {
            Console.ForegroundColor = ConsoleColor.Red;
            Console.WriteLine(format, objects);
            Console.ForegroundColor = ConsoleColor.White;
        }
    } 
    #endregion

    win32 structs#region win32 structs
    [StructLayout(LayoutKind.Sequential)]
    public class WaOverlapped
    {
        public IntPtr InternalLow;
        public IntPtr InternalHigh;
        public int OffsetLow;
        public int OffsetHigh;
        public IntPtr EventHandle;
        public IntPtr State;
        
    }

    [StructLayout(LayoutKind.Sequential)]
    internal struct WSABuffer
    {
        internal uint Length;
        internal IntPtr Pointer;
    }
    [StructLayout(LayoutKind.Sequential)]
    internal struct WSAData
    {
        internal short wVersion;
        internal short wHighVersion;
        [MarshalAs(UnmanagedType.ByValTStr, SizeConst = 0x101)]
        internal string szDescription;
        [MarshalAs(UnmanagedType.ByValTStr, SizeConst = 0x81)]
        internal string szSystemStatus;
        internal short iMaxSockets;
        internal short iMaxUdpDg;
        internal IntPtr lpVendorInfo;
    }
    [Flags]
    internal enum SocketConstructorFlags
    {
        WSA_FLAG_MULTIPOINT_C_LEAF = 4,
        WSA_FLAG_MULTIPOINT_C_ROOT = 2,
        WSA_FLAG_MULTIPOINT_D_LEAF = 0x10,
        WSA_FLAG_MULTIPOINT_D_ROOT = 8,
        WSA_FLAG_OVERLAPPED = 1
    }
    #endregion

    封装winsock和iocp的相关API原型#region 封装winsock和iocp的相关API原型
    class Win32Api
    {
        [DllImport("ws2_32.dll", CharSet = CharSet.Ansi, SetLastError = true)]
        internal static extern SocketError WSAStartup([In] short wVersionRequested, out WSAData lpWSAData);

        [DllImport("kernel32.dll", CharSet = CharSet.Auto, SetLastError = true)]
        public static extern SafeFileHandle CreateIoCompletionPort(IntPtr FileHandle, IntPtr ExistingCompletionPort, IntPtr CompletionKey, uint NumberOfConcurrentThreads);
        [DllImport("ws2_32.dll", CharSet = CharSet.Auto, SetLastError = true)]
        internal static extern SafeSocketHandle WSASocket([In] AddressFamily addressFamily, [In] SocketType socketType, [In] ProtocolType protocolType, [In] IntPtr protocolInfo, [In] uint group, [In] SocketConstructorFlags flags);
        [DllImport("Ws2_32.dll", EntryPoint = "WSAGetLastError", SetLastError = true, CharSet = CharSet.Ansi, ExactSpelling = true, CallingConvention = CallingConvention.StdCall)]
        public static extern int WSAGetLastError();
        [DllImport("ws2_32.dll", SetLastError = true)]
        internal static extern SocketError bind([In] SafeSocketHandle socketHandle, [In] byte[] socketAddress, [In] int socketAddressSize);
        [DllImport("ws2_32.dll", SetLastError = true)]
        internal static extern SocketError listen([In] SafeSocketHandle socketHandle, [In] int backlog);
        [DllImport("ws2_32.dll", SetLastError = true, ExactSpelling = true)]
        internal static extern SafeSocketHandle accept([In] IntPtr socketHandle, [Out] byte[] socketAddress, [In, Out] ref int socketAddressSize);
        [DllImport("ws2_32.dll", SetLastError = true)]
        internal static extern SocketError WSARecv([In] SafeSocketHandle socketHandle, [In, Out] ref WSABuffer buffer, [In] int bufferCount, out int bytesTransferred, [In, Out] ref SocketFlags socketFlags, [In] IntPtr overlapped, [In] IntPtr completionRoutine);
        [DllImport("kernel32.dll", CharSet = CharSet.Auto, SetLastError = true)]
        public static extern unsafe bool GetQueuedCompletionStatus(SafeFileHandle CompletionPort,
            out uint lpNumberOfBytes, out IntPtr lpCompletionKey,
            out IntPtr lpOverlapped, uint dwMilliseconds);
        [ReliabilityContract(Consistency.WillNotCorruptState, Cer.Success), DllImport("ws2_32.dll", SetLastError = true, ExactSpelling = true)]
        internal static extern SocketError closesocket([In] IntPtr socketHandle);
        [DllImport("ws2_32.dll", SetLastError = true)]
        internal static extern SocketError WSASend([In] SafeSocketHandle socketHandle, [In] ref WSABuffer buffer, [In] int bufferCount, out int bytesTransferred, [In] SocketFlags socketFlags, [In] IntPtr overlapped, [In] IntPtr completionRoutine);
    }
    #endregion

    public class WawaIocpTest
    {
        public static void Main(String[] args)
        {
            IocpTest.Run();
        }
    }
}

 

小结：希望通过这3篇帖子能加深大家对c#开发windows上的网络应用的了解。