C#数据序列化研究:改进版KLV
所谓KLV即Key-Length-Value,以【键-数据长度-数据】的形式将数据序列化成字节流,
这是一种高性能和强兼容性的数据序列化方案,,缺点就是用起来比较麻烦,
KLV类似于 TLV(Tag-Length-Value) 和谷歌的 protobuf,相比于protobuf(protobuf是基于TLV实现的),KLV更加简单清晰原始,易于实现,而 protobuf则规则繁多,使用了很多巧妙的设计,这让其数据更紧凑(序列化后占用的空间更小),适用性更强(KLV的key目前只支持最大255,protobuf没限制),但是实现起来比较麻烦(不过大部分主流语言都有现成的库) ,如果要序列化的数据是类型对象,使用protobuf是个很不错的选择
KLV出现的需求场景如下:
1,硬件和云端的数据交互,最开始是以流的形式顺序写入数据,但是由于版本迭代,数据字段难免出现新增插入更新移除等现象,流式结构加了一大堆版本判定,混乱不堪
2,于是考虑使用Json来序列化数据,但是json性能消耗以及资源占用不甚理想,而且硬件端也没有现成的Json库使用
3,protobuf-net貌似只能传递类型对象进行序列化,不太适用写入零散的变量
4,因此模仿Json搞出了一个KLV格式,为每个数据指定一个key,下位机根据key获取数据,解决兼容性问题
该方案的Length比较特殊,由于传输中存在大量基础数值类型,使用4个字节比较浪费,Varint又不适用(因为有的时候只有彻底写完数据才知道数据的长度,为了提高性能,需要预先为数据长度占位,而Varint是可变长度,无法确定字节长度),因此特使用一种特殊的机制来描述Length:当描述长度的首字节的最高位为0时,用4个字节的后31位表示数值;当首字节的最高位为1时,用1个字节的后7位表示数值(2024-9-3之前的版本是反过来的,即当首字节的最高位为0时,用1个字节的后7位表示数值;当首字节的最高位为1时,用4个字节的后31位表示数值,这样很怪,毕竟1个字节才是特殊情况,而且最原始的版本就是直接使用4字节的,为了兼容符合常理的4字节,就改成最新的了)
不使用Key-Type-Value是因为KLV更简单灵活,传输的时候不用关心数据是什么,用的时候才会根据文档读取指定的key转成具体的数据,当读取到不认识的Key时,可以直接跳过Length,不存在Type的兼容性问题
2024-9-3 改版,主要改动:
1,提供了几个未被封装为对象的数据的写入和读取方法,
2,默认使用4字节记录数据长度,1字节作为特殊情况(第一个字节高位为1时使用一个字节,否则使用4个字节)
测试示例

public void Test() { byte[] buffer = new byte[1024]; KLVWriter kw = new KLVWriter(buffer, 0); kw.WriteInt32(1, 123); kw.WriteString(2, "漳卅"); kw.WriteStringArray(3, new string[] { "aaa", "123", "测试啊" }); KLVReader kr = new KLVReader(kw.Data,0,kw.Length); var p1 = kr.ReadInt32(1); var p2 = kr.ReadString(2); var p3 = kr.ReadStringList(3); }
写入器核心逻辑

using System; namespace KLV { /// <summary> /// KLV写入器 /// </summary> partial class KLVWriter { /// <summary> /// 缓冲区 /// </summary> public byte[] Data { get; private set; } /// <summary> /// 当前KLV开始索引 /// </summary> public int StartIndex { get; private set; } /// <summary> /// 当前KLV偏移量 /// </summary> int Offset; /// <summary> /// 写入数据的长度 /// </summary> public int Length { get { return Offset - StartIndex; } } /// <summary> /// 创建一个KLV写入器,以传入的字节数组为缓冲区,从指定索引开始写数据,写入的数据不能超出缓冲区的大小 /// </summary> /// <param name="data">缓冲区</param> /// <param name="startIndex">数据开始的索引</param> public KLVWriter(byte[] data, int startIndex) { Data = data; StartIndex = startIndex; Offset = startIndex; } internal KLVWriter() { } /// <summary> /// 重置缓冲区和索引,达到复用效果 /// </summary> public void Reset(byte[] data, int startIndex) { Data = data; StartIndex = startIndex; Offset = startIndex; } /// <summary> /// 当已知数据长度时,传入长度,自动计算,减少空间浪费, /// 正常情况下用4个字节描述长度,特殊情况下可以用一个字节描述长度(第一个字节大于等于0x80) /// </summary> int WriteLength_Auto(int len) { if (len < 0x80) { //如果数值低于0x80,就用一个字节表示 len |= 0x80; //如果用一个字节表示,就需要将该字节的最高位置为1 Data[Offset++] = (byte)len; return 1; } else { WriteLength_FourBytes(len, Offset); Offset += 4; return 4; } } /// <summary> /// 写入固定4字节的长度,长度不能为负数,因此最高位必然为0,读取的时候根据第一个字节的最高位判断字节数 /// </summary> void WriteLength_FourBytes(int len, int offset) { if (len < 0) { throw new KLVException("数据长度不能为负数"); } len.GetBytes().CopyTo(Data, offset); } /// <summary> /// 写入key和len /// </summary> void WriteKeyAndLen(byte key, int len) { Data[Offset++] = key; WriteLength_Auto(len); } /// <summary> /// 写入单字节的val /// </summary> void WriteVal(byte val) { Data[Offset++] = val; } /// <summary> /// 写入字节数组的val /// </summary> int WriteVal(byte[] val) { Buffer.BlockCopy(val, 0, Data, Offset, val.Length); Offset += val.Length; return val.Length; } /// <summary> /// 只写入key /// </summary> void WriteKey(byte key) { Data[Offset++] = key; } int WriteLenAndVal(Action<KLVWriter> writer) { Offset += 4; //预留4个字节的空间写长度 KLVWriter kw = new KLVWriter(Data, Offset); writer(kw); Offset += kw.Length; WriteLenAtEnd(kw.Length); int count = 4 + kw.Length; //返回当前LV的总长度 return count; } /// <summary> /// 写入len及其val,一般用于已有的字节数组的写入,返回写入的len和val的总长度 /// </summary> int WriteLenAndVal(byte[] val) { int size = WriteLength_Auto(val.Length); //len WriteVal(val); //val return size + val.Length; } /// <summary> /// 将指定对象写入到缓冲区中且最后写入数据长度,返回写入的Len和Val的总长度 /// </summary> int WriteLenAndVal<T>(T val, KLVWriter kw, Action<T, KLVWriter> writer) { Offset += 4; //预留4个字节的空间写长度 kw.Reset(Data, Offset); //复用一个写入器 writer(val, kw); Offset += kw.Length; WriteLenAtEnd(kw.Length); int count = 4 + kw.Length; //返回当前LV的总长度 return count; } /// <summary> /// 最后写入长度,一些操作刚开始无法确认数据长度,因此先预留4个字节的长度,先写数据,后再填充长度 /// </summary> void WriteLenAtEnd(int length) { WriteLength_FourBytes(length, Offset - length - 4); //向预留的空间内写入数据的长度 } } }
写入器常见类型

using System; using System.Collections.Generic; using System.ComponentModel; using System.Text; namespace KLV { /// <summary> /// KLV写入器 /// </summary> public partial class KLVWriter { readonly object _lock = new object(); #region 基础类型数据 /// <summary> /// 写入一个布尔值 /// </summary> /// <param name="key">身份key</param> /// <param name="val">布尔值</param> public void WriteBoolem(byte key, bool val) { lock (_lock) { WriteKeyAndLen(key, 1); WriteVal((byte)(val ? 1 : 0)); } } /// <summary> /// 写入一个有符号8位整型 /// </summary> /// <param name="key">身份key</param> /// <param name="val">有符号8位整型</param> public void WriteInt8(byte key, sbyte val) { lock (_lock) { WriteKeyAndLen(key, 1); WriteVal((byte)val); } } /// <summary> /// 写入一个无符号8位整型 /// </summary> /// <param name="key">身份key</param> /// <param name="val">无符号8位整型</param> public void WriteUInt8(byte key, byte val) { lock (_lock) { WriteKeyAndLen(key, 1); WriteVal(val); } } /// <summary> /// 写入一个有符号16位整型 /// </summary> /// <param name="key">身份key</param> /// <param name="val">有符号16位整型</param> public void WriteInt16(byte key, short val) { lock (_lock) { WriteKeyAndLen(key, 2); WriteVal(val.GetBytes()); } } /// <summary> /// 写入一个无符号16位整型 /// </summary> /// <param name="key">身份key</param> /// <param name="val">无符号16位整型</param> public void WriteUInt16(byte key, ushort val) { lock (_lock) { WriteKeyAndLen(key, 2); WriteVal(val.GetBytes()); } } /// <summary> /// 写入一个有符号32位整型 /// </summary> /// <param name="key">身份key</param> /// <param name="val">有符号32位整型</param> public void WriteInt32(byte key, int val) { lock (_lock) { WriteKeyAndLen(key, 4); WriteVal(val.GetBytes()); } } /// <summary> /// 写入一个无符号32位整型 /// </summary> /// <param name="key">身份key</param> /// <param name="val">无符号32位整型</param> public void WriteUInt32(byte key, uint val) { lock (_lock) { WriteKeyAndLen(key, 4); WriteVal(val.GetBytes()); } } /// <summary> /// 写入一个有符号64位整型 /// </summary> /// <param name="key">身份key</param> /// <param name="val">有符号64位整型</param> public void WriteInt64(byte key, long val) { lock (_lock) { WriteKeyAndLen(key, 8); WriteVal(val.GetBytes()); } } /// <summary> /// 写入一个无符号64位整型 /// </summary> /// <param name="key">身份key</param> /// <param name="val">无符号64位整型</param> public void WriteUInt64(byte key, ulong val) { lock (_lock) { WriteKeyAndLen(key, 8); WriteVal(val.GetBytes()); } } /// <summary> /// 写入一个32位单精度浮点型 /// </summary> /// <param name="key">身份key</param> /// <param name="val">32位单精度浮点型</param> public void WriteFloat(byte key, float val) { lock (_lock) { WriteKeyAndLen(key, 4); WriteVal(val.GetBytes()); } } /// <summary> /// 写入一个64位双精度浮点型 /// </summary> /// <param name="key">身份key</param> /// <param name="val">64位双精度浮点型</param> public void WriteDouble(byte key, double val) { lock (_lock) { WriteKeyAndLen(key, 8); WriteVal(val.GetBytes()); } } /// <summary> /// 写入一个字符串 /// </summary> /// <param name="key">身份key</param> /// <param name="val">字符串</param> public void WriteString(byte key, string val) { lock (_lock) { byte[] bs = Encoding.UTF8.GetBytes(val); WriteKeyAndLen(key, bs.Length); WriteVal(bs); } } #endregion #region 基础类型集合数据 /// <summary> /// 写入一个布尔数值集 /// </summary> /// <param name="key">身份key</param> /// <param name="vals">布尔数值集</param> public void WriteBooleanArray(byte key, IList<bool> vals) { lock (_lock) { WriteKeyAndLen(key, vals.Count); foreach (var val in vals) { WriteVal((byte)(val ? 1 : 0)); } } } /// <summary> /// 写入一个有符号8位整型集 /// </summary> /// <param name="key">身份key</param> /// <param name="vals">有符号8位整型集</param> public void WriteInt8Array(byte key, IList<sbyte> vals) { lock (_lock) { WriteKeyAndLen(key, vals.Count); foreach (var val in vals) { WriteVal((byte)val); } } } /// <summary> /// 写入一个无符号8位整型数组 /// </summary> /// <param name="key">身份key</param> /// <param name="vals">8位整型数组</param> public void WriteUInt8Array(byte key, byte[] vals) { lock (_lock) { WriteKeyAndLen(key, vals.Length); WriteVal(vals); } } /// <summary> /// 写入一个有符号16位整型集 /// </summary> /// <param name="key">身份key</param> /// <param name="vals">有符号16位整型集</param> public void WriteInt16Array(byte key, IList<short> vals) { lock (_lock) { WriteKeyAndLen(key, vals.Count * 2); foreach (var val in vals) { WriteVal(val.GetBytes()); } } } /// <summary> /// 写入一个无符号16位整型集 /// </summary> /// <param name="key">身份key</param> /// <param name="vals">无符号16位整型集</param> public void WriteUInt16Array(byte key, IList<ushort> vals) { lock (_lock) { WriteKeyAndLen(key, vals.Count * 2); foreach (var val in vals) { WriteVal(val.GetBytes()); } } } /// <summary> /// 写入一个有符号32位整型集 /// </summary> /// <param name="key">身份key</param> /// <param name="vals">有符号32位整型集</param> public void WriteInt32Array(byte key, IList<int> vals) { lock (_lock) { WriteKeyAndLen(key, vals.Count * 4); foreach (var val in vals) { WriteVal(val.GetBytes()); } } } /// <summary> /// 写入一个无符号32位整型集 /// </summary> /// <param name="key">身份key</param> /// <param name="vals">无符号32位整型集</param> public void WriteUInt32Array(byte key, IList<uint> vals) { lock (_lock) { WriteKeyAndLen(key, vals.Count * 4); foreach (var val in vals) { WriteVal(val.GetBytes()); } } } /// <summary> /// 写入一个有符号64位整型数集 /// </summary> /// <param name="key">身份key</param> /// <param name="vals">有符号64位整型数集</param> public void WriteInt64Array(byte key, IList<long> vals) { lock (_lock) { WriteKeyAndLen(key, vals.Count * 8); foreach (var val in vals) { WriteVal(val.GetBytes()); } } } /// <summary> /// 写入一个无符号64位整型数集 /// </summary> /// <param name="key">身份key</param> /// <param name="vals">无符号64位整型数集</param> public void WriteUInt64Array(byte key, IList<ulong> vals) { lock (_lock) { WriteKeyAndLen(key, vals.Count * 8); foreach (var val in vals) { WriteVal(val.GetBytes()); } } } /// <summary> /// 写入一个32位单精度浮点集 /// </summary> /// <param name="key">身份key</param> /// <param name="vals">32位单精度浮点集</param> public void WriteFloatArray(byte key, IList<float> vals) { lock (_lock) { WriteKeyAndLen(key, vals.Count * 4); foreach (var val in vals) { WriteVal(val.GetBytes()); } } } /// <summary> /// 写入一个64位双精度浮点集 /// </summary> /// <param name="key">身份key</param> /// <param name="vals">64位双精度浮点集</param> public void WriteDoubleArray(byte key, IList<double> vals) { lock (_lock) { WriteKeyAndLen(key, vals.Count * 8); foreach (var val in vals) { WriteVal(val.GetBytes()); } } } #endregion /// <summary> /// 写入一个对象,必须在委托函数中写入,目的是限制写入顺序,避免出现数据混乱 /// </summary> /// <param name="key">身份key</param> /// <param name="writer">写对象字段的操作委托</param> public void WriteObject(byte key, Action<KLVWriter> writer) { lock (_lock) { WriteKey(key); WriteLenAndVal(writer); } } /// <summary> /// 写入空值的KLV,有的时候key暂时没有数据,需要写入长度为0的数据, /// 直接使用其他函数会创建一些用不到的对象,因此创建了该函数提高性能 /// </summary> /// <param name="key">身份key</param> public void WriteNull(byte key) { lock (_lock) { WriteKey(key); WriteLength_Auto(0); } } /// <summary> /// 以UTF-8编码写入一个字符串集 /// </summary> /// <param name="key">身份key</param> /// <param name="vals">字符串集</param> public void WriteStringArray(byte key, IList<string> vals) { lock (_lock) { WriteKey(key); Offset += 4; int count = 0; foreach (var val in vals) { count += WriteLenAndVal(Encoding.UTF8.GetBytes(val)); } WriteLenAtEnd(count); } } /// <summary> /// 写入一个对象集合 /// </summary> /// <typeparam name="T">对象泛型</typeparam> /// <param name="key">身份key</param> /// <param name="vals">对象集合</param> /// <param name="writer">单个对象写入操作委托</param> public void WriteObjectList<T>(byte key, IEnumerable<T> vals, Action<T, KLVWriter> writer) { lock (_lock) { WriteKey(key); Offset += 4; int count = 0; KLVWriter kw = new KLVWriter(); foreach (var val in vals) { count += WriteLenAndVal(val, kw, writer); } WriteLenAtEnd(count); } } /// <summary> /// 写入一个字典 /// </summary> /// <typeparam name="K">字典的键的泛型</typeparam> /// <typeparam name="V">字典的值的泛型</typeparam> /// <param name="key">身份key</param> /// <param name="vals">字典对象</param> /// <param name="writer">每次写入键值对的操作委托</param> public void WriteDictionary<K, V>(byte key, Dictionary<K, V> vals, Action<KeyValuePair<K, V>, KLVWriter> writer) { WriteObjectList(key, vals, writer); } } }
读取器核心逻辑

using System; using System.Collections.Generic; namespace KLV { /// <summary> /// KLV读取器 /// </summary> partial class KLVReader { /// <summary> /// 解析出来的数据 /// </summary> readonly Dictionary<byte, Block> dict = new Dictionary<byte, Block>(); /// <summary> /// 缓冲区 /// </summary> byte[] Data; /// <summary> /// 从缓冲区中指定位置开始指定长度中解析一个KLV结构,并提供读取操作 /// </summary> /// <param name="data">缓冲区数据</param> /// <param name="startIndex">开始索引</param> /// <param name="dataLength">数据长度</param> public KLVReader(byte[] data, int startIndex, int dataLength) { Parse(data, startIndex, dataLength); } internal KLVReader() { } /// <summary> /// 解析数据,提出来方便复用 /// </summary> public void Parse(byte[] data, int startIndex, int dataLength) { Data = data; dict.Clear(); int offset = startIndex; int offset_end = offset + dataLength; while (offset < offset_end) { //循环解析 byte key = data[offset++]; //先读取一个字节的key (int length, int size) = ReadLenght(data, offset); //在读取len offset += size; try { dict.Add(key, new Block(offset, length)); //最后读取val } catch (Exception ex) { Console.WriteLine(key); throw ex; } offset += length; }; } /// <summary> /// 读取KLV中的len,因为KLV采用了2种方式表示len,因此要特殊处理, /// 正常情况下,使用4个字节的后31位记录最多2的31次方的数值 /// 特殊情况下,使用1个字节的后7位记录最多127个数值 /// </summary> (int, int) ReadLenght(byte[] data, int offset) { int size = 1; //默认认为是1个字节 int length = data[offset]; if (length >= 0x80) { //如果第一个字节的最高位是1,0x80=0b1000_0000 length &= 0x7F; //将最高位的1还原位0,0x7F=127=0b0111_111; } else { //如果是4个字节,第一个字节肯定不可能大于等于0x80,否则就是负数了 length = data.ToInt32(offset); size = 4; } return (length, size); } /// <summary> /// 从字典中查找指定key的相关数据 /// </summary> Block GetBlock(byte key) { if (!dict.TryGetValue(key, out Block block)) { throw new KLVException($"未找到key:{key},data:{BitConverter.ToString(Data)}"); } return block; } /// <summary> /// 根据block循环读取val集合 /// </summary> IEnumerable<(int, int)> ReadVals(Block block) { lock (_lock) { int offset = block.Offset; int offset_end = offset + block.Length; while (offset < offset_end) { (int len, int size) = ReadLenght(Data, offset); //每个数据都有一个长度 offset += size; yield return (offset, len); offset += len; } } } /// <summary> /// 验证当前键值对中是否存在指定key /// </summary> /// <param name="key">字段的身份key</param> /// <returns></returns> public bool ContainsKey(byte key) { return dict.ContainsKey(key); } } }
读取器常见类型

using System; using System.Collections.Generic; using System.Linq; using System.Text; namespace KLV { /// <summary> /// KLV读取器 /// </summary> public partial class KLVReader { readonly object _lock = new object(); #region 基础类型数据 /// <summary> /// 读取一个布尔类型 /// </summary> /// <param name="key">身份key</param> /// <returns>布尔类型</returns> public bool ReadBoolem(byte key) { lock (_lock) { return Data[GetBlock(key).Offset] == 1; } } /// <summary> /// 读取一个有符号8位整型 /// </summary> /// <param name="key">身份key</param> /// <returns>有符号8位整型</returns> public sbyte ReadInt8(byte key) { lock (_lock) { return (sbyte)Data[GetBlock(key).Offset]; } } /// <summary> /// 读取一个无符号8位整型 /// </summary> /// <param name="key">身份key</param> /// <returns>无符号8位整型</returns> public byte ReadUInt8(byte key) { lock (_lock) { return Data[GetBlock(key).Offset]; } } /// <summary> /// 读取一个有符号16位整型 /// </summary> /// <param name="key">身份key</param> /// <returns>有符号16位整型</returns> public short ReadInt16(byte key) { lock (_lock) { return Data.ToInt16(GetBlock(key).Offset); } } /// <summary> /// 读取一个无符号16位整型 /// </summary> /// <param name="key">身份key</param> /// <returns>无符号16位整型</returns> public ushort ReadUInt16(byte key) { lock (_lock) { return Data.ToUInt16(GetBlock(key).Offset); } } /// <summary> /// 读取一个有符号32位整型 /// </summary> /// <param name="key">身份key</param> /// <returns>有符号32位整型</returns> public int ReadInt32(byte key) { lock (_lock) { return Data.ToInt32(GetBlock(key).Offset); } } /// <summary> /// 读取一个无符号32位整型 /// </summary> /// <param name="key">身份key</param> /// <returns>无符号32位整型</returns> public uint ReadUInt32(byte key) { lock (_lock) { return Data.ToUInt32(GetBlock(key).Offset); } } /// <summary> /// 读取一个有符号64位整型 /// </summary> /// <param name="key">身份key</param> /// <returns>有符号64位整型</returns> public long ReadInt64(byte key) { lock (_lock) { return Data.ToInt64(GetBlock(key).Offset); } } /// <summary> /// 读取一个无符号64位整型 /// </summary> /// <param name="key">身份key</param> /// <returns>无符号64位整型</returns> public ulong ReadUInt64(byte key) { lock (_lock) { return Data.ToUInt64(GetBlock(key).Offset); } } /// <summary> /// 读取一个32位单精度浮点 /// </summary> /// <param name="key">身份key</param> /// <returns>32位单精度浮点</returns> public float ReadFloat(byte key) { lock (_lock) { return Data.ToFloat(GetBlock(key).Offset); } } /// <summary> /// 读取一个64位双精度浮点 /// </summary> /// <param name="key">身份key</param> /// <returns>64位双精度浮点</returns> public double ReadDouble(byte key) { lock (_lock) { return Data.ToDouble(GetBlock(key).Offset); } } /// <summary> /// 读取一个字符串 /// </summary> /// <param name="key">身份key</param> /// <returns>字符串</returns> public string ReadString(byte key) { lock (_lock) { var block = GetBlock(key); return Encoding.UTF8.GetString(Data, block.Offset, block.Length); } } #endregion #region 基础类型数组数据 /// <summary> /// 读取一个布尔数组 /// </summary> /// <param name="key">身份key</param> /// <returns>布尔数组</returns> public bool[] ReadBoolemArray(byte key) { lock (_lock) { var block = GetBlock(key); bool[] arr = new bool[block.Length]; for (int i = 0; i < arr.Length; i++) { arr[i] = Data[block.Offset + i] == 1; } return arr; } } /// <summary> /// 读取一个有符号8位整型数组 /// </summary> /// <param name="key">身份key</param> /// <returns>有符号8位整型数组</returns> public sbyte[] ReadInt8Array(byte key) { lock (_lock) { var block = GetBlock(key); sbyte[] arr = new sbyte[block.Length]; for (int i = 0; i < arr.Length; i++) { arr[i] = (sbyte)Data[block.Offset + i]; } return arr; } } /// <summary> /// 读取一个无符号8位整型数组 /// </summary> /// <param name="key">身份key</param> /// <returns>无符号8位整型数组</returns> public byte[] ReadUInt8Array(byte key) { lock (_lock) { var block = GetBlock(key); byte[] arr = new byte[block.Length]; for (int i = 0; i < arr.Length; i++) { arr[i] = Data[block.Offset + i]; } return arr; } } /// <summary> /// 读取一个有符号16位整型数组 /// </summary> /// <param name="key">身份key</param> /// <returns>有符号16位整型数组</returns> public short[] ReadInt16Array(byte key) { lock (_lock) { var block = GetBlock(key); int size = 2; short[] arr = new short[block.Length / size]; for (int i = 0; i < arr.Length; i++) { arr[i] = Data.ToInt16(block.Offset + i * size); } return arr; } } /// <summary> /// 读取一个无符号16位整型数组 /// </summary> /// <param name="key">身份key</param> /// <returns>无符号16位整型数组</returns> public ushort[] ReadUInt16Array(byte key) { lock (_lock) { var block = GetBlock(key); int size = 2; ushort[] arr = new ushort[block.Length / size]; for (int i = 0; i < arr.Length; i++) { arr[i] = Data.ToUInt16(block.Offset + i * size); } return arr; } } /// <summary> /// 读取一个有符号32位整型数组 /// </summary> /// <param name="key">身份key</param> /// <returns>有符号32位整型数组</returns> public int[] ReadInt32Array(byte key) { lock (_lock) { var block = GetBlock(key); int size = 4; int[] arr = new int[block.Length / size]; for (int i = 0; i < arr.Length; i++) { arr[i] = Data.ToInt32(block.Offset + i * size); } return arr; } } /// <summary> /// 读取一个无符号32位整型数组 /// </summary> /// <param name="key">身份key</param> /// <returns>无符号32位整型数组</returns> public uint[] ReadUInt32Array(byte key) { lock (_lock) { var block = GetBlock(key); int size = 4; uint[] arr = new uint[block.Length / size]; for (int i = 0; i < arr.Length; i++) { arr[i] = Data.ToUInt32(block.Offset + i * size); } return arr; } } /// <summary> /// 读取一个有符号64位整型数组 /// </summary> /// <param name="key">身份key</param> /// <returns>有符号64位整型数组</returns> public long[] ReadInt64Array(byte key) { lock (_lock) { var block = GetBlock(key); int size = 8; long[] arr = new long[block.Length / size]; for (int i = 0; i < arr.Length; i++) { arr[i] = Data.ToInt64(block.Offset + i * size); } return arr; } } /// <summary> /// 读取一个无符号64位整型数组 /// </summary> /// <param name="key">身份key</param> /// <returns>无符号64位整型数组</returns> public ulong[] ReadUInt64Array(byte key) { lock (_lock) { var block = GetBlock(key); int size = 8; ulong[] arr = new ulong[block.Length / size]; for (int i = 0; i < arr.Length; i++) { arr[i] = Data.ToUInt64(block.Offset + i * size); } return arr; } } /// <summary> /// 读取一个32位单精度浮点数组 /// </summary> /// <param name="key">身份key</param> /// <returns>32位单精度浮点数组</returns> public float[] ReadFloatArray(byte key) { lock (_lock) { var block = GetBlock(key); int size = 4; float[] arr = new float[block.Length / size]; for (int i = 0; i < arr.Length; i++) { arr[i] = Data.ToFloat(block.Offset + i * size); } return arr; } } /// <summary> /// 读取一个64位双精度浮点数组 /// </summary> /// <param name="key">身份key</param> /// <returns>64位双精度浮点数组</returns> public double[] ReadDoubleArray(byte key) { lock (_lock) { var block = GetBlock(key); int size = 8; double[] arr = new double[block.Length / size]; for (int i = 0; i < arr.Length; i++) { arr[i] = Data.ToDouble(block.Offset + i * size); } return arr; } } #endregion /// <summary> /// 读取对象,返回一个KLV读取器,能够读取子对象的所有字段 /// </summary> /// <param name="key">身份key</param> /// <returns></returns> public KLVReader ReadObject(byte key) { lock (_lock) { var block = GetBlock(key); return new KLVReader(Data, block.Offset, block.Length); } } /// <summary> /// 读取对象,通过委托传递一个KLV读取器,能够读取子对象的所有字段 /// </summary> /// <param name="key">身份key</param> /// <param name="action">执行读取操作的委托</param> public void ReadObject(byte key, Action<KLVReader> action) { lock (_lock) { var block = GetBlock(key); action(new KLVReader(Data, block.Offset, block.Length)); } } /// <summary> /// 读取对象,通过委托方法传递一个KLV读取器,能够读取构建子对象,最终返回对象 /// </summary> /// <typeparam name="T">要返回的对象泛型</typeparam> /// <param name="key">身份key</param> /// <param name="func">执行读取操作并构建实例返回对象的委托</param> /// <returns>泛型对象</returns> public T ReadObject<T>(byte key, Func<KLVReader, T> func) { lock (_lock) { var block = GetBlock(key); return func(new KLVReader(Data, block.Offset, block.Length)); } } /// <summary> /// 读取字符串集合 /// </summary> /// <param name="key">身份key</param> /// <returns>字符串集合</returns> public List<string> ReadStringList(byte key) { lock (_lock) { var block = GetBlock(key); List<string> list = new List<string>(); //字符串长度不固定,因此不知道数量,这里使用集合 foreach (var (offset, len) in ReadVals(block)) { list.Add(Encoding.UTF8.GetString(Data, offset, len)); } return list; } } /// <summary> /// 读取泛型对象集合 /// </summary> /// <typeparam name="T">类型</typeparam> /// <param name="key">身份key</param> /// <param name="read">读取单一对象的逻辑委托</param> /// <returns>泛型对象集合</returns> public List<T> ReadObjectList<T>(byte key, Func<KLVReader, T> read) { lock (_lock) { List<T> list = new List<T>(); KLVReader kr = new KLVReader(); foreach (var (offset, len) in ReadVals(GetBlock(key))) { kr.Parse(Data, offset, len); //复用一个kr,提高性能 list.Add(read(kr)); } return list; } } /// <summary> /// 读取字典 /// </summary> /// <typeparam name="K">字典键泛型</typeparam> /// <typeparam name="V">字典值泛型</typeparam> /// <param name="key">身份key</param> /// <param name="read">读取键值对的逻辑委托</param> /// <returns>字典</returns> public Dictionary<K, V> ReadDictionary<K, V>(byte key, Func<KLVReader, KeyValuePair<K, V>> read) { return ReadObjectList(key, read).ToDictionary(p => p.Key, p => p.Value); } } }
其他模块未改动,可以参考2024-2-21
2024-2-21优化了下代码,基本逻辑没变,只是让代码的可读性提高了一些
使用BenchmarkDotNet对比KLV,Newtonsoft.Json,ProtoBuf性能
程序入口

internal class Program { static void Main(string[] args) { var br = BenchmarkRunner.Run<Test>(); new Test().Start(); Console.WriteLine("按任意键退出..."); Console.ReadKey(); } }
测试代码:

[MemoryDiagnoser] public class Test { readonly Entity entity = Entity.CreateTestInstance(); public void Start() { var aaa = ByKLV(); var bbb = ByNewtonsoftJson(); var ccc = ByProtoBuf(); //序列化后在发序列化回来,最后统一用Json对比是否一致 var json1 = JsonConvert.SerializeObject(aaa); var json2 = JsonConvert.SerializeObject(bbb); var json3 = JsonConvert.SerializeObject(ccc); Console.WriteLine(json1 == json2); Console.WriteLine(json1 == json3); Console.WriteLine(json1); Console.WriteLine(json2); } [Benchmark] public Entity ByKLV() { KLVWriter kw = WriteEntity(entity); return ReadEntity(kw.Data, kw.StartIndex, kw.Length); } [Benchmark] public Entity ByNewtonsoftJson() { string json = JsonConvert.SerializeObject(entity); return JsonConvert.DeserializeObject<Entity>(json); } [Benchmark] public Entity ByProtoBuf() { byte[] buffer; long length; using (MemoryStream ms = new MemoryStream()) { Serializer.Serialize(ms, entity); buffer = ms.GetBuffer(); length = ms.Length; } using (MemoryStream ms2 = new MemoryStream(buffer, 0, (int)length)) { return Serializer.Deserialize<Entity>(ms2); } } KLVWriter WriteEntity(Entity entity) { byte[] data = new byte[1024]; int offset = 0; KLVWriter kw = new KLVWriter(data, offset); kw.WriteBoolem(1, entity.P1); kw.WriteInt8(2, entity.P2); kw.WriteUInt8(3, entity.P3); kw.WriteInt16(4, entity.P4); kw.WriteUInt16(5, entity.P5); kw.WriteInt32(6, entity.P6); kw.WriteUInt32(7, entity.P7); kw.WriteInt64(8, entity.P8); kw.WriteUInt64(9, entity.P9); kw.WriteFloat(10, entity.P10); kw.WriteDouble(11, entity.P11); kw.WriteString(12, entity.P12); kw.WriteBooleanArray(13, entity.P13); kw.WriteInt8Array(14, entity.P14); kw.WriteUInt8Array(15, entity.P15); kw.WriteInt16Array(16, entity.P16); kw.WriteUInt16Array(17, entity.P17); kw.WriteInt32Array(18, entity.P18); kw.WriteUInt32Array(19, entity.P19); kw.WriteInt64Array(20, entity.P20); kw.WriteUInt64Array(21, entity.P21); kw.WriteFloatArray(22, entity.P22); kw.WriteDoubleArray(23, entity.P23); kw.WriteStringArray(24, entity.P24); kw.WriteObject(25, entity.P25, (val, p) => { p.WriteInt32(1, val.P1); p.WriteDouble(2, val.P2); p.WriteString(3, val.P3); }); kw.WriteObjectList(26, entity.P26, (val, p) => { p.WriteInt32(1, val.P1); p.WriteDouble(2, val.P2); p.WriteString(3, val.P3); }); kw.WriteDictionary(27, entity.P27, (val, p) => { p.WriteInt32(1, val.Key); p.WriteString(2, val.Value); }); kw.WriteDictionary(28, entity.P28, (val, p) => { p.WriteString(1, val.Key); p.WriteInt32(2, val.Value); }); kw.WriteDictionary(29, entity.P29, (val, p) => { p.WriteInt32(1, val.Key); p.WriteObject(2, val.Value, (val_sub, p_sub) => { p_sub.WriteInt32(1, val_sub.P1); p_sub.WriteDouble(2, val_sub.P2); p_sub.WriteString(3, val_sub.P3); }); }); return kw; } Entity ReadEntity(byte[] data, int startIndex, int length) { KLVReader kr = new KLVReader(data, startIndex, length); Entity entity = new Entity { P1 = kr.ReadBoolem(1), P2 = kr.ReadInt8(2), P3 = kr.ReadUInt8(3), P4 = kr.ReadInt16(4), P5 = kr.ReadUInt16(5), P6 = kr.ReadInt32(6), P7 = kr.ReadUInt32(7), P8 = kr.ReadInt64(8), P9 = kr.ReadUInt64(9), P10 = kr.ReadFloat(10), P11 = kr.ReadDouble(11), P12 = kr.ReadString(12), P13 = kr.ReadBoolemArray(13), P14 = kr.ReadInt8Array(14), P15 = kr.ReadUInt8Array(15), P16 = kr.ReadInt16Array(16), P17 = kr.ReadUInt16Array(17), P18 = kr.ReadInt32Array(18), P19 = kr.ReadUInt32Array(19), P20 = kr.ReadInt64Array(20), P21 = kr.ReadUInt64Array(21), P22 = kr.ReadFloatArray(22), P23 = kr.ReadDoubleArray(23), P24 = kr.ReadStringList(24), P25 = kr.ReadObject(25, read => new EntitySub { P1 = read.ReadInt32(1), P2 = read.ReadDouble(2), P3 = read.ReadString(3), }), P26 = kr.ReadObjectList(26, read => new EntitySub { P1 = read.ReadInt32(1), P2 = read.ReadDouble(2), P3 = read.ReadString(3) }), P27 = kr.ReadDictionary(27, p => new KeyValuePair<int, string>(p.ReadInt32(1), p.ReadString(2))), P28 = kr.ReadDictionary(28, p => new KeyValuePair<string, int>(p.ReadString(1), p.ReadInt32(2))), P29 = kr.ReadDictionary(29, p => new KeyValuePair<int, EntitySub>(p.ReadInt32(1), p.ReadObject(2, read => new EntitySub { P1 = read.ReadInt32(1), P2 = read.ReadDouble(2), P3 = read.ReadString(3) }))) }; return entity; } }
写入器核心逻辑:

partial class KLVWriter { /// <summary> /// 缓冲区 /// </summary> public byte[] Data { get; private set; } /// <summary> /// 当前KLV开始索引 /// </summary> public int StartIndex { get; private set; } /// <summary> /// 当前KLV偏移量 /// </summary> int Offset; /// <summary> /// 写入数据的长度 /// </summary> public int Length { get { return Offset - StartIndex; } } /// <summary> /// 以传入的字节数组为基础,创建一个KLV写入器,从指定索引写数据 /// </summary> /// <param name="data">缓冲区</param> /// <param name="startIndex">数据开始的索引</param> public KLVWriter(byte[] data, int startIndex) { Data = data; StartIndex = startIndex; Offset = startIndex; } internal KLVWriter() { } /// <summary> /// 重置缓冲区和索引,达到复用效果 /// </summary> /// <param name="data"></param> /// <param name="startIndex"></param> public void Reset(byte[] data, int startIndex) { Data = data; StartIndex = startIndex; Offset = startIndex; } /// <summary> /// 当已知数据长度时,传入长度,自动计算,减少空间浪费 /// </summary> int WriteLength_Auto(int len) { if (len < 0x80) { Data[Offset++] = (byte)len; //最高位为0时,用1个字节中的后7位表示数值 return 1; } else { WriteLength_FourBytes(len, Offset); Offset += 4; return 4; } } /// <summary> /// 写入固定4字节的长度,当先写数据时,可以通过固定4个字节来提高性能 /// </summary> void WriteLength_FourBytes(int len, int offset) { len = (int)(len | 0x80000000); //最高位为1时,用4个字节中后31位表示数值 len.GetBytes().CopyTo(Data, offset); } /// <summary> /// 写入key和len /// </summary> void WriteKeyAndLen(byte key, int len) { Data[Offset++] = key; WriteLength_Auto(len); } /// <summary> /// 写入单字节的val /// </summary> void WriteVal(byte val) { Data[Offset++] = val; } /// <summary> /// 写入字节数组的val /// </summary> int WriteVal(byte[] val) { Buffer.BlockCopy(val, 0, Data, Offset, val.Length); Offset += val.Length; return val.Length; } /// <summary> /// 只写入key /// </summary> void WriteKey(byte key) { Data[Offset++] = key; } /// <summary> /// 写入val及其len,一般用于已有的字节数组的写入,返回写入的len和val的总长度 /// </summary> int WriteValAndLen(byte[] val) { int size = WriteLength_Auto(val.Length); //len WriteVal(val); //val return size + val.Length; } /// <summary> /// 将指定对象写入到缓冲区中且最后写入数据长度,返回写入的Len和Val的总长度 /// </summary> int WriteValAndLen<T>(T val, KLVWriter kw, Action<T, KLVWriter> writer) { Offset += 4; //预留4个字节的空间写长度 kw.Reset(Data, Offset); writer(val, kw); Offset += kw.Length; WriteLenAtEnd(kw.Length); int count = 4 + kw.Length; //返回当前LV的总长度 return count; } /// <summary> /// 最后写入长度,一些操作刚开始无法确认数据长度,因此先预留4个字节的长度,先写数据,后再填充长度 /// </summary> void WriteLenAtEnd(int length) { WriteLength_FourBytes(length, Offset - length - 4); //向预留的空间内写入数据的长度 } }
写入器写入常见类型

/// <summary> /// KLV写入器 /// </summary> public partial class KLVWriter { readonly object _lock = new object(); #region 基础类型数据 public void WriteBoolem(byte key, bool val) { lock (_lock) { WriteKeyAndLen(key, 1); WriteVal((byte)(val ? 1 : 0)); } } public void WriteInt8(byte key, sbyte val) { lock (_lock) { WriteKeyAndLen(key, 1); WriteVal((byte)val); } } public void WriteUInt8(byte key, byte val) { lock (_lock) { WriteKeyAndLen(key, 1); WriteVal(val); } } public void WriteInt16(byte key, short val) { lock (_lock) { WriteKeyAndLen(key, 2); WriteVal(val.GetBytes()); } } public void WriteUInt16(byte key, ushort val) { lock (_lock) { WriteKeyAndLen(key, 2); WriteVal(val.GetBytes()); } } public void WriteInt32(byte key, int val) { lock (_lock) { WriteKeyAndLen(key, 4); WriteVal(val.GetBytes()); } } public void WriteUInt32(byte key, uint val) { lock (_lock) { WriteKeyAndLen(key, 4); WriteVal(val.GetBytes()); } } public void WriteInt64(byte key, long val) { lock (_lock) { WriteKeyAndLen(key, 8); WriteVal(val.GetBytes()); } } public void WriteUInt64(byte key, ulong val) { lock (_lock) { WriteKeyAndLen(key, 8); WriteVal(val.GetBytes()); } } public void WriteFloat(byte key, float val) { lock (_lock) { WriteKeyAndLen(key, 4); WriteVal(val.GetBytes()); } } public void WriteDouble(byte key, double val) { lock (_lock) { WriteKeyAndLen(key, 8); WriteVal(val.GetBytes()); } } public void WriteString(byte key, string val) { lock (_lock) { byte[] bs = Encoding.UTF8.GetBytes(val); WriteKeyAndLen(key, bs.Length); WriteVal(bs); } } #endregion #region 基础类型集合数据 public void WriteBooleanArray(byte key, IList<bool> vals) { lock (_lock) { WriteKeyAndLen(key, vals.Count); foreach (var val in vals) { WriteVal((byte)(val ? 1 : 0)); } } } public void WriteInt8Array(byte key, IList<sbyte> vals) { lock (_lock) { WriteKeyAndLen(key, vals.Count); foreach (var val in vals) { WriteVal((byte)val); } } } public void WriteUInt8Array(byte key, byte[] vals) { lock (_lock) { WriteKeyAndLen(key, vals.Length); WriteVal(vals); } } public void WriteInt16Array(byte key, IList<short> vals) { lock (_lock) { WriteKeyAndLen(key, vals.Count * 2); foreach (var val in vals) { WriteVal(val.GetBytes()); } } } public void WriteUInt16Array(byte key, IList<ushort> vals) { lock (_lock) { WriteKeyAndLen(key, vals.Count * 2); foreach (var val in vals) { WriteVal(val.GetBytes()); } } } public void WriteInt32Array(byte key, IList<int> vals) { lock (_lock) { WriteKeyAndLen(key, vals.Count * 4); foreach (var val in vals) { WriteVal(val.GetBytes()); } } } public void WriteUInt32Array(byte key, IList<uint> vals) { lock (_lock) { WriteKeyAndLen(key, vals.Count * 4); foreach (var val in vals) { WriteVal(val.GetBytes()); } } } public void WriteInt64Array(byte key, IList<long> vals) { lock (_lock) { WriteKeyAndLen(key, vals.Count * 8); foreach (var val in vals) { WriteVal(val.GetBytes()); } } } public void WriteUInt64Array(byte key, IList<ulong> vals) { lock (_lock) { WriteKeyAndLen(key, vals.Count * 8); foreach (var val in vals) { WriteVal(val.GetBytes()); } } } public void WriteFloatArray(byte key, IList<float> vals) { lock (_lock) { WriteKeyAndLen(key, vals.Count * 4); foreach (var val in vals) { WriteVal(val.GetBytes()); } } } public void WriteDoubleArray(byte key, IList<double> vals) { lock (_lock) { WriteKeyAndLen(key, vals.Count * 8); foreach (var val in vals) { WriteVal(val.GetBytes()); } } } #endregion /// <summary> /// 写入一个对象,必须在委托函数中写入,目的是限制写入顺序,避免出现数据混乱 /// </summary> public void WriteObject<T>(byte key, T val, Action<T, KLVWriter> writer) where T : class { lock (_lock) { if (val == null) { WriteKeyAndLen(key, 0); } else { WriteKey(key); WriteValAndLen(val, new KLVWriter(), writer); } } } /// <summary> /// 写入一个字符串集合 /// </summary> public void WriteStringArray(byte key, IList<string> vals) { lock (_lock) { WriteKey(key); Offset += 4; int count = 0; foreach (var val in vals) { count += WriteValAndLen(Encoding.UTF8.GetBytes(val)); } WriteLenAtEnd(count); } } /// <summary> /// 写入一个对象集合 /// </summary> public void WriteObjectList<T>(byte key, IEnumerable<T> vals, Action<T, KLVWriter> writer) { lock (_lock) { WriteKey(key); Offset += 4; int count = 0; KLVWriter kw = new KLVWriter(); foreach (var val in vals) { count += WriteValAndLen(val, kw, writer); } WriteLenAtEnd(count); } } /// <summary> /// 写入一个字典 /// </summary> public void WriteDictionary<K, V>(byte key, Dictionary<K, V> vals, Action<KeyValuePair<K, V>, KLVWriter> writer) { WriteObjectList(key, vals, writer); } }
读取器核心逻辑

/// <summary> /// 读取器 /// </summary> partial class KLVReader { /// <summary> /// 解析出来的数据 /// </summary> readonly Dictionary<byte, Block> dict = new Dictionary<byte, Block>(); /// <summary> /// 缓冲区 /// </summary> byte[] Data; /// <summary> /// 从缓冲区中指定位置开始指定长度中解析一个KLV结构,并提供读取操作 /// </summary> /// <param name="data">缓冲区数据</param> /// <param name="startIndex">开始索引</param> /// <param name="dataLength">数据长度</param> public KLVReader(byte[] data, int startIndex, int dataLength) { Parse(data, startIndex, dataLength); } internal KLVReader() { } /// <summary> /// 解析数据,提出来方便复用 /// </summary> public void Parse(byte[] data, int startIndex, int dataLength) { Data = data; dict.Clear(); int offset = startIndex; int offset_end = offset + dataLength; while (offset < offset_end) { //循环解析 byte key = data[offset++]; //先读取一个字节的key (int length, int size) = ReadLenght(data, offset); //在读取len offset += size; try { dict.Add(key, new Block(offset, length)); //最后读取val } catch (Exception ex) { Console.WriteLine(key); throw ex; } offset += length; }; } /// <summary> /// 读取KLV中的len,因为KLV采用了2中方式表示len,因此要特殊处理, /// 当作为len的首字节的最高位为0,则说明用1个字节表示len,这1个字节的后7位表示最多127个数值, /// 当作为len的首字节的最高位为1,则说明用4个字节表示len,这4个字节的后31位标识最多2的31次方的数值 /// </summary> (int, int) ReadLenght(byte[] data, int offset) { int size = 1; //默认认为是1个字节,最高位为0,用一个字节的后7位表示,小于0x80 int length = data[offset]; if (length >= 0x80) { //大于等于0x80就认为是用4字节 length = data.ToInt32(offset); length &= 0x7FFFFFFF; //因为最高位固定为1,数值只用了后31位,所以要将最高位置为0 size = 4; } return (length, size); } /// <summary> /// 从字典中查找指定key的相关数据 /// </summary> Block GetBlock(byte key) { if (!dict.TryGetValue(key, out Block block)) { throw new KLVException($"未找到key:{key},data:{BitConverter.ToString(Data)}"); } return block; } /// <summary> /// 根据block循环读取val集合 /// </summary> IEnumerable<(int, int)> ReadVals(Block block) { lock (_lock) { int offset = block.Offset; int offset_end = offset + block.Length; while (offset < offset_end) { (int len, int size) = ReadLenght(Data, offset); //每个字符串都有一个长度 offset += size; yield return (offset, len); offset += len; } } } }
读取器读取常见类型

public partial class KLVReader { readonly object _lock = new object(); #region 基础类型数据 public bool ReadBoolem(byte key) { lock (_lock) { return Data[GetBlock(key).Offset] == 1; } } public sbyte ReadInt8(byte key) { lock (_lock) { return (sbyte)Data[GetBlock(key).Offset]; } } public byte ReadUInt8(byte key) { lock (_lock) { return Data[GetBlock(key).Offset]; } } public short ReadInt16(byte key) { lock (_lock) { return Data.ToInt16(GetBlock(key).Offset); } } public ushort ReadUInt16(byte key) { lock (_lock) { return Data.ToUInt16(GetBlock(key).Offset); } } public int ReadInt32(byte key) { lock (_lock) { return Data.ToInt32(GetBlock(key).Offset); } } public uint ReadUInt32(byte key) { lock (_lock) { return Data.ToUInt32(GetBlock(key).Offset); } } public long ReadInt64(byte key) { lock (_lock) { return Data.ToInt64(GetBlock(key).Offset); } } public ulong ReadUInt64(byte key) { lock (_lock) { return Data.ToUInt64(GetBlock(key).Offset); } } public float ReadFloat(byte key) { lock (_lock) { return Data.ToFloat(GetBlock(key).Offset); } } public double ReadDouble(byte key) { lock (_lock) { return Data.ToDouble(GetBlock(key).Offset); } } public string ReadString(byte key) { lock (_lock) { var block = GetBlock(key); return Encoding.UTF8.GetString(Data, block.Offset, block.Length); } } #endregion #region 基础类型数组数据 public bool[] ReadBoolemArray(byte key) { lock (_lock) { var block = GetBlock(key); bool[] arr = new bool[block.Length]; for (int i = 0; i < arr.Length; i++) { arr[i] = Data[block.Offset + i] == 1; } return arr; } } public sbyte[] ReadInt8Array(byte key) { lock (_lock) { var block = GetBlock(key); sbyte[] arr = new sbyte[block.Length]; for (int i = 0; i < arr.Length; i++) { arr[i] = (sbyte)Data[block.Offset + i]; } return arr; } } public byte[] ReadUInt8Array(byte key) { lock (_lock) { var block = GetBlock(key); byte[] arr = new byte[block.Length]; for (int i = 0; i < arr.Length; i++) { arr[i] = Data[block.Offset + i]; } return arr; } } public short[] ReadInt16Array(byte key) { lock (_lock) { var block = GetBlock(key); int size = 2; short[] arr = new short[block.Length / size]; for (int i = 0; i < arr.Length; i++) { arr[i] = Data.ToInt16(block.Offset + i * size); } return arr; } } public ushort[] ReadUInt16Array(byte key) { lock (_lock) { var block = GetBlock(key); int size = 2; ushort[] arr = new ushort[block.Length / size]; for (int i = 0; i < arr.Length; i++) { arr[i] = Data.ToUInt16(block.Offset + i * size); } return arr; } } public int[] ReadInt32Array(byte key) { lock (_lock) { var block = GetBlock(key); int size = 4; int[] arr = new int[block.Length / size]; for (int i = 0; i < arr.Length; i++) { arr[i] = Data.ToInt32(block.Offset + i * size); } return arr; } } public uint[] ReadUInt32Array(byte key) { lock (_lock) { var block = GetBlock(key); int size = 4; uint[] arr = new uint[block.Length / size]; for (int i = 0; i < arr.Length; i++) { arr[i] = Data.ToUInt32(block.Offset + i * size); } return arr; } } public long[] ReadInt64Array(byte key) { lock (_lock) { var block = GetBlock(key); int size = 8; long[] arr = new long[block.Length / size]; for (int i = 0; i < arr.Length; i++) { arr[i] = Data.ToInt64(block.Offset + i * size); } return arr; } } public ulong[] ReadUInt64Array(byte key) { lock (_lock) { var block = GetBlock(key); int size = 8; ulong[] arr = new ulong[block.Length / size]; for (int i = 0; i < arr.Length; i++) { arr[i] = Data.ToUInt64(block.Offset + i * size); } return arr; } } public float[] ReadFloatArray(byte key) { lock (_lock) { var block = GetBlock(key); int size = 4; float[] arr = new float[block.Length / size]; for (int i = 0; i < arr.Length; i++) { arr[i] = Data.ToFloat(block.Offset + i * size); } return arr; } } public double[] ReadDoubleArray(byte key) { lock (_lock) { var block = GetBlock(key); int size = 8; double[] arr = new double[block.Length / size]; for (int i = 0; i < arr.Length; i++) { arr[i] = Data.ToDouble(block.Offset + i * size); } return arr; } } #endregion /// <summary> /// 读取对象 /// </summary> public T ReadObject<T>(byte key, Func<KLVReader, T> read) where T : class { lock (_lock) { var block = GetBlock(key); return read(new KLVReader(Data, block.Offset, block.Length)); } } /// <summary> /// 读取字符串集合 /// </summary> public List<string> ReadStringList(byte key) { lock (_lock) { var block = GetBlock(key); List<string> list = new List<string>(); //字符串长度不固定,因此不知道数量,这里使用集合 foreach (var (offset, len) in ReadVals(block)) { list.Add(Encoding.UTF8.GetString(Data, offset, len)); } return list; } } /// <summary> /// 读取对象集 /// </summary> public List<T> ReadObjectList<T>(byte key, Func<KLVReader, T> read) { lock (_lock) { List<T> list = new List<T>(); KLVReader kr = new KLVReader(); foreach (var (offset, len) in ReadVals(GetBlock(key))) { kr.Parse(Data, offset, len); //复用一个kr,提高性能 list.Add(read(kr)); } return list; } } /// <summary> /// 读取字典 /// </summary> public Dictionary<K, V> ReadDictionary<K, V>(byte key, Func<KLVReader, KeyValuePair<K, V>> read) { return ReadObjectList(key, read).ToDictionary(p => p.Key, p => p.Value); } }
读取时用到模型,现在改成结构体了

/// <summary> /// 解析出来的数据块信息 /// </summary> internal struct Block { /// <summary> /// 数据在缓冲区中的偏移量 /// </summary> public int Offset; /// <summary> /// 数据的字节长度 /// </summary> public int Length; /// <summary> /// /// </summary> public Block(int offset, int length) { Offset = offset; Length = length; } }
测试用模型

[ProtoContract] public class Entity { [ProtoMember(1)] public bool P1 { get; set; } [ProtoMember(2)] public sbyte P2 { get; set; } [ProtoMember(3)] public byte P3 { get; set; } [ProtoMember(4)] public short P4 { get; set; } [ProtoMember(5)] public ushort P5 { get; set; } [ProtoMember(6)] public int P6 { get; set; } [ProtoMember(7)] public uint P7 { get; set; } [ProtoMember(8)] public long P8 { get; set; } [ProtoMember(9)] public ulong P9 { get; set; } [ProtoMember(10)] public float P10 { get; set; } [ProtoMember(11)] public double P11 { get; set; } [ProtoMember(12)] public string P12 { get; set; } [ProtoMember(13)] public bool[] P13 { get; set; } [ProtoMember(14)] public sbyte[] P14 { get; set; } [ProtoMember(15)] public byte[] P15 { get; set; } [ProtoMember(16)] public short[] P16 { get; set; } [ProtoMember(17)] public ushort[] P17 { get; set; } [ProtoMember(18)] public int[] P18 { get; set; } [ProtoMember(19)] public uint[] P19 { get; set; } [ProtoMember(20)] public long[] P20 { get; set; } [ProtoMember(21)] public ulong[] P21 { get; set; } [ProtoMember(22)] public float[] P22 { get; set; } [ProtoMember(23)] public double[] P23 { get; set; } [ProtoMember(24)] public List<string> P24 { get; set; } [ProtoMember(25)] public EntitySub P25 { get; set; } [ProtoMember(26)] public List<EntitySub> P26 { get; set; } [ProtoMember(27)] public Dictionary<int, string> P27 { get; set; } [ProtoMember(28)] public Dictionary<string, int> P28 { get; set; } [ProtoMember(29)] public Dictionary<int, EntitySub> P29 { get; set; } public static Entity CreateTestInstance() { Entity entity = new Entity() { P1 = false, P2 = -126, P3 = 255, P4 = -1234, P5 = 2345, P6 = -345678, P7 = 45678910, P8 = -56789101112, P9 = 67891011121314, P10 = 3.1415f, P11 = 2.718281828459045, P12 = "测试啊123", P13 = new bool[] { true, false, true }, P14 = new sbyte[] { -1, 2, -3, 4 }, P15 = new byte[] { 200, 201, 203 }, P16 = new short[] { -17904, 17905, -17906 }, P17 = new ushort[] { 34567, 45612, 44323 }, P18 = new int[] { -1234, 4567, -5678 }, P19 = new uint[] { 1234567, 7890321, 567431 }, P20 = new long[] { -44444444, 55555555555, -66666666666 }, P21 = new ulong[] { 888888888, 99999999 }, P22 = new float[] { 2.34f, -45.1f, 78.4f }, P23 = new double[] { -123.321, 234.432, -567.897 }, P24 = new List<string> { "测试啊123", "测试啊abc", "测试啊+-*" }, P25 = new EntitySub() { P1 = 123, P2 = 1.23, P3 = "子实体类测试123", }, P26 = new List<EntitySub>{ new EntitySub() { P1 = 1, P2=1.111, P3 = "第一个" }, new EntitySub() { P1 = 2,P2=2.222, P3 = "第二个" }, new EntitySub() { P1 = 3,P2=3.333, P3 = "第三个" } }, P27 = new Dictionary<int, string>() { { 1,"aaa"}, { 2,"bbb"}, { 3,"ccc"}, }, P28 = new Dictionary<string, int>() { { "aaa",1}, { "bbb",2}, { "ccc",3}, }, P29 = new Dictionary<int, EntitySub> { { 1, new EntitySub() { P1 = 1, P2 = 1.1, P3 = "测试111", } }, { 2, new EntitySub() { P1 = 2, P2 = 2.2, P3 = "测试222", } } } }; return entity; } }
子模型

[ProtoContract] public class EntitySub { [ProtoMember(1)] public int P1 { get; set; } [ProtoMember(2)] public double P2 { get; set; } [ProtoMember(3)] public string P3 { get; set; } }
自定义异常

/// <summary> /// 自定义一个Ktv异常类,便于识别异常类型 /// </summary> public class KLVException : Exception { public KLVException(string errMessage) : base(errMessage) { } }
扩展方法,由于是目前还是使用net48,因此这个类库只能用netstandard2.0,很多net core的新性能提升无法享受

public static class BitConverterExtend { /// <summary> /// 得到short的字节数组 /// </summary> /// <param name="num"></param> /// <param name="bigEndian">是否为大端字节序,默认大端</param> /// <returns></returns> public static byte[] GetBytes(this short num, bool bigEndian = true) { if (bigEndian) { num = IPAddress.HostToNetworkOrder(num); //大端网络字节序 } return BitConverter.GetBytes(num); } /// <summary> /// 得到ushort的字节数组 /// </summary> /// <param name="num"></param> /// <param name="bigEndian">是否为大端字节序,默认大端</param> /// <returns></returns> public static byte[] GetBytes(this ushort num, bool bigEndian = true) { return ((short)num).GetBytes(bigEndian); //这样比反转数组快 } /// <summary> /// 得到int的字节数组 /// </summary> /// <param name="num"></param> /// <param name="bigEndian">是否为大端字节序,默认大端</param> /// <returns></returns> public static byte[] GetBytes(this int num, bool bigEndian = true) { if (bigEndian) { num = IPAddress.HostToNetworkOrder(num); } return BitConverter.GetBytes(num); } /// <summary> /// 得到uint的字节数组 /// </summary> /// <param name="num"></param> /// <param name="bigEndian">是否为大端字节序,默认大端</param> /// <returns></returns> public static byte[] GetBytes(this uint num, bool bigEndian = true) { return ((int)num).GetBytes(bigEndian); } /// <summary> /// 得到long的字节数组 /// </summary> /// <param name="num"></param> /// <param name="bigEndian">是否为大端字节序,默认大端</param> /// <returns></returns> public static byte[] GetBytes(this long num, bool bigEndian = true) { if (bigEndian) { num = IPAddress.HostToNetworkOrder(num); } return BitConverter.GetBytes(num); } /// <summary> /// 得到ulong的字节数组 /// </summary> /// <param name="num"></param> /// <param name="bigEndian">是否为大端字节序,默认大端</param> /// <returns></returns> public static byte[] GetBytes(this ulong num, bool bigEndian = true) { return ((long)num).GetBytes(bigEndian); } /// <summary> /// 得到float的字节数组 /// </summary> /// <param name="num"></param> /// <param name="bigEndian">是否为大端字节序,默认大端</param> /// <returns></returns> public static byte[] GetBytes(this float num, bool bigEndian = true) { byte[] arr = BitConverter.GetBytes(num); //本地是小端 if (bigEndian) { arr.ReverseOneself(); //反转为大端 } return arr; } /// <summary> /// 得到double的字节数组 /// </summary> /// <param name="num"></param> /// <param name="bigEndian">是否为大端字节序,默认大端</param> /// <returns></returns> public static byte[] GetBytes(this double num, bool bigEndian = true) { byte[] arr = BitConverter.GetBytes(num); if (bigEndian) { arr.ReverseOneself(); } return arr; } /// <summary> /// 从字节数组总的开始索引开始构建一个short数值 /// </summary> /// <param name="bs">外部传入的字节数组,内部不能更改其数据</param> /// <param name="startIndex">开始索引</param> /// <param name="bigEndian">是否是大端字节序,默认真</param> /// <returns></returns> public static short ToInt16(this byte[] bs, int startIndex, bool bigEndian = true) { short num = BitConverter.ToInt16(bs, startIndex); //本地是小端 if (bigEndian) { num = IPAddress.NetworkToHostOrder(num); //转为大端 } return num; } /// <summary> /// 从字节数组总的开始索引开始构建一个ushort数值 /// </summary> /// <param name="bs">外部传入的字节数组,内部不能更改其数据</param> /// <param name="startIndex">开始索引</param> /// <param name="bigEndian">是否是大端字节序,默认真</param> /// <returns></returns> public static ushort ToUInt16(this byte[] bs, int startIndex, bool bigEndian = true) { return (ushort)bs.ToInt16(startIndex, bigEndian); //因为没有IPAddress.NetworkToHostOrder,所以直接调用ToInt16转换 } /// <summary> /// 从字节数组总的开始索引开始构建一个int数值 /// </summary> /// <param name="bs">外部传入的字节数组,内部不能更改其数据</param> /// <param name="startIndex">开始索引</param> /// <param name="bigEndian">是否是大端字节序,默认真</param> /// <returns></returns> public static int ToInt32(this byte[] bs, int startIndex, bool bigEndian = true) { int num = BitConverter.ToInt32(bs, startIndex); if (bigEndian) { num = IPAddress.NetworkToHostOrder(num); } return num; } /// <summary> /// 从字节数组总的开始索引开始构建一个uint数值 /// </summary> /// <param name="bs">外部传入的字节数组,内部不能更改其数据</param> /// <param name="startIndex">开始索引</param> /// <param name="bigEndian">是否是大端字节序,默认真</param> /// <returns></returns> public static uint ToUInt32(this byte[] bs, int startIndex, bool bigEndian = true) { return (uint)bs.ToInt32(startIndex, bigEndian); } /// <summary> /// 从字节数组总的开始索引开始构建一个long数值 /// </summary> /// <param name="bs">外部传入的字节数组,内部不能更改其数据</param> /// <param name="startIndex">开始索引</param> /// <param name="bigEndian">是否是大端字节序,默认真</param> /// <returns></returns> public static long ToInt64(this byte[] bs, int startIndex, bool bigEndian = true) { long num = BitConverter.ToInt64(bs, startIndex); if (bigEndian) { num = IPAddress.NetworkToHostOrder(num); } return num; } /// <summary> /// 从字节数组总的开始索引开始构建一个ulong数值 /// </summary> /// <param name="bs">外部传入的字节数组,内部不能更改其数据</param> /// <param name="startIndex">开始索引</param> /// <param name="bigEndian">是否是大端字节序,默认真</param> /// <returns></returns> public static ulong ToUInt64(this byte[] bs, int startIndex, bool bigEndian = true) { return (ulong)(bs.ToInt64(startIndex, bigEndian)); } /// <summary> /// 从字节数组总的开始索引开始构建一个float数值 /// </summary> /// <param name="bs">外部传入的字节数组,内部不能更改其数据</param> /// <param name="startIndex">开始索引</param> /// <param name="bigEndian">是否是大端字节序,默认真</param> /// <returns></returns> public static float ToFloat(this byte[] bs, int startIndex, bool bigEndian = true) { if (bigEndian) { return BitConverter.ToSingle(bs.ReverseCopy(startIndex, 4), 0); } return BitConverter.ToSingle(bs, startIndex); } /// <summary> /// 从字节数组总的开始索引开始构建一个double数值 /// </summary> /// <param name="bs">外部传入的字节数组,内部不能更改其数据</param> /// <param name="startIndex">开始索引</param> /// <param name="bigEndian">是否是大端字节序,默认真</param> /// <returns></returns> public static double ToDouble(this byte[] bs, int startIndex, bool bigEndian = true) { if (bigEndian) { return BitConverter.ToDouble(bs.ReverseCopy(startIndex, 8), 0); } return BitConverter.ToDouble(bs, startIndex); } /// <summary> /// 反转传入的数组的数据 /// </summary> public static void ReverseOneself(this IList<byte> bytes) { int len = bytes.Count / 2; for (int i = 0; i < len; i++) { (bytes[bytes.Count - 1 - i], bytes[i]) = (bytes[i], bytes[bytes.Count - 1 - i]); } } /// <summary> /// 从bytes中反转拷贝数据,不破坏原bytes的数据 /// </summary> public static byte[] ReverseCopy(this IList<byte> bytes, int startIndex, int length) { byte[] arr = new byte[length]; for (int i = 0; i < length; i++) { arr[i] = bytes[startIndex + length - 1 - i]; } return arr; } }
【推荐】国内首个AI IDE,深度理解中文开发场景,立即下载体验Trae
【推荐】编程新体验,更懂你的AI,立即体验豆包MarsCode编程助手
【推荐】抖音旗下AI助手豆包,你的智能百科全书,全免费不限次数
【推荐】轻量又高性能的 SSH 工具 IShell:AI 加持,快人一步
· TypeScript + Deepseek 打造卜卦网站:技术与玄学的结合
· Manus的开源复刻OpenManus初探
· AI 智能体引爆开源社区「GitHub 热点速览」
· 从HTTP原因短语缺失研究HTTP/2和HTTP/3的设计差异
· 三行代码完成国际化适配,妙~啊~