scala位压缩与行情转换二进制
import org.jboss.netty.buffer.{ChannelBuffers, ChannelBuffer} import java.nio.charset.Charset import BigDecimal.RoundingMode._ /* * 采用LittleEndian字节顺序。结构为控制字节+附加字节 * 控制字节 * 7 符号位 * 6-4 数据字节数量,在实际数据字节中保存原始数据的绝对值 * 3-0 特定小整数保留位,0-15的数字可以不使用数据字节直接在控制字节中表示 * * 范围截断。由于控制字节的限制,最多附加4bit + 7个字节(即60bit)的数据,如果超过范围,则进行截断。 */ class BitCompress(buf: ChannelBuffer) { def compressLong(in: Long) { //抽取符号位、如果为负值则取绝对值,同时进行范围截断 var (control, abs) = { val MaxAbsValue = 0x0fffffffffffffffl //60bit if (in < 0) (0x80.toByte, (-in) & MaxAbsValue) else (0x00.toByte, in & MaxAbsValue) } //附加数据缓冲区 val additionalBuffer = ChannelBuffers.buffer(8) //去除待压缩数据中多余的0 var additionalLength = 0 //为方便,将控制字节中的附加数据也作为一个附加数据处理,存储附加数据个数时应减1 if (abs == 0) additionalLength += 1 //0无法进入else中的循环,无法为附加数据长度赋值,因而特殊处理 else { val BytesOfLong = 8 for (i <- 1 to BytesOfLong) { val dataByte = ((abs >>> (BytesOfLong - i) * 8) & 0xff).toByte if ((additionalLength != 0) || (dataByte != 0)) { //有效数据开始:附加数据不为空或者字节数据不为0 if (additionalLength != 0) { additionalBuffer.writeByte(dataByte) additionalLength += 1 } else { //附加数据为空,有可能在控制字节中写入数据 //高4位有数据,为偶数个4bit,控制字节低4bit不用存放数据 //高4位没有数据,为奇数个4bit,控制字节低4bit需要存放数据 if ((dataByte & 0xf0) != 0) { additionalBuffer.writeByte(dataByte) additionalLength += 2 } else { control = (control | dataByte).toByte additionalLength += 1 } } } } } //附加数据长度 additionalLength -= 1 //为方便,将控制字节中的附加数据也作为一个附加数据处理,存储附加数据个数时应减1 control = (control | (additionalLength << 4)).toByte //写入结果 buf.writeByte(control) if (additionalLength != 0) buf.writeBytes(additionalBuffer) } //压缩数据 def <-<(in: Byte) { compressLong(in.toLong) } def <-<(in: Short) { compressLong(in.toLong) } def <-<(in: Int) { compressLong(in.toLong)} def <-<(in: Long) { compressLong(in) } //压缩BigDecimal数据 def <-<(in: BigDecimal) { compressLong(in.setScale(0, HALF_UP).toLong) } //压缩字符串(以0为结尾标志) def compressString(bytes: Array[Byte]) { buf.writeBytes(bytes) buf.writeByte(0) } def <-<(bytes: Array[Byte]) { compressString(bytes) } //写入数据 def <--(in: Byte) { buf.writeByte(in) } def <--(in: Short) { buf.writeShort(in) } def <--(in: Int) { buf.writeInt(in) } def <--(in: Long) { buf.writeLong(in) } def <--(in: Float) { buf.writeFloat(in) } def <--(in: Double) { buf.writeDouble(in) } //解压Long数据 def decompressLong() = { //控制字节 //7 符号位 //6-4 数据字节数,数据字节中保存原始数据的绝对值 //3-0 特定小整数保留位,0-15的数字可以不使用数据字节直接在控制字节中表示 val control = buf.readByte() val negative = (control & 0x80) == 0x80 //是否为负数 //附加数据长度 val additionalLength = (control & 0x70) >>> 4 //控制字节所包含的附加数据 var result: Long = control.toLong & 0x0f //解压 for (i <- 1 to additionalLength) { result <<= 8 result |= (buf.readByte().toLong & 0xffL) //byte转为Long时,如为负数则默认填充位为1 } //符号 if (negative) result = -result result } def decompressString(charsetName: String = "GBK") = { val dup = buf.duplicate() var length = 0 var done = false while (dup.readable() && !done) { if (dup.readByte() != 0) length += 1 else done = true } val result = buf.readBytes(length).toString(Charset.forName(charsetName)) buf.skipBytes(1) result } def readByte() = buf.readByte() def readShort() = buf.readShort() def readInt() = buf.readInt() def readLong() = buf.readLong() def readFloat() = buf.readFloat() def readDouble() = buf.readDouble() }
行情二进制写
import java.nio.ByteOrder import java.util import org.jboss.netty.buffer.ChannelBuffers class SnapshotBuffer() { def read(bytes: Array[Byte]): Snapshot = { null } def write(snap: Snapshot): Array[Byte] = { val bufferSize = 1024 * 1024 val buffer = ChannelBuffers.dynamicBuffer(ByteOrder.LITTLE_ENDIAN, bufferSize) val bc = new BitCompress(buffer) val points = 10000 bc.compressString(snap.from.getBytes("UTF-8")) bc <-- snap.totalSn bc <-- snap.exchangeSn bc <-- snap.varietySn val year = snap.dateTime.getYear val month = snap.dateTime.getMonthOfYear val day = snap.dateTime.getDayOfMonth val date = year * 10000 + month * 100 + day val hour = snap.dateTime.getHourOfDay val minute = snap.dateTime.getMinuteOfHour val second = snap.dateTime.getSecondOfMinute val mmsec = hour * 10000 + minute * 100 + second bc.compressLong(date) bc.compressLong(mmsec) bc.compressLong(snap.exchangeType.id) bc.compressLong(snap.varietyType.id) bc.compressLong(snap.marketStatus.id) bc.compressLong(snap.snapshots.size) snap.snapshots.foreach { item => bc <-< 4 //不压缩 bc <-- (item.dateTime.getMillis / 1000).toInt bc <-- item.marketId.id.toShort //字符串以0结束 bc.compressString(item.symbol.getBytes("UTF-8")) bc.compressString(item.name.getBytes("UTF-8")) bc <-< item.prevClose * points bc <-< (item.open - item.prevClose) * points bc <-< (item.high - item.prevClose) * points bc <-< (item.low - item.prevClose) * points bc <-< (item.close - item.prevClose) * points //不压缩 bc <-- item.volume.toFloat bc <-- item.amount.toFloat //保留2位压缩 bc <-< item.pe * 100 val bids = Array.fill(5)(new OrderBook(0, 0)) val asks = Array.fill(5)(new OrderBook(0, 0)) item.bids.toArray.copyToArray(bids) item.asks.toArray.copyToArray(asks) val buyPrice = bids(0).price bc <-< buyPrice * points bc <-< (bids(1).price - buyPrice) * points bc <-< (bids(2).price - buyPrice) * points bc <-< (bids(3).price - buyPrice) * points bc <-< (bids(4).price - buyPrice) * points bc <-< bids(0).lots bc <-< bids(1).lots bc <-< bids(2).lots bc <-< bids(3).lots bc <-< bids(4).lots bc <-< (asks(0).price - buyPrice) * points bc <-< (asks(1).price - buyPrice) * points bc <-< (asks(2).price - buyPrice) * points bc <-< (asks(3).price - buyPrice) * points bc <-< (asks(4).price - buyPrice) * points bc <-< asks(0).lots bc <-< asks(1).lots bc <-< asks(2).lots bc <-< asks(3).lots bc <-< asks(4).lots bc <-< item.tradeLots bc <-< { if (item.suspended) 1 else 0 } bc <-< item.holds bc <-< 0 } util.Arrays.copyOf(buffer.array, buffer.writerIndex) } }
其中copyOf方法是将buffer复制到一个新数组,但是把多分配的size删除。
行情二进制的抽取
object SnapshotWrapper { val defaultTimeZone = DateTimeZone.forID("Asia/Shanghai") def unapply(binary: Array[Byte]): Option[Snapshot] = { val c = new BitCompress(ChannelBuffers.wrappedBuffer(ByteOrder.LITTLE_ENDIAN, binary)) val from = c.decompressString("UTF-8") val sn = c.readLong() val dateTime = { val date = c.decompressLong().toInt val time = c.decompressLong().toInt DateTimeFormat.forPattern("yyyyMMddHHmmssZ").withOffsetParsed().parseDateTime( f"""$date%08d$time%06d+0800""" ) } val marketId = MarketId(c.decompressLong().toInt) //9:25 - 9:30 之间市场标志已经处于连续交易状态,但实际市场处于连续交易之前的准备阶段 //特殊处理一下,将9:30之前的连续交易状态处理成集合竞价状态 val marketStatus = { val cur = MarketStatus(c.decompressLong().toInt) val before930 = dateTime.getHourOfDay == 9 && dateTime.getMinuteOfHour < 30 val before9 = dateTime.getHourOfDay < 9 import MarketStatus._ if ( cur == ContinueTrade && (before930 || before9) ) { Auction } else { cur } } val count = c.decompressLong().toInt val buf = new ListBuffer[SnapshotItem] for (i <- 1 to count) { val divider = if (c.decompressLong() == 4) 10000 else 1000 //价格除数 val time = new DateTime(c.readInt()*1000L, defaultTimeZone) //date time val marketId = MarketId(c.readShort()) val symbol = c.decompressString() val cnName = c.decompressString() val prevClose = BigDecimal(c.decompressLong()) / divider val open = (BigDecimal(c.decompressLong()) / divider) + prevClose val high = (BigDecimal(c.decompressLong()) / divider) + prevClose val low = (BigDecimal(c.decompressLong()) / divider) + prevClose val close = (BigDecimal(c.decompressLong()) / divider) + prevClose val volume = BigDecimal(c.readFloat()) val amount = BigDecimal(c.readFloat()) val pe = BigDecimal(c.decompressLong()) / 100 //order books val (bids, asks) = orderBooks(c, divider) val tradeLots = c.decompressLong() //成交笔数 val suspended = if (c.decompressLong() == 0) false else true //停牌 val holds = c.decompressLong() //持仓 c.decompressLong() //unused buf += SnapshotItem( time, marketId, symbol, cnName, prevClose, open, high, low, close, volume, amount, pe, bids, asks, tradeLots, suspended, holds ) } Some(Snapshot( from, sn, dateTime, marketId, marketStatus, buf.toList )) } /** * 优化OrderBook性能 * * order book结构如下 * bids: price1 ... price5 lots1 ... lots5 * asks: price1 ... price5 lots1 ... lots5 */ @inline private def orderBooks(c: BitCompress, divider: Int) = { val base = c.decompressLong() val bidsPrice = Array( base, c.decompressLong() + base, c.decompressLong() + base, c.decompressLong() + base, c.decompressLong() + base ) val bidsLots = Array( c.decompressLong(), c.decompressLong(), c.decompressLong(), c.decompressLong(), c.decompressLong() ) val asksPrice = Array( c.decompressLong() + base, c.decompressLong() + base, c.decompressLong() + base, c.decompressLong() + base, c.decompressLong() + base ) val asksLots = Array( c.decompressLong(), c.decompressLong(), c.decompressLong(), c.decompressLong(), c.decompressLong() ) var bids: List[OrderBook] = Nil for (i <- 0 to 4) { if (bidsPrice(i) != 0) { bids = OrderBook( BigDecimal(bidsPrice(i)) / divider, BigDecimal(bidsLots(i)) ) :: bids } } var asks: List[OrderBook] = Nil for (i <- 0 to 4) { if (asksPrice(i) != 0) { asks = OrderBook( BigDecimal(asksPrice(i)) / divider, BigDecimal(asksLots(i)) ) :: asks } } (bids.reverse, asks.reverse) } }