resurfaceio goreplay output-resurface 处理简单说明
resurfaceio goreplay output-resurface 的处理实际上就是开发了一个新的output 插件，对于数据的写入是使用了
resurfaceio 的golang logger sdk
实现简单说明
output_resurface.go，核心就是一个标准的goreplay plugin，通过go channel 实现消息处理，包装为一个自己的http message
之后通过 logger sdk 写入到resurfaceio api 服务中
- 消息处理
func (o *ResurfaceOutput) buildMessages() {
// Keep track of both requests and responses for matching
messages := make(map[string]*HTTPMessage, o.config.bufferSize)
// Manually check and remove orphaned requests/responses every 10 s
straysTicker := time.NewTicker(time.Second * 10)
// (Debug) Count messages received/sent
messageCounter := [3]int{0, 0, 0}
for {
select {
case msg := <-o.messages:
messageCounter[0]++
metaSlice := payloadMeta(msg.Meta)
// UUID shared by a given request and its corresponding response
messageID := byteutils.SliceToString(metaSlice[1])
message, messageFound := messages[messageID]
if !messageFound {
// Message timestamp
messageTimestamp := byteutils.SliceToString(metaSlice[2])
message = &HTTPMessage{}
messages[messageID] = message
messageTime, err := strconv.ParseInt(messageTimestamp, 10, 64)
if err == nil {
message.initTime = time.Unix(0, messageTime)
} else {
message.initTime = time.Now()
Debug(2, logPrefix, "Error parsing message timestamp", err.Error())
}
}
reqFound := message.request != nil
if !reqFound && isRequestPayload(msg.Meta) {
message.request = msg
reqFound = true
}
respFound := message.response != nil
if !respFound && !isRequestPayload(msg.Meta) {
message.response = msg
respFound = true
}
if reqFound && respFound {
o.httpMessages <- message
messageCounter[1]++
delete(messages, messageID)
}
case <-straysTicker.C:
if n := len(messages); n > 0 {
Debug(3, logPrefix, "Number of messages in queue:", n)
for id, message := range messages {
Debug(4, logPrefix, "Checking message:", id)
hasRequest := message.request != nil
hasResponse := message.response != nil
if ((hasRequest && !hasResponse) || (!hasRequest && hasResponse)) &&
time.Since(message.initTime) >= time.Minute*2 {
Debug(3, logPrefix, "STRAY MESSAGE:", id)
if Settings.Verbose > 3 {
if hasRequest {
Debug(4, logPrefix, "REQUEST:", byteutils.SliceToString(message.request.Meta))
Debug(5, logPrefix, "REQUEST:\n", byteutils.SliceToString(message.request.Data))
}
if hasResponse {
Debug(4, logPrefix, "RESPONSE:", byteutils.SliceToString(message.response.Meta))
Debug(5, logPrefix, "RESPONSE:\n", byteutils.SliceToString(message.response.Data))
}
}
delete(messages, id)
Debug(3, logPrefix, "MESSAGE", id, "DELETED")
messageCounter[2]++
}
}
}
if messageCounter[0]+messageCounter[1]+messageCounter[2] != 0 {
Debug(1, logPrefix, "messages received:", messageCounter[0],
", full messages sent:", messageCounter[1], ", orphans deleted:", messageCounter[2])
messageCounter = [3]int{0, 0, 0}
}
}
}
}
- 数据写入处理
func (o *ResurfaceOutput) sendMessages() {
for message := range o.httpMessages {
req, reqErr := http.ReadRequest(bufio.NewReader(bytes.NewReader(message.request.Data)))
if reqErr != nil {
continue
}
resp, respErr := http.ReadResponse(bufio.NewReader(bytes.NewReader(message.response.Data)), req)
if respErr != nil {
continue
}
reqMeta := payloadMeta(message.request.Meta)
respMeta := payloadMeta(message.response.Meta)
//Debug(4, "[OUTPUT][RESURFACE]", "Processing Message:", id)
if Settings.Verbose > 4 {
Debug(5, logPrefix, "Processing Request:", byteutils.SliceToString(reqMeta[1]))
Debug(6, logPrefix, byteutils.SliceToString(message.request.Data))
Debug(5, logPrefix, "Processing Response:", byteutils.SliceToString(respMeta[1]))
Debug(6, logPrefix, byteutils.SliceToString(message.response.Data))
}
reqTimestamp, _ := strconv.ParseInt(byteutils.SliceToString(reqMeta[2]), 10, 64)
respTimestamp, _ := strconv.ParseInt(byteutils.SliceToString(respMeta[2]), 10, 64)
interval := (respTimestamp - reqTimestamp) / 1000000
if interval < 0 {
interval = 0
}
resurfaceLogger.SendHttpMessage(o.rLogger, resp, req, respTimestamp/1000000, interval, nil)
}
}
说明
resurfaceio goreplay 的实现并不难，实际上我们也可以基于此实现一个graylog gelf 协议的
参考资料
https://resurface.io/docs
https://github.com/resurfaceio/goreplay
https://github.com/resurfaceio/logger-go
