Golang性能调优示例
1. 性能调优过程
性能调试总体思路
2. 常见的分析指标
1. Wall Time
即墙上时钟时间（wall clock time）：从进程从开始运行到结束，时钟走过的时间，这其中包含了进程在阻塞和等待状态的时间。
2. CPU Time
1. 用户CPU时间
就是用户的进程获得了CPU资源以后，在用户态执行的时间
2. 系统CPU时间
用户进程获得了CPU资源以后，在内核态的执行时间。
3. Block Time
4. Memery allocation
5. GC times/time spent
3. 调优示例
1. 源代码
optmization.go
package profiling
import (
"encoding/json"
"strconv"
"strings"
)
func createRequest() string {
payload := make([]int, 100, 100)
for i := 0; i < 100; i++ {
payload[i] = i
}
req := Request{"demo_transaction", payload}
v, err := json.Marshal(&req)
if err != nil {
panic(err)
}
return string(v)
}
func processRequest(reqs []string) []string {
reps := []string{}
for _, req := range reqs {
reqObj := &Request{}
reqObj.UnmarshalJSON([]byte(req))
// json.Unmarshal([]byte(req), reqObj)
var buf strings.Builder
for _, e := range reqObj.PayLoad {
buf.WriteString(strconv.Itoa(e))
buf.WriteString(",")
}
repObj := &Response{reqObj.TransactionID, buf.String()}
repJson, err := repObj.MarshalJSON()
//repJson, err := json.Marshal(&repObj)
if err != nil {
panic(err)
}
reps = append(reps, string(repJson))
}
return reps
}
structs.go
package profiling
type Request struct {
TransactionID string `json:"transaction_id"`
PayLoad []int `json:"payload"`
}
type Response struct {
TransactionID string `json:"transaction_id"`
Expression string `json:"exp"`
}
structs_easyjson.go
这段代码是自动生成的，不是手写的，用的是easyjson这个库，这个库的作用是反序列化json,比go lib中原始的json.Unmarshal要快很多。
// Code generated by easyjson for marshaling/unmarshaling. DO NOT EDIT.
package profiling
import (
json "encoding/json"
easyjson "github.com/mailru/easyjson"
jlexer "github.com/mailru/easyjson/jlexer"
jwriter "github.com/mailru/easyjson/jwriter"
)
// suppress unused package warning
var (
_ *json.RawMessage
_ *jlexer.Lexer
_ *jwriter.Writer
_ easyjson.Marshaler
)
func easyjson6a975c40DecodeCh47(in *jlexer.Lexer, out *Response) {
isTopLevel := in.IsStart()
if in.IsNull() {
if isTopLevel {
in.Consumed()
}
in.Skip()
return
}
in.Delim('{')
for !in.IsDelim('}') {
key := in.UnsafeString()
in.WantColon()
if in.IsNull() {
in.Skip()
in.WantComma()
continue
}
switch key {
case "transaction_id":
out.TransactionID = string(in.String())
case "exp":
out.Expression = string(in.String())
default:
in.SkipRecursive()
}
in.WantComma()
}
in.Delim('}')
if isTopLevel {
in.Consumed()
}
}
func easyjson6a975c40EncodeCh47(out *jwriter.Writer, in Response) {
out.RawByte('{')
first := true
_ = first
{
const prefix string = ",\"transaction_id\":"
if first {
first = false
out.RawString(prefix[1:])
} else {
out.RawString(prefix)
}
out.String(string(in.TransactionID))
}
{
const prefix string = ",\"exp\":"
if first {
first = false
out.RawString(prefix[1:])
} else {
out.RawString(prefix)
}
out.String(string(in.Expression))
}
out.RawByte('}')
}
// MarshalJSON supports json.Marshaler interface
func (v Response) MarshalJSON() ([]byte, error) {
w := jwriter.Writer{}
easyjson6a975c40EncodeCh47(&w, v)
return w.Buffer.BuildBytes(), w.Error
}
// MarshalEasyJSON supports easyjson.Marshaler interface
func (v Response) MarshalEasyJSON(w *jwriter.Writer) {
easyjson6a975c40EncodeCh47(w, v)
}
// UnmarshalJSON supports json.Unmarshaler interface
func (v *Response) UnmarshalJSON(data []byte) error {
r := jlexer.Lexer{Data: data}
easyjson6a975c40DecodeCh47(&r, v)
return r.Error()
}
// UnmarshalEasyJSON supports easyjson.Unmarshaler interface
func (v *Response) UnmarshalEasyJSON(l *jlexer.Lexer) {
easyjson6a975c40DecodeCh47(l, v)
}
func easyjson6a975c40DecodeCh471(in *jlexer.Lexer, out *Request) {
isTopLevel := in.IsStart()
if in.IsNull() {
if isTopLevel {
in.Consumed()
}
in.Skip()
return
}
in.Delim('{')
for !in.IsDelim('}') {
key := in.UnsafeString()
in.WantColon()
if in.IsNull() {
in.Skip()
in.WantComma()
continue
}
switch key {
case "transaction_id":
out.TransactionID = string(in.String())
case "payload":
if in.IsNull() {
in.Skip()
out.PayLoad = nil
} else {
in.Delim('[')
if out.PayLoad == nil {
if !in.IsDelim(']') {
out.PayLoad = make([]int, 0, 8)
} else {
out.PayLoad = []int{}
}
} else {
out.PayLoad = (out.PayLoad)[:0]
}
for !in.IsDelim(']') {
var v1 int
v1 = int(in.Int())
out.PayLoad = append(out.PayLoad, v1)
in.WantComma()
}
in.Delim(']')
}
default:
in.SkipRecursive()
}
in.WantComma()
}
in.Delim('}')
if isTopLevel {
in.Consumed()
}
}
func easyjson6a975c40EncodeCh471(out *jwriter.Writer, in Request) {
out.RawByte('{')
first := true
_ = first
{
const prefix string = ",\"transaction_id\":"
if first {
first = false
out.RawString(prefix[1:])
} else {
out.RawString(prefix)
}
out.String(string(in.TransactionID))
}
{
const prefix string = ",\"payload\":"
if first {
first = false
out.RawString(prefix[1:])
} else {
out.RawString(prefix)
}
if in.PayLoad == nil && (out.Flags&jwriter.NilSliceAsEmpty) == 0 {
out.RawString("null")
} else {
out.RawByte('[')
for v2, v3 := range in.PayLoad {
if v2 > 0 {
out.RawByte(',')
}
out.Int(int(v3))
}
out.RawByte(']')
}
}
out.RawByte('}')
}
// MarshalJSON supports json.Marshaler interface
func (v Request) MarshalJSON() ([]byte, error) {
w := jwriter.Writer{}
easyjson6a975c40EncodeCh471(&w, v)
return w.Buffer.BuildBytes(), w.Error
}
// MarshalEasyJSON supports easyjson.Marshaler interface
func (v Request) MarshalEasyJSON(w *jwriter.Writer) {
easyjson6a975c40EncodeCh471(w, v)
}
// UnmarshalJSON supports json.Unmarshaler interface
func (v *Request) UnmarshalJSON(data []byte) error {
r := jlexer.Lexer{Data: data}
easyjson6a975c40DecodeCh471(&r, v)
return r.Error()
}
// UnmarshalEasyJSON supports easyjson.Unmarshaler interface
func (v *Request) UnmarshalEasyJSON(l *jlexer.Lexer) {
easyjson6a975c40DecodeCh471(l, v)
}
optimization_test.go
package profiling
import "testing"
func TestCreateRequest(t *testing.T) {
str := createRequest()
t.Log(str)
}
func TestProcessRequest(t *testing.T) {
reqs := []string{}
reqs = append(reqs, createRequest())
reps := processRequest(reqs)
t.Log(reps[0])
}
func BenchmarkProcessRequest(b *testing.B) {
reqs := []string{}
reqs = append(reqs, createRequest())
b.ResetTimer()
for i := 0; i < b.N; i++ {
_ = processRequest(reqs)
}
b.StopTimer()
}
以上的代码主要是对比使用easyjson与go原装的json.Unmarshal,使用性能调优工具直观的看出easyjson的性能更佳。
2. 调优步骤
1. 生成cpu.prof文件
## benchmark不理解的在博客中寻找讲golang测试的章节，有队benchmark对一个介绍。
go test -bench=. -cpuprofile=cpu.prof
go test -bench=. -memprofile=mem.prof
2. 执行pprof调优
1. cpu调优示例
go tool pprof cpu.prof
File: ch47.test
Type: cpu
Time: Aug 2, 2020 at 2:27pm (BST)
Duration: 1.41s, Total samples = 1.25s (88.59%)
Entering interactive mode (type "help" for commands, "o" for options)
(pprof) list processRequest
Total: 1.25s
ROUTINE ======================== go_learning/code/ch47.processRequest in /root/codes/go/src/go_learning/code/ch47/optmization.go
30ms 1.04s (flat, cum) 83.20% of Total
. . 20:}
. . 21:
. . 22:func processRequest(reqs []string) []string {
. . 23: reps := []string{}
. . 24: for _, req := range reqs {
. 10ms 25: reqObj := &Request{}
. 140ms 26: reqObj.UnmarshalJSON([]byte(req))
. 320ms 27: json.Unmarshal([]byte(req), reqObj)
. . 28:
. . 29: var buf strings.Builder
. . 30: var ret string
. . 31: for _, e := range reqObj.PayLoad {
20ms 290ms 32: ret += strconv.Itoa(e) + ","
. 50ms 33: buf.WriteString(strconv.Itoa(e))
. . 34: buf.WriteString(",")
. . 35: }
10ms 30ms 36: repObj := &Response{reqObj.TransactionID, buf.String()}
. . 37: repObj = &Response{reqObj.TransactionID, ret}
. 20ms 38: repJson, err := repObj.MarshalJSON()
. 160ms 39: repJson, err = json.Marshal(&repObj)
. . 40: if err != nil {
. . 41: panic(err)
. . 42: }
. 20ms 43: reps = append(reps, string(repJson))
. . 44: }
. . 45: return reps
. . 46:}
从结果上来看可见processRequest 这个方法的CPU占用的总时间是1.26s ,其中占用时间最长的是json.Unmarshal([]byte(req), reqObj)，使用了680ms，这也是golang自己的官方lib中的方法json.Unmarshal，相较于上面的reqObj.UnmarshalJSON([]byte(req))用的时间（230ms），用easyJson去做反序列化，看出性能上是快了不少，几乎是两倍的差距。第二个占用较多cpu资源的就是ret += strconv.Itoa(e) + ","，使用string直接拼接的方式确实是不赞同，尤其是在大量字符串拼接的场景下，对cpu和内存都是比较大的消耗，具体原因在下面的内存调优中会解释。
2.内存调优示例
go tool pprof mem.prof
File: ch47.test
Type: alloc_space
Time: Aug 2, 2020 at 2:13pm (BST)
Entering interactive mode (type "help" for commands, "o" for options)
(pprof) list processRequest
Total: 32.51MB
ROUTINE ======================== go_learning/code/ch47.processRequest in /root/codes/go/src/go_learning/code/ch47/optmization.go
28.01MB 32.51MB (flat, cum) 100% of Total
. . 21:
. . 22:func processRequest(reqs []string) []string {
. . 23: reps := []string{}
. . 24: for _, req := range reqs {
. . 25: reqObj := &Request{}
1MB 3.50MB 26: reqObj.UnmarshalJSON([]byte(req))
1.50MB 1.50MB 27: json.Unmarshal([]byte(req), reqObj)
. . 28:
. . 29: var buf strings.Builder
. . 30: var ret string
. . 31: for _, e := range reqObj.PayLoad {
25.51MB 25.51MB 32: ret += strconv.Itoa(e) + ","
. 1MB 33: buf.WriteString(strconv.Itoa(e))
. . 34: buf.WriteString(",")
. . 35: }
. . 36: repObj := &Response{reqObj.TransactionID, buf.String()}
. . 37: repObj = &Response{reqObj.TransactionID, ret}
. . 38: repJson, err := repObj.MarshalJSON()
. 1MB 39: repJson, err = json.Marshal(&repObj)
. . 40: if err != nil {
. . 41: panic(err)
. . 42: }
. . 43: reps = append(reps, string(repJson))
. . 44: }
从结果上看最消耗内存的是ret += strconv.Itoa(e) + ",",这个要比较好理解，go的字符串拼接会消耗很大的内存，go的string底层其实是一个slice，熟悉go的slice原理的都知道，slice的append操作是要另寻一块连续的内存地址的，正因为要另开辟一块连续的内存空间才使内存消耗看起来如此巨大，这里采取的策略是使用buffer的WriteString方法，go的buffer的扩充方式有三种，这里不依依赘述了，以后我会开专栏去讲解的go的buffer，在这里只是提供了一种解决方案而已，本篇的核心思想是提现如何去调试你的代码的性能。
4. 总结
性能调优其实是一门很大的学问，对于程序员本身来说要求比较高，除了熟练的掌握语言本身以外，还必须要有一定的经验，这样才能在调试中快速的定位问题以及处理问题。
stay hungry,stay foolish
