protobuf-grpc
Quick Start
1. protobuf & JSON
download protoc-3.14.0-win64.zip
unzip, add to env variable
go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.26
go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.1
helloworld.proto
syntax = "proto3"; option go_package = "./;helloworld"; // helloworld is the package name message HelloRequest { string name = 1; // 1是编号不是值 int32 age = 2; repeated string courses = 3; }
generate source code: cd to proto folder
protoc --go_out=. helloworld.proto protoc --go-grpc_out=. helloworld.proto // to use grpc
helloworld.pb.go:
// Code generated by protoc-gen-go. DO NOT EDIT. // versions: // protoc-gen-go v1.26.0 // protoc v3.14.0 // source: helloworld.proto package helloworld
helloworld_grpc.pb.go
// Code generated by protoc-gen-go-grpc. DO NOT EDIT. // versions: // - protoc-gen-go-grpc v1.3.0 // - protoc v3.14.0 // source: helloworld.proto package proto // now I change the package to proto
quick start and test the performance
package main import ( helloworld "GoProjects/helloworld/proto" "encoding/json" "fmt" "github.com/golang/protobuf/proto" ) type Hello struct { Name string `json:"name"` Age int32 `json:"age"` Courses []string `json:"courses"` } func main() { // compare the compression rate of proto with json req := helloworld.HelloRequest{Name: "bobby", Age: 12, Courses: []string{"miminan", "sss", "qqq"}} rsp, _ := proto.Marshal(&req) fmt.Println(len(rsp)) // test unmarshal newReq := helloworld.HelloRequest{} _ = proto.Unmarshal(rsp, &newReq) fmt.Println(newReq.Name) // json JsonStruct := Hello{Name: "zhuzhu", Age: 12, Courses: []string{"miminan", "sss", "qqq"}} JsonRsp, _ := json.Marshal(JsonStruct) fmt.Println(len(JsonRsp)) }
2. protobuf-grpc demo
attention: message里atrr有默认值，不可以赋值nil进去
proto/helloworld.proto
syntax = "proto3"; option go_package = "./;proto"; service Greeter { rpc SayHello (HelloRequest) returns (HelloReply); } message HelloRequest { string name = 1; } message HelloReply { string message = 1; }
generate source code:
protoc --go_out=. helloworld.proto protoc --go-grpc_out=. helloworld.proto
server/server.go
proto.UnimplementedGreeterServer的解释：
Go语言中没有inheritance，instead，use nest and composition
proto.UnimplementedGreeterServer中实现了所有methods，那么Server1就不需要实现所有接口方法才能使用了
package main import ( "GoProjects/grpc_test/proto" "context" "google.golang.org/grpc" "net" ) type Server1 struct { proto.UnimplementedGreeterServer // } func (s *Server1) SayHello(ctx context.Context, request *proto.HelloRequest) (*proto.HelloReply, error) { return &proto.HelloReply{Message: request.Name}, nil } func main() { g := grpc.NewServer() proto.RegisterGreeterServer(g, &Server1{}) // grpc-server, implemented server lis, err := net.Listen("tcp", ":1234") if err != nil { panic("listen error") } err = g.Serve(lis) if err != nil { panic("fail to start grpc") } }
client/client.go
package main import ( "GoProjects/grpc_test/proto" "context" "fmt" "google.golang.org/grpc" ) func main() { conn, err := grpc.Dial(":1234", grpc.WithInsecure()) if err != nil { panic("conn error") } defer conn.Close() c := proto.NewGreeterClient(conn) reply, err := c.SayHello(context.Background(), &proto.HelloRequest{ Name: "boby", }) if err != nil { panic(err) } fmt.Println(reply.Message) }
3. streaming grpc
1. simple rpc
2. server-side streaming rpc
cliend -> stock code; server -> continuous info
3.client-side streaming rpc
monitoring client
4.bidirectional streaming rpc
chat service
proto:
syntax = "proto3"; option go_package = "./;proto"; service Greeter { rpc GetStream (StreamReqData) returns (stream StreamRspData); // server-side rpc PutStream (stream StreamReqData) returns (StreamRspData); // client-side rpc BiStream (stream StreamReqData) returns (stream StreamRspData); // bi- } message StreamReqData { string name = 1; } message StreamRspData { string message = 1; }
server
package main import ( "GoProjects/stream_grpc_test/proto" "fmt" "google.golang.org/grpc" "net" "sync" "time" ) type Server1 struct { proto.UnimplementedGreeterServer // } func (s *Server1) PutStream(cli proto.Greeter_PutStreamServer) error { fmt.Println("enter server-side...") for { fmt.Println("for-loop in server-side...") if m, err := cli.Recv(); err != nil { panic(err) } else { fmt.Println(m.Name) } } return nil } func (s *Server1) BiStream(cli proto.Greeter_BiStreamServer) error { var wg sync.WaitGroup wg.Add(2) go func() { defer wg.Done() for { data, _ := cli.Recv() fmt.Println("server receive: ", data.Name) } }() go func() { defer wg.Done() for { _ = cli.Send(&proto.StreamRspData{Message: "server send: miminan"}) time.Sleep(time.Second) } }() wg.Wait() return nil } func (s *Server1) GetStream(req *proto.StreamReqData, res proto.Greeter_GetStreamServer) error { i := 0 for { i++ _ = res.Send(&proto.StreamRspData{Message: fmt.Sprintf("%v", time.Now().Unix())}) time.Sleep(time.Second) if i > 10 { break } } return nil } const PORT = "127.0.0.1:50052" func main() { g := grpc.NewServer() proto.RegisterGreeterServer(g, &Server1{}) // grpc-server, implemented server lis, err := net.Listen("tcp", PORT) if err != nil { panic("listen error") } err = g.Serve(lis) if err != nil { panic("fail to start grpc") } }
client
common prehend:
conn, err := grpc.Dial("127.0.0.1:50052", grpc.WithInsecure()) if err != nil { panic("conn error") } defer conn.Close() var c = proto.NewGreeterClient(conn)
client stream - server receive
cli, err := c.PutStream(context.Background()) i := 0 for { i++ fmt.Println("client ready to send") err = cli.Send(&proto.StreamReqData{ Name: fmt.Sprintf("name is %d", i), }) if err != nil { panic(err) } time.Sleep(time.Second) if i > 3 { break } } r, err := cli.CloseAndRecv() if err != nil { fmt.Println(err) } fmt.Println(r.Message)
client receive- server stream
reply, err := c.GetStream(context.Background(), &proto.StreamReqData{ Name: "momo", }) for { m, err := reply.Recv() if err != nil { panic(err) } fmt.Println(m.Message) }
cient streaming - server streaming
//bi cli, err := c.BiStream(context.Background()) var wg sync.WaitGroup wg.Add(2) go func() { defer wg.Done() for { data, _ := cli.Recv() fmt.Println("client receive: ", data.Message) } }() go func() { defer wg.Done() for { _ = cli.Send(&proto.StreamReqData{Name: "client send: zhuzhunan"}) time.Sleep(time.Second) } }() wg.Wait()
Advanced
1. default value
2. package：cd 到 proto下，
option_package = "common/pkg/v1"
表示generate proto/common/pkg/v1下的v1包
场景：多个微服务公用common目录，生成到common目录中
3. string name = 1; 编码代表顺序，一定要保证编码正确
4. 1 proto中import other proto
proto/base/helloworld.proto: define Empty and Pong
syntax = "proto3"; message Empty{ } message Pong{ string id = 1; }
base/helloworld.proto: import
syntax = "proto3"; option go_package = "./;proto"; import "base/helloworld.proto"; service Greeter { rpc SayHello (HelloRequest) returns (HelloReply); rpc Ping(Empty) returns (Pong); }
4.2 proto中import "github/..."
import "google/protobuf/empty.proto"; rpc Ping(google.protobuf.Empty) returns (Pong);
click on the source code link: import the Empty to .go from "github.com/golang/protobuf/ptypes/empty";
5. nested message
message HelloReply { string message = 1; message Result{ string rst = 1; } repeated Result lst = 2; }
client.go中获取Result使用： proto.HelloReply_Result{}
为了获取Pong, 需要生成base.proto的源码，并且保持在同一个 package下
6. define enum
enum Gender{ MALE = 0; FEMAILE = 1; } message HelloRequest { string name = 1; Gender g = 2; }
获取： proto.Gender_FEMAILE
7. timestamp
.proto:
import "google/protobuf/timestamp.proto"; message HelloRequest { string name = 1; Gender g = 2; map<string,string> map = 3; google.protobuf.Timestamp time = 4; }
点进import找路径： option go_package = "github.com/golang/protobuf/ptypes/timestamp";
点进路径发现：
timestamppb "google.golang.org/protobuf/types/known/timestamppb" type Timestamp = timestamppb.Timestamp
点进路径发现New()
于是，在client.go实例化时：
import "timestamppb "google.golang.org/protobuf/types/known/timestamppb"" Time: timestamppb.New(time.Now()),
8. metadata
type MD map[string][]string
是map类型，value为string切片
client.go：在invoke 方法前把metadata设置塞进ctx
md := metadata.New(map[string]string{"myname": "metadata"}) ctx := metadata.NewOutgoingContext(context.Background(), md)
server.go：只取出“myname”：
if slice, ok := md["myname"]; ok { for idx, v := range slice { fmt.Printf("key: %d, value: %s \n", idx, v) } }
输出整个metadata内容
for key, value := range md { for _, v := range value { fmt.Printf("key: %s, value: %s \n", key, v) } }
9. Interception
server.go: write the code starting from UnaryInterception
interceptor := func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp any, err error) { fmt.Println("server use an interceptor") return handler(ctx, req) } so := grpc.UnaryInterceptor(interceptor) g := grpc.NewServer(so)
client.go
interceptor := func(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { fmt.Println("client has an interception...") now := time.Now() err := invoker(ctx, method, req, reply, cc) fmt.Println("cost ", time.Since(now)) return err } do := grpc.WithUnaryInterceptor(interceptor) // DailOption is a slice var slice []grpc.DialOption slice = append(slice, grpc.WithInsecure(), do) conn, err := grpc.Dial(":1234", slice...)
10. auth实现 (2 ways)
1. use metadata/interception
server.go; 注意!! metadata里面的map只能传lowercase!!! 现在appNm传到服务端就变成了appnm
interceptor := func(ctx context.Context, req any, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp any, err error) { fmt.Println("server use an interceptor") md, ok := metadata.FromIncomingContext(ctx) if !ok { return resp, status.Error(codes.Unauthenticated, "miss token info...") } //for key, value := range md { // for _, v := range value { // fmt.Printf("key: %s, value: %s \n", key, v) // } //} var ( appId string appNm string ) if id, ok := md["appid"]; ok { appId = id[0] fmt.Println(appId) } if nm, ok := md["appnm"]; ok { appNm = nm[0] fmt.Println(appNm) } if appId != "101" || appNm != "nxx" { return resp, status.Error(codes.Unauthenticated, "appid != \"101\" || appnm != \"nxx\"") } return handler(ctx, req) } so := grpc.UnaryInterceptor(interceptor) g := grpc.NewServer(so)
client.go
interceptor := func(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { fmt.Println("client has an interception...") now := time.Now() md := metadata.New(map[string]string{ "appid": "1011", "appnm": "nxx", }) ctx = metadata.NewOutgoingContext(ctx, md) err := invoker(ctx, method, req, reply, cc) fmt.Println("cost ", time.Since(now)) return err } do := grpc.WithUnaryInterceptor(interceptor) // DailOption is a slice var slice []grpc.DialOption slice = append(slice, grpc.WithInsecure(), do) conn, err := grpc.Dial(":1234", slice...)
2. client use grpc.WithPerRPCCredentials(customed implemented Credential{})
client.go
type Credential struct { } func (c Credential) GetRequestMetadata(ctx context.Context, uri ...string) (map[string]string, error) { return map[string]string{ "appid": "101", "appnm": "nxx", }, nil } func (c Credential) RequireTransportSecurity() bool { return false } func main() { do := grpc.WithPerRPCCredentials(Credential{}) var slice []grpc.DialOption
11. validator
download: protoc-gen-validate.zip , then Copy the exe file in the zip file to $GOROOT/bin
copy the validator.proto from github https://github.com/envoyproxy/protoc-gen-validate/blob/master/validate/validate.proto
new helloworld.proto
syntax = "proto3"; import "validate.proto"; option go_package=".;proto"; service Greeter { rpc SayHello (Person) returns (Person); } message Person { uint64 id = 1 [(validate.rules).uint64.gt = 999]; // gt means > string email = 2 [(validate.rules).string.email = true]; string name = 3 [(validate.rules).string = { pattern: "^[^[0-9]A-Za-z]+( [^[0-9]A-Za-z]+)*$",max_bytes: 256,}]; }
generate protoc -I . --go_out=. --go-grpc_out=. --validate_out="lang=go:." helloworld.proto
server:
package main import ( "GoProjects/validate_grpc/proto" "context" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "net" ) type Server struct { proto.UnimplementedGreeterServer } func (s *Server) SayHello(ctx context.Context, request *proto.Person) (*proto.Person, error) { return &proto.Person{ Id: 32, // return fixed id }, nil } type Validator interface { Validate() error } func main() { var interceptor grpc.UnaryServerInterceptor interceptor = func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp interface{}, err error) { // 继续处理请求 if r, ok := req.(Validator); ok { // validator plugin will generate validate()
//to each message, in this case,`Person` , so we can say Person implement Validator interface if err := r.Validate(); err != nil { return nil, status.Error(codes.InvalidArgument, err.Error()) } } return handler(ctx, req) } var opts []grpc.ServerOption opts = append(opts, grpc.UnaryInterceptor(interceptor)) g := grpc.NewServer(opts...) proto.RegisterGreeterServer(g, &Server{}) lis, err := net.Listen("tcp", "0.0.0.0:50051") if err != nil { panic("failed to listen:" + err.Error()) } err = g.Serve(lis) if err != nil { panic("failed to start grpc:" + err.Error()) } }
client:
package main import ( "GoProjects/validate_grpc/proto" "context" "fmt" "google.golang.org/grpc" ) type customCredential struct{} func main() { var opts []grpc.DialOption //opts = append(opts, grpc.WithUnaryInterceptor(interceptor)) opts = append(opts, grpc.WithInsecure()) conn, err := grpc.Dial("localhost:50051", opts...) if err != nil { panic(err) } defer conn.Close() c := proto.NewGreeterClient(conn) //rsp, _ := c.Search(context.Background(), &empty.Empty{}) rsp, err := c.SayHello(context.Background(), &proto.Person{ Id: 1000, Email: "bobby@123.com", Phone: "12222", // wrong }) if err != nil { panic(err) } fmt.Println(rsp.Id) }
12. error
1. status code list: https://github.com/grpc/grpc/blob/master/doc/statuscodes.md
2. send the error: status.Errorf(codes.NotFound, "test no fount, req: %s", request)
3. receive the error:
st, ok := status.FromError(err) if !ok { // Error was not a status error } st.Message() st.Code()
13. timeout mechanism
1. Network jitter, network congestion
2. The server is slow Client A->B->C cannot rely on the timeout mechanism of the receiving end and is irresponsible
ctx, _ = context.WithTimeout(ctx, time.Second*1)
