kubernetes event 收集

kubernetes event 收集

背景

刚开始准备使用 kubernetes 的官方 python 库，但是这个 python 官方库一直落后于 kubernetes 的正式版本好几个版本,而且用这个库的时候监听 event 老是报错，所以决定使用 kubernetes 官方的 client-go 这个库。

代码介绍

我是参考 (kube-eventer)[https://github.com/AliyunContainerService/kube-eventer.git] 这个代码写的，没用这个是因为我想把我们几个集群的 event 都收集到一个 elasticsearch 的 index，然后通过集群名区分。

下面是改写好的代码，本文用的 client-go 的 git hash ea0a6e11838c4413b5d51574a50da6312d572658，集群名是通过 KUBECONFIG 这个环境变量传给程序的，比如 /ops/kubernetes/bj 集群名就是 bj。

package main

import (
	"context"
	"encoding/json"
	"flag"
	"fmt"
	"github.com/elastic/go-elasticsearch/v8"
	"github.com/elastic/go-elasticsearch/v8/esapi"
	"io/ioutil"
	kubeapi "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	kubewatch "k8s.io/apimachinery/pkg/watch"
	"k8s.io/client-go/kubernetes"
	v1 "k8s.io/client-go/kubernetes/typed/core/v1"
	"k8s.io/client-go/tools/clientcmd"
	"k8s.io/client-go/util/homedir"
	"k8s.io/klog"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"time"
)

const IndexName = "k8s-event"

type EsEvent struct {
	Cluster   string `json:"cluster"`
	Time      string `json:"time"`
	Namespace string `json:"namespace"`
	Kind      string `json:"kind"`
	Name      string `json:"name"`
	Type      string `json:"type"`
	Reason    string `json:"reason"`
	Message   string `json:"message"`
}

func NewEsEvent(event kubeapi.Event) (e EsEvent) {
	e.Cluster = GetClusterName()
	e.Namespace = event.InvolvedObject.Namespace
	e.Kind = event.InvolvedObject.Kind
	e.Name = event.InvolvedObject.Name
	e.Time = time.Now().Format(time.RFC3339)
	e.Type = event.Type
	e.Message = event.Message
	return
}

func GetClusterName() string {
	KUBECONFIG := os.Getenv("KUBECONFIG")
	if KUBECONFIG == "" {
		return "test"
	}
	return strings.Split(KUBECONFIG, "/")[len(strings.Split(KUBECONFIG, "/"))-1]
}

func (e *EsEvent) Req(es *elasticsearch.Client) {
	data, err := json.Marshal(e)
	if err != nil {
		klog.Fatal(err)
	}
	fmt.Printf("%s\n\n", data)
	req := esapi.IndexRequest{
		Index: IndexName,
		Body:  strings.NewReader(string(data)),
	}

	res, err := req.Do(context.Background(), es)
	CheckErr(err)
	defer res.Body.Close()

	if res.IsError() {
		body, _ := ioutil.ReadAll(res.Body)
		klog.Fatalln(string(body), res.Status())
	}
}

func CheckErr(err error) {
	if err != nil {
		klog.Fatalln(err)
	}
}

func main() {
	eventClient := GetKubeClient()
	es := GetEsClient()
	for {
		events, err := eventClient.List(metav1.ListOptions{})
		if err != nil {
			klog.Errorf("Failed to load events: %v", err)
			time.Sleep(time.Second)
			continue
		}
		// Do not write old events.

		resourceVersion := events.ResourceVersion

		watcher, err := eventClient.Watch(
			metav1.ListOptions{
				Watch:           true,
				ResourceVersion: resourceVersion})
		if err != nil {
			klog.Errorf("Failed to start watch for new events: %v", err)
			time.Sleep(time.Second)
			continue
		}

		watchChannel := watcher.ResultChan()
		// Inner loop, for update processing.
	inner_loop:
		for {
			select {
			case watchUpdate, ok := <-watchChannel:
				if !ok {
					klog.Errorf("Event watch channel closed")
					break inner_loop
				}

				if watchUpdate.Type == kubewatch.Error {
					if status, ok := watchUpdate.Object.(*metav1.Status); ok {
						klog.Errorf("Error during watch: %#v", status)
						break inner_loop
					}
					klog.Errorf("Received unexpected error: %#v", watchUpdate.Object)
					break inner_loop
				}

				if event, ok := watchUpdate.Object.(*kubeapi.Event); ok {
					switch watchUpdate.Type {
					case kubewatch.Added, kubewatch.Modified:
						data, err := json.Marshal(event)
						if err != nil {
							klog.Fatal(err)
						}
						fmt.Printf("%s\n\n", data)
						esevent := NewEsEvent(*event)
						esevent.Req(es)
					case kubewatch.Deleted:
						// Deleted events are silently ignored.
					default:
						klog.Warningf("Unknown watchUpdate.Type: %#v", watchUpdate.Type)
					}
				} else {
					klog.Errorf("Wrong object received: %v", watchUpdate)
				}

				//case <-this.stopChannel:
				//	klog.Infof("Event watching stopped")
				//	return
			}
		}
	}
}

func GetEsClient() (*elasticsearch.Client) {
	EsUrl := "http://192.168.56.11:9200"
	if runtime.GOOS == "darwin" {
		EsUrl = "http://es.wis.com"
	}
	cfg := elasticsearch.Config{
		Addresses: []string{
			EsUrl,
		},
	}
	es, err := elasticsearch.NewClient(cfg)
	if err != nil {
		klog.Fatalln(err)
	}
	return es
}

func GetKubeClient() (v1.EventInterface ) {
	var kubeconfig *string
	if home := homedir.HomeDir(); home != "" {
		kubeconfig = flag.String("kubeconfig", filepath.Join(home, ".kube", "config"), "(optional) absolute path to the kubeconfig file")
	} else {
		kubeconfig = flag.String("kubeconfig", "", "absolute path to the kubeconfig file")
	}
	flag.Parse()
	if KUBECONFIG := os.Getenv("KUBECONFIG"); KUBECONFIG != "" {
		*kubeconfig = KUBECONFIG
	}

	config, err := clientcmd.BuildConfigFromFlags("", *kubeconfig)
	if err != nil {
		panic(err)
	}
	clientset, err := kubernetes.NewForConfig(config)
	CheckErr(err)
	return clientset.CoreV1().Events(kubeapi.NamespaceAll)
}

总结

最后写完发现，之前用 python 库的时候应该是没做错误处理，所以收集 event 有问题，不过这个过程中自己也学习到了好多 go 的编程知识。

更新于20201120

上面是日志产生一条就往 elasticsearch 传一条，下面是简单实现了到一定时间或者事件数到设定的数就上传的伪逻辑：

package main

import (
	"fmt"
	"time"
)

const Size = 10

func in(c chan<- int) {
	base := 0
	for {
		base += 1
		select {
		case c <- base:
			wait := time.Second * time.Duration(base % 10)
			time.Sleep(wait)
		}

	}
}

func main() {
	var c1 = make(chan int, 100)

	t := time.Now()
	go in(c1)
	fmt.Println(cap(c1))
	for {
		elapseTime := time.Now().Sub(t).Seconds()
		if elapseTime > 3 {
			fmt.Printf("elapseTime(%.2f) biger 3\n", elapseTime)
		}
		if len(c1) == Size || elapseTime > 3 {
			t = time.Now()
			ProcessEvent(c1)
		}

	}

}

func ProcessEvent(c chan int) {
	for i := 0; i < len(c); i++ {
		v := <-c
		fmt.Println(v)
	}
}