kubebuilder开发kubernetes operator demo

环境准备

go环境配置

wget https://golang.google.cn/dl/go1.19.8.linux-amd64.tar.gz
tar zxvf go1.19.8.linux-amd64.tar.gz
mv go /usr/local/

vim /etc/profile在最结尾添加

export HOME=/root
export GOROOT=/usr/local/go
export GOPATH=/opt/idcus/go
export PATH=$PATH:$GOPATH/bin:$GOROOT/bin

source /etc/profile
go version #查看是否生效

配置kubebuilder

# 下载最新版本的kubebuilder
curl -L -o kubebuilder https://go.kubebuilder.io/dl/latest/$(go env GOOS)/$(go env GOARCH)
# 本次使用的v3.9.1会出现报错bug，所以使用v3.8.0
wget https://github.com/kubernetes-sigs/kubebuilder/releases/download/v3.8.0/kubebuilder_linux_amd64
chmod +x kubebuilder && mv kubebuilder /usr/local/bin/

验证版本

[root@test ~]# kubebuilder version
Version: main.version{KubeBuilderVersion:"3.8.0", KubernetesVendor:"1.25.0", GitCommit:"184ff7465947ced153b031db8de297a778cecf36", BuildDate:"2022-12-04T18:17:10Z", GoOs:"linux", GoArch:"amd64"}

创建k8s-host.yaml

kind: Cluster
apiVersion: kind.x-k8s.io/v1alpha4
name: host
nodes:
- role: control-plane
  image: kindest/node:v1.21.14@sha256:9d9eb5fb26b4fbc0c6d95fa8c790414f9750dd583f5d7cee45d92e8c26670aa1
- role: worker
  image: kindest/node:v1.21.14@sha256:9d9eb5fb26b4fbc0c6d95fa8c790414f9750dd583f5d7cee45d92e8c26670aa1
- role: worker
  image: kindest/node:v1.21.14@sha256:9d9eb5fb26b4fbc0c6d95fa8c790414f9750dd583f5d7cee45d92e8c26670aa1
networking:
  apiServerAddress: "127.0.0.1"
  apiServerPort: 6443

使用kind创建一个k8s集群

kind create cluster --config k8s-host.yaml

验证集群

[root@test ~]# kubectl get no
NAME                 STATUS   ROLES                  AGE   VERSION
host-control-plane   Ready    control-plane,master   12d   v1.21.14
host-worker          Ready    <none>                 12d   v1.21.14
host-worker2         Ready    <none>                 12d   v1.21.14

初始化项目

mkdir application-operator
cd application-operator
go mod init application-operator
kubebuilder init --domain=qwbyx.com --owner qwbyx
kubebuilder create api --group apps --version v1 --kind Application

CRD实现和部署

使用goland远程ssh模式进行开发
将api/v1/application_types.go中修改以下部分

import (
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

// EDIT THIS FILE!  THIS IS SCAFFOLDING FOR YOU TO OWN!
// NOTE: json tags are required.  Any new fields you add must have json tags for the fields to be serialized.

// ApplicationSpec defines the desired state of Application
type ApplicationSpec struct {
	// INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
	// Important: Run "make" to regenerate code after modifying this file

	// Foo is an example field of Application. Edit application_types.go to remove/update
	//Foo string `json:"foo,omitempty"`
	Replicas int32                  `json:"replicas,omitempty"`
	Template corev1.PodTemplateSpec `json:"template,omitempty"`
}

CRD部署

make install

验证

[root@test ~]# kubectl get crd
NAME                          CREATED AT
applications.apps.qwbyx.com   2023-04-14T09:15:49Z

CR部署
将config/samples/apps_v1_application.yaml改为如下

apiVersion: apps.qwbyx.com/v1
kind: Application
metadata:
  name: application-sample
  namespace: default
  labels:
    app: nginx
#  labels:
#    app.kubernetes.io/name: application
#    app.kubernetes.io/instance: application-sample
#    app.kubernetes.io/part-of: application-operator
#    app.kubernetes.io/managed-by: kustomize
#    app.kubernetes.io/created-by: application-operator
#  name: application-sample
spec:
  replicas: 3
  template:
    spec:
      containers:
        - name: nginx
          image: nginx:1.14.2
          ports:
            - containerPort: 80

进行部署

[root@test application-operator]# kubectl apply -f config/samples/apps_v1_application.yaml 
application.apps.qwbyx.com/application-sample created

查询

[root@test ~]# kubectl get application
NAME                 AGE
application-sample   3s

Controller的实现和部署

修改controllers/application_controller.go内容如下

package controllers

import (
	"context"
	"fmt"
	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"time"

	"k8s.io/apimachinery/pkg/runtime"
	ctrl "sigs.k8s.io/controller-runtime"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/log"

	dappsv1 "application-operator/api/v1"
)

// ApplicationReconciler reconciles a Application object
type ApplicationReconciler struct {
	client.Client
	Scheme *runtime.Scheme
}

//+kubebuilder:rbac:groups=apps.qwbyx.com,resources=applications,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=apps.qwbyx.com,resources=applications/status,verbs=get;update;patch
//+kubebuilder:rbac:groups=apps.qwbyx.com,resources=applications/finalizers,verbs=update

// Reconcile is part of the main kubernetes reconciliation loop which aims to
// move the current state of the cluster closer to the desired state.
// TODO(user): Modify the Reconcile function to compare the state specified by
// the Application object against the actual cluster state, and then
// perform operations to make the cluster state reflect the state specified by
// the user.
//
// For more details, check Reconcile and its Result here:
// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.13.1/pkg/reconcile
func (r *ApplicationReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
	l := log.FromContext(ctx)

	// get the Application
	app := &dappsv1.Application{}
	if err := r.Get(ctx, req.NamespacedName, app); err != nil {
		if errors.IsNotFound(err) {
			l.Info("the Application is not found")
			return ctrl.Result{}, nil
		}
		l.Error(err, "failed to get the Application")
		return ctrl.Result{RequeueAfter: 1 * time.Minute}, err
	}

	// create pods
	for i := 0; i < int(app.Spec.Replicas); i++ {
		pod := &corev1.Pod{
			ObjectMeta: metav1.ObjectMeta{
				Name:      fmt.Sprintf("%s-%d", app.Name, i),
				Namespace: app.Namespace,
				Labels:    app.Labels,
			},
			Spec: app.Spec.Template.Spec,
		}

		if err := r.Create(ctx, pod); err != nil {
			l.Error(err, "failed to create Pod")
			return ctrl.Result{RequeueAfter: 1 * time.Minute}, err
		}
		l.Info(fmt.Sprintf("the Pod (%s) has created", pod.Name))
	}

	l.Info("all pods has created")

	return ctrl.Result{}, nil
}

// SetupWithManager sets up the controller with the Manager.
func (r *ApplicationReconciler) SetupWithManager(mgr ctrl.Manager) error {
	return ctrl.NewControllerManagedBy(mgr).
		For(&dappsv1.Application{}).
		Complete(r)
}

启动controller

make run

查询

[root@test ~]# kubectl get po
NAME                   READY   STATUS    RESTARTS   AGE
application-sample-0   1/1     Running   0          23h
application-sample-1   1/1     Running   0          23h
application-sample-2   1/1     Running   0          23h

以容器形式部署controller
在Dockerfile第三行添加如下

ENV GOPROXY=https://goproxy.io

准备镜像

docker pull kubeimages/distroless-static
docker tag kubeimages/distroless-static:latest gcr.io/distroless/static:nonroot
docker pull quay.io/brancz/kube-rbac-proxy:v0.13.1
docker tag quay.io/brancz/kube-rbac-proxy:v0.13.1 gcr.io/kubebuilder/kube-rbac-proxy:v0.13.1

将镜像注入k8s集群

make docker-build IMG=application-operator:v0.0.1
kind load docker-image application-operator:v0.0.1 --name host
kind load docker-image gcr.io/kubebuilder/kube-rbac-proxy:v0.13.1 --name host

部署

make deploy IMG=application-operator:v0.0.1

查询

[root@test ~]# kubectl get po -n application-operator-system
NAME                                                      READY   STATUS    RESTARTS   AGE
application-operator-controller-manager-c97f78f45-457s7   2/2     Running   0          28m

资源清理

卸载controller

make undeploy

卸载CRD

make uninstall