Kubebuilder模块
CRD创建
Group表示CRD所属的组，它可以支持多种不同版本、不同类型的资源构建，Version表示CRD的版本号，Kind表示CRD的类型
kubebuilder create api --group ship --version v1beta1 --kind Demo
kubebuilder create api --group ship --version v1 --kind Test
kubebuilder create api --group ship --version v1beta1 --kind Test2生成的资源目录结构如下
[root@k8s-01 project2]# tree api/
api/
├── v1
│ ├── groupversion_info.go
│ ├── test_types.go
│ └── zz_generated.deepcopy.go
└── v1beta1
├── demo_types.go
├── groupversion_info.go
├── test2_types.go
└── zz_generated.deepcopy.go
2 directories, 7 files
[root@k8s-01 project2]#
[root@k8s-01 project2]# tree .
.
├── api
│ ├── v1
│ │ ├── groupversion_info.go
│ │ ├── test_types.go
│ │ └── zz_generated.deepcopy.go
│ └── v1beta1
│ ├── demo_types.go
│ ├── groupversion_info.go
│ ├── test2_types.go
│ └── zz_generated.deepcopy.go
├── bin
│ └── controller-gen
├── config
│ ├── crd
│ │ ├── kustomization.yaml
│ │ ├── kustomizeconfig.yaml
│ │ └── patches
│ │ ├── cainjection_in_demoes.yaml
│ │ ├── cainjection_in_test2s.yaml
│ │ ├── cainjection_in_tests.yaml
│ │ ├── webhook_in_demoes.yaml
│ │ ├── webhook_in_test2s.yaml
│ │ └── webhook_in_tests.yaml
│ ├── default
│ │ ├── kustomization.yaml
│ │ ├── manager_auth_proxy_patch.yaml
│ │ └── manager_config_patch.yaml
│ ├── manager
│ │ ├── controller_manager_config.yaml
│ │ ├── kustomization.yaml
│ │ └── manager.yaml
│ ├── prometheus
│ │ ├── kustomization.yaml
│ │ └── monitor.yaml
│ ├── rbac
│ │ ├── auth_proxy_client_clusterrole.yaml
│ │ ├── auth_proxy_role_binding.yaml
│ │ ├── auth_proxy_role.yaml
│ │ ├── auth_proxy_service.yaml
│ │ ├── demo_editor_role.yaml
│ │ ├── demo_viewer_role.yaml
│ │ ├── kustomization.yaml
│ │ ├── leader_election_role_binding.yaml
│ │ ├── leader_election_role.yaml
│ │ ├── role_binding.yaml
│ │ ├── service_account.yaml
│ │ ├── test2_editor_role.yaml
│ │ ├── test2_viewer_role.yaml
│ │ ├── test_editor_role.yaml
│ │ └── test_viewer_role.yaml
│ └── samples
│ ├── ship_v1beta1_demo.yaml
│ ├── ship_v1beta1_test2.yaml
│ └── ship_v1_test.yaml
├── controllers
│ ├── demo_controller.go
│ ├── suite_test.go
│ ├── test2_controller.go
│ └── test_controller.go
├── Dockerfile
├── go.mod
├── go.sum
├── hack
│ └── boilerplate.go.txt
├── main.go
├── Makefile
├── PROJECT
└── README.md
14 directories, 54 files{kind}controller.go，字段生成的Reconciler对象名称时{kind}Reconsiler，它的主要方法是Reconcile()，即通过在这个函数的空白处填入逻辑完成对应的CRD构造工作。 SetupWithManager方法，可以完成CRD在Manager对象中的安装，最后通过Manager对象的start方法来完成CRD Controller的运行。
//+kubebuilder:rbac:groups=ship.demo.domain,resources=tests,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=ship.demo.domain,resources=tests/status,verbs=get;update;patch
//+kubebuilder:rbac:groups=ship.demo.domain,resources=tests/finalizers,verbs=update
// Reconcile is part of the main kubernetes reconciliation loop which aims to
// move the current state of the cluster closer to the desired state.
// TODO(user): Modify the Reconcile function to compare the state specified by
// the Test object against the actual cluster state, and then
// perform operations to make the cluster state reflect the state specified by
// the user.
//
// For more details, check Reconcile and its Result here:
// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.12.1/pkg/reconcile
func (r *TestReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
_ = log.FromContext(ctx)
// TODO(user): your logic here
return ctrl.Result{}, nil
}
// SetupWithManager sets up the controller with the Manager.
func (r *TestReconciler) SetupWithManager(mgr ctrl.Manager) error {
return ctrl.NewControllerManagedBy(mgr).
For(&shipv1.Test{}).
Complete(r)
}Manager初始化
// controller-tuntime/pkg/manager/manager.go 文件中的New方法
Controller初始化
通过SetupWithManager方法，就可以完成CRD在Manager对象中的安装，最后通过Manager对象的start方法来完成CRD Controller的运行
// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.12.1/pkg/reconcile
func (r *TestReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
_ = log.FromContext(ctx)
// TODO(user): your logic here
return ctrl.Result{}, nil
}
// SetupWithManager sets up the controller with the Manager.
func (r *TestReconciler) SetupWithManager(mgr ctrl.Manager) error {
return ctrl.NewControllerManagedBy(mgr).
For(&shipv1.Test{}).
Complete(r)
}它首先借助Controller-runtime包初始化Builder对象，当它完成Complete方法时，实例完成了CRD Reconciler对象的初始化，而这个对象是一个接口方法，它必须实现Reconcile方法。（代码片段）
// pkg/builder/controller.go
// Complete builds the Application Controller.
func (blder *Builder) Complete(r reconcile.Reconciler) error {
_, err := blder.Build(r)
return err
}
// Build builds the Application Controller and returns the Controller it created.
func (blder *Builder) Build(r reconcile.Reconciler) (controller.Controller, error) {
if r == nil {
return nil, fmt.Errorf("must provide a non-nil Reconciler")
}
if blder.mgr == nil {
return nil, fmt.Errorf("must provide a non-nil Manager")
}
if blder.forInput.err != nil {
return nil, blder.forInput.err
}
// Checking the reconcile type exist or not
if blder.forInput.object == nil {
return nil, fmt.Errorf("must provide an object for reconciliation")
}
// Set the ControllerManagedBy
if err := blder.doController(r); err != nil {
return nil, err
}
// Set the Watch
if err := blder.doWatch(); err != nil {
return nil, err
}
return blder.ctrl, nil
}在构建Controller的方法中最重要的两个步骤时doController和DoWatch。在doController的过程中，实际的核心步骤是完成Controller对象的构建，从而实现基于Scheme和Controller对象的CRD的监听流程。而在构建Controller的过程中，它的Do字段实际对应的是Reconciler接口类型定义的方法，也就是在Controller对象生成之后，必须实现这个定义的方法。它是如何使Reconciler对象同Controller产生联系的？ 实际上，在Controller初始化的过程中，借助了Options参数对象中设计的Reconciler对象，并将其传递给了Controller对象的do字段。所以当我们调用SetupWithManager方法的时候，不仅完成了Controller的初始化，还完成了Controller监听资源的注册于发现过程，同时将CRD的必要实现方法（Reconcile方法）进行了再现。
// Controller implements controller.Controller.
type Controller struct {
// Name is used to uniquely identify a Controller in tracing, logging and monitoring. Name is required.
Name string
// MaxConcurrentReconciles is the maximum number of concurrent Reconciles which can be run. Defaults to 1.
MaxConcurrentReconciles int
// Reconciler is a function that can be called at any time with the Name / Namespace of an object and
// ensures that the state of the system matches the state specified in the object.
// Defaults to the DefaultReconcileFunc.
Do reconcile.Reconciler
}
Reconciliation is level-based, meaning action isn't driven off changes in individual Events, but instead is
driven by actual cluster state read from the apiserver or a local cache.
For example if responding to a Pod Delete Event, the Request won't contain that a Pod was deleted,
instead the reconcile function observes this when reading the cluster state and seeing the Pod as missing.
*/
type Reconciler interface {
// Reconcile performs a full reconciliation for the object referred to by the Request.
// The Controller will requeue the Request to be processed again if an error is non-nil or
// Result.Requeue is true, otherwise upon completion it will remove the work from the queue.
Reconcile(context.Context, Request) (Result, error)
}
Client初始化
实现Controller时，不可避免地需要对某些资源类型进行创建、删除、更新和查询，这些操作就是通过Client实现的，查询功能实际查询的是本地的Cache，写操作是直接访问APIServer。Client是进行初始化的过程见如下代码
manager, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{})
if err != nil {
log.Error(err, "could not create manager")
os.Exit(1)
}
Finalizers
Finalizers是每种资源在声明周期结束时都会用到的字段。该字段属于Kubernetes GC来及收集器，它是一种删除拦截机制，可以让控制器在删除资源前（Pre-delete）进行回调。
Finalizers是在对象删除之前需要执行的逻辑，比如你给资源类型中的每个对象都创建了对应的外部资源，并且希望在Kubernetes删除对应资源的同时删除关联的外部资源，那么可以通过Finalizers来实现。当Finalizers字段存在时，相关资源不允许被强制删除。所有的对象在被彻底删除之前，它的Finalizers字段必须为空，即必须保证在所有对象被彻底删除之前，与它关联的所有相关资源已被删除。
Finzlizers存在于任何一个资源对象的Meta中，在Kubenertes资源中声明为Finalizers []string类型
type ObjectMeta struct {
// Must be empty before the object is deleted from the registry. Each entry
// is an identifier for the responsible component that will remove the entry
// from the list. If the deletionTimestamp of the object is non-nil, entries
// in this list can only be removed.
// Finalizers may be processed and removed in any order. Order is NOT enforced
// because it introduces significant risk of stuck finalizers.
// finalizers is a shared field, any actor with permission can reorder it.
// If the finalizer list is processed in order, then this can lead to a situation
// in which the component responsible for the first finalizer in the list is
// waiting for a signal (field value, external system, or other) produced by a
// component responsible for a finalizer later in the list, resulting in a deadlock.
// Without enforced ordering finalizers are free to order amongst themselves and
// are not vulnerable to ordering changes in the list.
// +optional
// +patchStrategy=merge
Finalizers []string `json:"finalizers,omitempty" patchStrategy:"merge" protobuf:"bytes,14,rep,name=finalizers"`
// DeletionTimestamp is RFC 3339 date and time at which this resource will be deleted. This
// field is set by the server when a graceful deletion is requested by the user, and is not
// directly settable by a client. The resource is expected to be deleted (no longer visible
// from resource lists, and not reachable by name) after the time in this field, once the
// finalizers list is empty. As long as the finalizers list contains items, deletion is blocked.
// Once the deletionTimestamp is set, this value may not be unset or be set further into the
// future, although it may be shortened or the resource may be deleted prior to this time.
// For example, a user may request that a pod is deleted in 30 seconds. The Kubelet will react
// by sending a graceful termination signal to the containers in the pod. After that 30 seconds,
// the Kubelet will send a hard termination signal (SIGKILL) to the container and after cleanup,
// remove the pod from the API. In the presence of network partitions, this object may still
// exist after this timestamp, until an administrator or automated process can determine the
// resource is fully terminated.
// If not set, graceful deletion of the object has not been requested.
//
// Populated by the system when a graceful deletion is requested.
// Read-only.
// More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata
// +optional
DeletionTimestamp *Time `json:"deletionTimestamp,omitempty" protobuf:"bytes,9,opt,name=deletionTimestamp"`
}存在Finalizers字段的资源对象接收的第一个删除请求，设置metadata.Deletion-Timestamp字段的值，但不删除具体资源，在设置该字段后，Finalizers列表的对象只能被删除，不能进行其它操作。
当metadata.DeletionTimestamp字段为非空时，Controller监听对象并执行对应Finalizers对象，在所有动作执行完成后，将该Finalizer从列表中移除。一旦Finalizers列表为空，就意味着所有Finzliser都被执行过，最终Kubernetes会删除该资源。
在Operator Controller中，最重要的逻辑就是Reconcile方法，Finalizers也是在Reconcile中实现的，代码如下：
//+kubebuilder:rbac:groups=webapp.my.domain,resources=guestbooks,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=webapp.my.domain,resources=guestbooks/status,verbs=get;update;patch
//+kubebuilder:rbac:groups=webapp.my.domain,resources=guestbooks/finalizers,verbs=update
// Reconcile is part of the main kubernetes reconciliation loop which aims to
// move the current state of the cluster closer to the desired state.
// TODO(user): Modify the Reconcile function to compare the state specified by
// the Guestbook object against the actual cluster state, and then
// perform operations to make the cluster state reflect the state specified by
// the user.
//
// For more details, check Reconcile and its Result here:
// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.11.0/pkg/reconcile
func (r *GuestbookReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
_ = log.FromContext(ctx)
// TODO(user): your logic here
var cronJob v1beta1.CronJob
if err := r.Get(ctx, req.NamespacedName, &cronJob); err != nil {
log.Log.Error(err, "unable to fetch CronJob")
return ctrl.Result{}, client.IgnoreNotFound(err)
}
// 声明Finalizer字段，类型为字符串
// 自定义Finalizer的标识符包含一个域名、一个正向斜线和Finalizer名称
myFinalizerName := "storage.finalizers.tutorial.kubebuilder.io"
// 通过检查DeletionTimestamp 字段是否为0， 判断资源是否被删除
if cronJob.ObjectMeta.DeletionTimestamp.IsZero() {
// 如果DeletionTimestamp字段为0，说明资源未被删除，此时需要检测是否存在Finalizer，如果不存在，则添加，并更新到资源对象中
if !containsString(cronJob.ObjectMeta.Finalizers, myFinalizerName) {
cronJob.ObjectMeta.Finalizers = append(cronJob.ObjectMeta.Finalizers, myFinalizerName)
if err := r.Update(ctx, &cronJob); err != nil {
return ctrl.Result{}, err
}
}
} else {
// 如果DeletionTimestamp字段不为0，说明对象处于删除状态中
if containsString(cronJob.ObjectMeta.Finalizers, myFinalizerName) {
// 如果存在Finalizer且与上述声明的finalizer匹配，那么执行对应的hook逻辑
if err := r.deleteExternalResources(&cronJob); err != nil {
// 如果删除失败，则直接返回对应的err， Controller会自动执行重试逻辑
return ctrl.Result{}, err
}
// 如果对应的hook执行成功，那么清空finalizers， kubernetes删除对应资源
cronJob.ObjectMeta.Finalizers = removeString(cronJob.ObjectMeta.Finalizers, myFinalizerName)
if err := r.Update(ctx, &cronJob); err != nil {
return ctrl.Result{}, err
}
}
return ctrl.Result{}, nil
}
}
func containsString(slice []string, s string) bool {
for _, item := range slice {
if item == s {
return true
}
}
return false
}
func removeString(slice []string, s string) (result []string) {
for _, item := range slice {
if item == s {
continue
}
result = append(result, item)
}
return
}
func (r *GuestbookReconciler) deleteExternalResources(cronJob *v1beta1.CronJob) error {
// 删除cronJob关联的外部资源逻辑
// 需要确保实现幂等
return nil
}在Kubernetes中，只要对象ObjectMeta中的Finalizers不为空，该对象的Delete操作就会转变为Update操作，Update DeletionTimestamp字段的意义是告诉Kubernetes的垃圾回收器，在DeletionTimestamp这个时刻之后，只要Finalizers为空，就立马删除该对象。
所以一般的使用方法就是在创建对象时把Finalizers设置好（任意String），然后处理DeletionTimestamp不为空的Update操作（实际是Delete），根据Finalizers的值执行完所有的Pre-delete Hook（此时可以在Cache中读取被删除对象的任何信息）之后将Finalizers设置为空即可。
