kubernetes集群之高可用实现

部署coredns

复制代码
[root@master-1 ~]# cat  coredns.yaml
apiVersion: v1
kind: ServiceAccount
metadata:
  name: coredns
  namespace: kube-system
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  labels:
    kubernetes.io/bootstrapping: rbac-defaults
  name: system:coredns
rules:
  - apiGroups:
    - ""
    resources:
    - endpoints
    - services
    - pods
    - namespaces
    verbs:
    - list
    - watch
  - apiGroups:
    - discovery.k8s.io
    resources:
    - endpointslices
    verbs:
    - list
    - watch
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  annotations:
    rbac.authorization.kubernetes.io/autoupdate: "true"
  labels:
    kubernetes.io/bootstrapping: rbac-defaults
  name: system:coredns
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:coredns
subjects:
- kind: ServiceAccount
  name: coredns
  namespace: kube-system
---
apiVersion: v1
kind: ConfigMap
metadata:
  name: coredns
  namespace: kube-system
data:
  Corefile: |
    .:53 {
        errors
        health {
          lameduck 5s
        }
        ready
        kubernetes cluster.local in-addr.arpa ip6.arpa {
          fallthrough in-addr.arpa ip6.arpa
        }
        prometheus :9153
        forward . /etc/resolv.conf {
          max_concurrent 1000
        }
        cache 30
        loop
        reload
        loadbalance
    }
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: coredns
  namespace: kube-system
  labels:
    k8s-app: kube-dns
    kubernetes.io/name: "CoreDNS"
spec:
  # replicas: not specified here:
  # 1. Default is 1.
  # 2. Will be tuned in real time if DNS horizontal auto-scaling is turned on.
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxUnavailable: 1
  selector:
    matchLabels:
      k8s-app: kube-dns
  template:
    metadata:
      labels:
        k8s-app: kube-dns
    spec:
      priorityClassName: system-cluster-critical
      serviceAccountName: coredns
      tolerations:
        - key: "CriticalAddonsOnly"
          operator: "Exists"
      nodeSelector:
        kubernetes.io/os: linux
      affinity:
         podAntiAffinity:
           preferredDuringSchedulingIgnoredDuringExecution:
           - weight: 100
             podAffinityTerm:
               labelSelector:
                 matchExpressions:
                   - key: k8s-app
                     operator: In
                     values: ["kube-dns"]
               topologyKey: kubernetes.io/hostname
      containers:
      - name: coredns
        image: coredns/coredns:1.7.0
        imagePullPolicy: IfNotPresent
        resources:
          limits:
            memory: 170Mi
          requests:
            cpu: 100m
            memory: 70Mi
        args: [ "-conf", "/etc/coredns/Corefile" ]
        volumeMounts:
        - name: config-volume
          mountPath: /etc/coredns
          readOnly: true
        ports:
        - containerPort: 53
          name: dns
          protocol: UDP
        - containerPort: 53
          name: dns-tcp
          protocol: TCP
        - containerPort: 9153
          name: metrics
          protocol: TCP
        securityContext:
          allowPrivilegeEscalation: false
          capabilities:
            add:
            - NET_BIND_SERVICE
            drop:
            - all
          readOnlyRootFilesystem: true
        livenessProbe:
          httpGet:
            path: /health
            port: 8080
            scheme: HTTP
          initialDelaySeconds: 60
          timeoutSeconds: 5
          successThreshold: 1
          failureThreshold: 5
        readinessProbe:
          httpGet:
            path: /ready
            port: 8181
            scheme: HTTP
      dnsPolicy: Default
      volumes:
        - name: config-volume
          configMap:
            name: coredns
            items:
            - key: Corefile
              path: Corefile
---
apiVersion: v1
kind: Service
metadata:
  name: kube-dns
  namespace: kube-system
  annotations:
    prometheus.io/port: "9153"
    prometheus.io/scrape: "true"
  labels:
    k8s-app: kube-dns
    kubernetes.io/cluster-service: "true"
    kubernetes.io/name: "CoreDNS"
spec:
  selector:
    k8s-app: kube-dns
  clusterIP: 10.255.0.2
  ports:
  - name: dns
    port: 53
    protocol: UDP
  - name: dns-tcp
    port: 53
    protocol: TCP
  - name: metrics
    port: 9153
    protocol: TCP
复制代码

 

    执行

?
1
[root@master-1 ~]# kubectl apply -f coredns.yaml

　　查看

?
1
2
3
4
5
[root@master-1 ~]# kubectl get pods -n kube-system
NAME                                       READY   STATUS    RESTARTS   AGE
calico-kube-controllers-6949477b58-2mfdh   1/1     Running   1          11h
calico-node-tlb6d                          1/1     Running   1          11h
coredns-7bf4bd64bd-9fqkz                   1/1     Running   0          11m

　　查看集群状态

?
1
2
3
[root@master-1 ~]# kubectl get nodes
NAME     STATUS   ROLES    AGE   VERSION
node-1   Ready    <none>   12h   v1.20.7

　　运行tomcat 测试

?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
[root@master-1 ~]# cat tomcat.yaml
apiVersion: v1  #pod属于k8s核心组v1
kind: Pod  #创建的是一个Pod资源
metadata:  #元数据
  name: demo-pod  #pod名字
  namespace: default  #pod所属的名称空间
  labels:
    app: myapp  #pod具有的标签
    env: dev      #pod具有的标签
spec:
  containers:      #定义一个容器，容器是对象列表，下面可以有多个name
  - name:  tomcat-pod-java  #容器的名字
    ports:
    - containerPort: 8080
    image: tomcat:8.5-jre8-alpine   #容器使用的镜像
    imagePullPolicy: IfNotPresent
  - name: busybox
    image: busybox:latest
    command:  #command是一个列表，定义的时候下面的参数加横线
    - "/bin/sh"
    - "-c"
    - "sleep 3600"
 
您在 /var/spool/mail/root 中有新邮件
[root@master-1 ~]# kubectl apply -f tomcat.yaml
pod/demo-pod created
您在 /var/spool/mail/root 中有新邮件
[root@master-1 ~]# cat tomcat-service.yaml
apiVersion: v1
kind: Service
metadata:
  name: tomcat
spec:
  type: NodePort
  ports:
    - port: 8080
      nodePort: 30080
  selector:
    app: myapp
    env: dev
[root@master-1 ~]#  kubectl apply -f tomcat-service.yaml
service/tomcat unchanged
您在 /var/spool/mail/root 中有新邮件
[root@master-1 ~]# kubectl get svc
NAME         TYPE        CLUSTER-IP    EXTERNAL-IP   PORT(S)          AGE
kubernetes   ClusterIP   10.255.0.1    <none>        443/TCP          13h
tomcat       NodePort    10.255.92.3   <none>        8080:30080/TCP   80s

　　

在浏览器访问node-1节点的ip:30080即可请求到浏览器http://192.168.10.32:30080/

 

 测试DNS

?
1
2
3
4
5
6
7
8
9
10
11
12
[root@master-1 ~]# kubectl run busybox --image busybox:1.28 --restart=Never --rm -it busybox -- sh
If you don't see a command prompt, try pressing enter.
/ # ping www.baidu.co
ping: bad address 'www.baidu.co'
/ # ping www.baidu.com
PING www.baidu.com (103.235.46.40): 56 data bytes
64 bytes from 103.235.46.40: seq=0 ttl=127 time=27.786 ms
64 bytes from 103.235.46.40: seq=1 ttl=127 time=26.360 ms
^C
--- www.baidu.com ping statistics ---
2 packets transmitted, 2 packets received, 0% packet loss
round-trip min/avg/max = 26.360/27.073/27.786 ms

　　安装keepalived+nginx实现k8s apiserver高可用 设置两个master节点允许监听非本机端口

?
1
2
echo "net.ipv4.ip_nonlocal_bind=1" >> /etc/sysctl.conf
sysctl -p

　　安装nginx   nginx-mod-stream  (四层模块)

?
1
yum -y install nginx keepalived nginx-mod-stream

　　ng主从配置一样

?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
[root@master-1 ~]# cat /etc/nginx/nginx.conf
user nginx;
worker_processes auto;
error_log /var/log/nginx/error.log;
pid /run/nginx.pid;
  
include /usr/share/nginx/modules/*.conf;
  
events {
    worker_connections 1024;
}
  
# 四层负载均衡，为两台Master apiserver组件提供负载均衡
stream {
  
    log_format  main  '$remote_addr $upstream_addr - [$time_local] $status $upstream_bytes_sent';
  
    access_log  /var/log/nginx/k8s-access.log  main;
  
    upstream k8s-apiserver {
       server 192.168.10.29:6443;   # Master1 APISERVER IP:PORT
       server 192.168.10.30:6443;   # Master2 APISERVER IP:PORT
       server 192.168.10.31:6443;   # Master3 APISERVER IP:PORT
    }
      
    server {
       listen 192.168.10.28:16443; # 由于nginx与master节点复用，这个IP是vip监听
       proxy_pass k8s-apiserver;
    }
}
  
http {
    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';
  
    access_log  /var/log/nginx/access.log  main;
  
    sendfile            on;
    tcp_nopush          on;
    tcp_nodelay         on;
    keepalive_timeout   65;
    types_hash_max_size 2048;
  
    include             /etc/nginx/mime.types;
    default_type        application/octet-stream;
  
    server {
        listen       80 default_server;
        server_name  _;
  
        location / {
        }
    }
}

　　启动ng

?
1
2
3
4
5
6
7
[root@master-1 ~]# systemctl start nginx.service
[root@master-1 ~]# ss -lntp |grep 192.168.10.28
LISTEN     0      128    192.168.10.28:16443                    *:*                   users:(("nginx",pid=21422,fd=7),("nginx",pid=21421,fd=7),("nginx",pid=21420,fd=7))
[root@master-2 ~]# systemctl start nginx.service
您在 /var/spool/mail/root 中有新邮件
[root@master-2 ~]# ss -lntp | grep 192.168.10.28
LISTEN     0      128    192.168.10.28:16443                    *:*                   users:(("nginx",pid=21263,fd=7),("nginx",pid=21262,fd=7),("nginx",pid=21261,fd=7))

　　配置高可用主节点

?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
vim /etc/keepalived/keepalived.conf
global_defs {
   notification_email {
     root@localhost
   }
   notification_email_from ka1@localhost
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   vrrp_mcast_group4 224.111.111.111
}
   
vrrp_script chk_ng {
    script "ss -lntp | grep  192.168.10.28"
    interval 2
    weight -10
    fall 2
    rise 2
}
   
vrrp_instance External_1 {
    state MASTER
    interface ens33
    virtual_router_id 171
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1402b1b5
    }
    virtual_ipaddress {
        192.168.10.28/24
    }
    track_script {
        chk_ng
    }
}

　　从节点

?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
cat   /etc/keepalived/keepalived.conf
global_defs {
   notification_email {
     root@localhost
   }
   notification_email_from ka2@localhost
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   vrrp_mcast_group4 224.111.111.111
}
   
vrrp_script chk_ng {
    script "ss -lntp | grep 192.168.10.28"
    interval 2
    weight -10
    fall 2
    rise 2
}
   
vrrp_instance External_1 {
    state BACKUP
    interface ens33
    virtual_router_id 171
    priority 95
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 1402b1b5
    }
    virtual_ipaddress {
        192.168.10.28/24
    }
    track_script {
        chk_ng
    }
}

　　启动高可用服务并查看VIP是否正常

?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
[root@master-1 ~]# systemctl start keepalived.service
[root@master-2 ~]# systemctl start keepalived.service
[root@master-2 ~]# systemctl enable keepalived.service
Created symlink from /etc/systemd/system/multi-user.target.wants/keepalived.service to /usr/lib/systemd/system/keepalived.service.
[root@master-2 ~]# systemctl enable nginx.service
Created symlink from /etc/systemd/system/multi-user.target.wants/nginx.service to /usr/lib/systemd/system/nginx.service.
[root@master-1 ~]# systemctl enable keepalived.service
Created symlink from /etc/systemd/system/multi-user.target.wants/keepalived.service to /usr/lib/systemd/system/keepalived.service.
[root@master-1 ~]# systemctl enable nginx.service
Created symlink from /etc/systemd/system/multi-user.target.wants/nginx.service to /usr/lib/systemd/system/nginx.service.
[root@master-1 ~]# hostname -I
192.168.10.29 192.168.10.28
[root@master-2 ~]# hostname -I
192.168.10.30

　　停止主节点ng 查看vip是否飘走

?
1
2
3
4
5
[root@master-1 ~]# systemctl stop nginx.service
[root@master-1 ~]# hostname -I
192.168.10.29
[root@master-2 ~]# hostname -I
192.168.10.30 192.168.10.28

　　启动主节点ng 查看vip

?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
[root@master-1 ~]# systemctl start nginx.service
您在 /var/spool/mail/root 中有新邮件
[root@master-1 ~]# hostname -I   协商过程
192.168.10.29
[root@master-1 ~]# hostname -I
192.168.10.29
[root@master-1 ~]# hostname -I
192.168.10.29
[root@master-1 ~]# hostname -I
192.168.10.29
[root@master-1 ~]# hostname -I
192.168.10.29
[root@master-1 ~]# hostname -I
192.168.10.29
[root@master-1 ~]# hostname -I
192.168.10.29
[root@master-1 ~]# hostname -I   成功飘回来
192.168.10.29 192.168.10.28
[root@master-2 ~]# hostname -I   从节点vip 移动走
192.168.10.30
您在 /var/spool/mail/root 中有新邮件

　　

 目前所有的Worker Node组件连接都还是master1 Node，如果不改为连接VIP走负载均衡器，那么Master还是单点故障。

因此接下来就是要改所有Worker Node（kubectl get node命令查看到的节点）组件配置文件，由原来192.168.10.29修改为192.168.10.28（VIP）。

?
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
[root@node-1 ~]#  sed -i 's#192.168.10.29:6443#192.168.10.28:16443#' /etc/kubernetes/kubelet-bootstrap.kubeconfig
[root@node-1 ~]#  sed -i 's#192.168.10.29:6443#192.168.10.28:16443#' /etc/kubernetes/kubelet.json
[root@node-1 ~]#  sed -i 's#192.168.10.29:6443#192.168.10.28:16443#' /etc/kubernetes/kubelet.kubeconfig
您在 /var/spool/mail/root 中有新邮件
[root@node-1 ~]#  sed -i 's#192.168.10.29:6443#192.168.10.28:16443#' /etc/kubernetes/kube-proxy.yaml
[root@node-1 ~]#  sed -i 's#192.168.10.29:6443#192.168.10.28:16443#' /etc/kubernetes/kube-proxy.kubeconfig
[root@node-1 ~]# systemctl restart kubelet kube-proxy
您在 /var/spool/mail/root 中有新邮件
[root@node-1 ~]# systemctl status kubelet kube-proxy
● kubelet.service - Kubernetes Kubelet
   Loaded: loaded (/usr/lib/systemd/system/kubelet.service; enabled; vendor preset: disabled)
   Active: active (running) since 五 2022-08-12 09:42:56 CST; 10s ago
     Docs: https://github.com/kubernetes/kubernetes
 Main PID: 67657 (kubelet)
    Tasks: 12
   Memory: 57.4M
   CGroup: /system.slice/kubelet.service
           └─67657 /usr/local/bin/kubelet --bootstrap-kubeconfig=/etc/kubernetes/kubelet-bootstrap.kubeconfig --cert-dir=/etc/kubernetes/ssl --kubeconfig=/etc/kubernetes/kubelet.kubeconfig --config=/etc/kub...
 
8月 12 09:42:58 node-1 kubelet[67657]: I0812 09:42:58.144370   67657 operation_generator.go:672] MountVolume.SetUp succeeded for volume "calico-node-token-k7k4g" (UniqueName: "kubernetes.io/s...92371e6afa06")
8月 12 09:42:58 node-1 kubelet[67657]: I0812 09:42:58.144507   67657 operation_generator.go:672] MountVolume.SetUp succeeded for volume "config-volume" (UniqueName: "kubernetes.io/configmap/c...1958d0692677")
8月 12 09:42:58 node-1 kubelet[67657]: I0812 09:42:58.144540   67657 operation_generator.go:672] MountVolume.SetUp succeeded for volume "lib-modules" (UniqueName: "kubernetes.io/host-path/762...92371e6afa06")
8月 12 09:42:58 node-1 kubelet[67657]: I0812 09:42:58.144627   67657 operation_generator.go:672] MountVolume.SetUp succeeded for volume "coredns-token-fn5th" (UniqueName: "kubernetes.io/secre...1958d0692677")
8月 12 09:42:58 node-1 kubelet[67657]: I0812 09:42:58.144651   67657 operation_generator.go:672] MountVolume.SetUp succeeded for volume "flexvol-driver-host" (UniqueName: "kubernetes.io/host-...92371e6afa06")
8月 12 09:42:58 node-1 kubelet[67657]: I0812 09:42:58.144732   67657 operation_generator.go:672] MountVolume.SetUp succeeded for volume "default-token-xmj6q" (UniqueName: "kubernetes.io/secre...df99ab8e6be9")
8月 12 09:42:58 node-1 kubelet[67657]: I0812 09:42:58.144754   67657 operation_generator.go:672] MountVolume.SetUp succeeded for volume "sysfs" (UniqueName: "kubernetes.io/host-path/762c1818-...92371e6afa06")
8月 12 09:42:58 node-1 kubelet[67657]: I0812 09:42:58.144766   67657 operation_generator.go:672] MountVolume.SetUp succeeded for volume "cni-bin-dir" (UniqueName: "kubernetes.io/host-path/762...92371e6afa06")
8月 12 09:42:58 node-1 kubelet[67657]: I0812 09:42:58.144780   67657 operation_generator.go:672] MountVolume.SetUp succeeded for volume "cni-net-dir" (UniqueName: "kubernetes.io/host-path/762...92371e6afa06")
8月 12 09:42:58 node-1 kubelet[67657]: I0812 09:42:58.144792   67657 operation_generator.go:672] MountVolume.SetUp succeeded for volume "var-run-calico" (UniqueName: "kubernetes.io/host-path/...92371e6afa06")
 
● kube-proxy.service - Kubernetes Kube-Proxy Server
   Loaded: loaded (/usr/lib/systemd/system/kube-proxy.service; enabled; vendor preset: disabled)
   Active: active (running) since 五 2022-08-12 09:42:56 CST; 10s ago
     Docs: https://github.com/kubernetes/kubernetes
 Main PID: 67658 (kube-proxy)
    Tasks: 6
   Memory: 18.9M
   CGroup: /system.slice/kube-proxy.service
           └─67658 /usr/local/bin/kube-proxy --config=/etc/kubernetes/kube-proxy.yaml --alsologtostderr=true --logtostderr=false --log-dir=/var/log/kubernetes --v=2
 
8月 12 09:42:56 node-1 kube-proxy[67658]: I0812 09:42:56.692151   67658 proxier.go:1036] Not syncing ipvs rules until Services and Endpoints have been received from master
8月 12 09:42:56 node-1 kube-proxy[67658]: I0812 09:42:56.692160   67658 shared_informer.go:247] Caches are synced for service config
8月 12 09:42:56 node-1 kube-proxy[67658]: I0812 09:42:56.692669   67658 service.go:390] Adding new service port "default/kubernetes:https" at 10.255.0.1:443/TCP
8月 12 09:42:56 node-1 kube-proxy[67658]: I0812 09:42:56.692685   67658 service.go:390] Adding new service port "kube-system/kube-dns:dns" at 10.255.0.2:53/UDP
8月 12 09:42:56 node-1 kube-proxy[67658]: I0812 09:42:56.692690   67658 service.go:390] Adding new service port "kube-system/kube-dns:dns-tcp" at 10.255.0.2:53/TCP
8月 12 09:42:56 node-1 kube-proxy[67658]: I0812 09:42:56.692694   67658 service.go:390] Adding new service port "kube-system/kube-dns:metrics" at 10.255.0.2:9153/TCP
8月 12 09:42:56 node-1 kube-proxy[67658]: I0812 09:42:56.692699   67658 service.go:390] Adding new service port "default/tomcat" at 10.255.92.3:8080/TCP
8月 12 09:42:56 node-1 kube-proxy[67658]: I0812 09:42:56.692767   67658 proxier.go:1067] Stale udp service kube-system/kube-dns:dns -> 10.255.0.2
8月 12 09:42:56 node-1 kube-proxy[67658]: I0812 09:42:56.738753   67658 proxier.go:2243] Opened local port "nodePort for default/tomcat" (:30080/tcp)
8月 12 09:43:04 node-1 kube-proxy[67658]: I0812 09:43:04.257220   67658 proxier.go:1067] Stale udp service kube-system/kube-dns:dns -> 10.255.0.2
Hint: Some lines were ellipsized, use -l to show in full.

　　查看集群状态

?
1
2
3
4
[root@master-1 ~]# kubectl get nodes
NAME     STATUS   ROLES    AGE   VERSION
node-1   Ready    <none>   13h   v1.20.7
您在 /var/spool/mail/root 中有新邮件

　　

 

posted @   烟雨楼台，行云流水  阅读(175)  评论(0)  编辑  收藏  举报
相关博文：
· cka 考题
· kubernetes集群之部署kubelet
· K8s高可用集群二进制部署-V1.20
· Kubernetes04-安装Kubernetes，多master（使用二进制）
· ubuntu 二进制部署k8s 1.23版本
阅读排行：
· 无需6万激活码！GitHub神秘组织3小时极速复刻Manus，手把手教你使用OpenManus搭建本
· C#/.NET/.NET Core优秀项目和框架2025年2月简报
· 葡萄城 AI 搜索升级：DeepSeek 加持，客户体验更智能
· 什么是nginx的强缓存和协商缓存
· 一文读懂知识蒸馏
历史上的今天：
2019-08-12 Kubernetes的service资源介绍
2019-08-12 Kubernetes的pod控制器之DaemonSet
点击右上角即可分享
微信分享提示