利用ansible自动化部署kubernetes集群
机器环境介绍
1.1. 机器信息介绍
| 主机名 | OS版本 | ip地址 | cpu | 内存 | 磁盘 |
|---|---|---|---|---|---|
| master.k8s.local | Ubuntu 22.04 | 10.22.4.11 | 2core | 6G | 80G |
| node01.k8s.local | Ubuntu 22.04 | 10.22.4.12 | 2core | 6G | 80G |
| node02.k8s.local | Ubuntu 22.04 | 10.22.4.13 | 2core | 6G | 80G |
1.2. 规划IP地址介绍
在Kubernetes中CNI网络插件采用Calico,划分三个网段
| 网络信息介绍 | 网段 | 备注 |
|---|---|---|
| pod ip网段 | 10.224.0.0/16 | |
| cluster ip网段 | 10.96.0.0/16 |
1.3. kuberntes安装信息介绍
安装的kubernetets版本为1.28.5，Calico版本为3.26.4，容器运行环境为containerd
如果需要其他版本kuberneres，需要修改下面的脚本
- 修改kubernetes源里面的版本
- 修改安装master和worker节点里面定义的版本变量值
如下需要使用其他版本的CNI插件或者不同版本的calico插件，需要对网络插件部分脚本进行修改
安装配置ansible
2.1. ansible软件部署
- 安装ansible软件
apt update && apt install ansible -y
- 配置ansible配置
mkdir /etc/ansible/ && touch /etc/ansible/hosts
- 配置/etc/ansible/hosts文件
[master]
10.22.4.11
[worker]
10.22.4.12
10.22.4.13
- 配置免密登录, 此过程中不要输入密码
ssh-keygen -t rsa
- 分发免密登录
ssh-copy-id root@10.22.4.11
ssh-copy-id root@10.22.4.12
ssh-copy-id root@10.22.4.13
- 配置hosts
cat >> /etc/hosts <<EOF
10.22.4.11 master master.k8s.local
10.22.4.12 worker01 node01.k8s.local
10.22.4.13 worker02 node02.k8s.local
EOF
2.2. 测试ansible连接性
- 编写测试脚本
cat >test_nodes.yml <<EOF
---
- name: test nodes
hosts:
master
worker
tasks:
- name: Ping nodes
ping:
EOF
- 执行ansible测试
root@master:~/ansible# ansible-playbook test_node.yml
PLAY [Manage nodes] ***********************************************************************************************************
TASK [Gathering Facts] ********************************************************************************************************
ok: [10.22.4.13]
ok: [10.22.4.12]
ok: [10.22.4.11]
TASK [Ping nodes] *************************************************************************************************************
ok: [10.22.4.13]
ok: [10.22.4.12]
ok: [10.22.4.11]
PLAY RECAP ********************************************************************************************************************
10.22.4.11 : ok=2 changed=0 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0
10.22.4.12 : ok=2 changed=0 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0
10.22.4.13 : ok=2 changed=0 unreachable=0 failed=0 skipped=0 rescued=0 ignored=0
root@master:~/ansible#
配置kubernetes脚本
3.1. 编写的kubernetes 脚本
- 编写的install-kubernetes.yml文件内容如下
---
- name: Performance Basic Config
hosts:
master
worker
become: yes
tasks:
- name: Check if fstab contains swap
shell: grep -q "swap" /etc/fstab
register: fstab_contains_swap
- name: Temp Disable swap
command: swapoff -a
when: fstab_contains_swap.rc == 0
- name: Permanent Disable swap
shell: sed -i 's/.*swap.*/#&/g' /etc/fstab
when: fstab_contains_swap.rc == 0
- name: Disable Swap unit-files
shell: |
swap_units=$(systemctl list-unit-files | grep swap | awk '{print $1}')
for unit in $swap_units; do
systemctl mask $unit
done
- name: Stop UFW service
service:
name: ufw
state: stopped
- name: Disable UFW at boot
service:
name: ufw
enabled: no
- name: Set timezone
shell: TZ='Asia/Shanghai'; export TZ
- name: Set timezone permanently
shell: |
cat >> /etc/profile << EOF
TZ='Asia/Shanghai'; export TZ
EOF
- name: Create .hushlogin file in $HOME
file:
path: "{{ ansible_env.HOME }}/.hushlogin"
state: touch
- name: Install required packages
apt:
name: "{{ packages }}"
state: present
vars:
packages:
- apt-transport-https
- ca-certificates
- curl
- gnupg
- lsb-release
- name: Add Aliyun Docker GPG key
shell: curl -fsSL https://mirrors.aliyun.com/docker-ce/linux/ubuntu/gpg | sudo apt-key add
- name: Add Aliyun Docker repository
shell: echo "deb [arch=amd64 signed-by=/etc/apt/trusted.gpg] https://mirrors.aliyun.com/docker-ce/linux/ubuntu $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker-ce.list
- name: Add Aliyun Kubernetes GPG key
shell: curl -fsSL https://mirrors.aliyun.com/kubernetes-new/core/stable/v1.28/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
- name: Add Aliyun Kubernetes repository
shell: echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://mirrors.aliyun.com/kubernetes-new/core/stable/v1.28/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.list
- name: Set apt sources to use USTC mirrors
shell: sed -i 's#cn.archive.ubuntu.com#mirrors.aliyun.com#g' /etc/apt/sources.list
- name: Update apt cache
apt:
update_cache: yes
- name: Load br_netfilter on start
shell: echo "modprobe br_netfilter" >> /etc/profile
- name: Load br_netfilter
shell: modprobe br_netfilter
- name: Update sysctl settings
sysctl:
name: "{{ item.name }}"
value: "{{ item.value }}"
state: present
reload: yes
with_items:
- { name: "net.bridge.bridge-nf-call-iptables", value: "1" }
- { name: "net.bridge.bridge-nf-call-ip6tables", value: "1" }
- { name: "net.ipv4.ip_forward", value: "1" }
- name: Install IPVS
apt:
name: "{{ packages }}"
state: present
vars:
packages:
- ipset
- ipvsadm
- name: Create ipvs modules
file:
name: /etc/modules-load.d/ipvs.modules
mode: 0755
state: touch
- name: Write ipvs.modules file
lineinfile:
dest: /etc/modules-load.d/ipvs.modules
line: "#!/bin/bash\nmodprobe -- ip_vs\nmodprobe -- ip_vs_rr\nmodprobe -- ip_vs_wrr\nmodprobe -- ip_vs_sh\nmodprobe -- nf_conntrack\nmodprobe -- overlay\nmodprobe -- br_netfilter"
- name: Execute ipvs.modules script
shell: sh /etc/modules-load.d/ipvs.modules
- name: Install Containerd
apt:
name: "{{ packages }}"
state: present
vars:
packages:
- containerd.io
- name: Generate default containerd file
shell: containerd config default > /etc/containerd/config.toml
- name: Config sandbox image
shell: sed -i 's#sandbox_image = "registry.k8s.io/pause:3.6"#sandbox_image = "registry.aliyuncs.com/google_containers/pause:3.9"#g' /etc/containerd/config.toml
- name: Modify Systemd Cgroup
shell: sed -i 's#SystemdCgroup = false#SystemdCgroup = true#g' /etc/containerd/config.toml
- name: Restart Containerd
shell: systemctl restart containerd
- name: Systemctl enable containerd
shell: systemctl enable containerd
- name: Install Kubernetes Master
hosts: master
become: yes
vars:
kubernetes_version: "1.28.5"
pod_network_cidr: "10.244.0.0/16"
service_cidr: "10.96.0.0/16"
image_repository: "registry.aliyuncs.com/google_containers"
calico_version: "v3.26.4"
tasks:
- name: Install Master kubernetes packages
apt:
name: "{{ packages }}"
state: present
vars:
packages:
- kubelet={{ kubernetes_version }}-1.1
- kubeadm={{ kubernetes_version }}-1.1
- kubectl={{ kubernetes_version }}-1.1
- name: Initialize Kubernetes Master
command: kubeadm init --kubernetes-version={{ kubernetes_version }} --pod-network-cidr={{ pod_network_cidr }} --service-cidr={{ service_cidr }} --image-repository={{ image_repository }}
register: kubeadm_output
changed_when: "'kubeadm join' in kubeadm_output.stdout"
- name: Save join command
copy:
content: |
{{ kubeadm_output.stdout_lines [-2] }}
{{ kubeadm_output.stdout_lines [-1] }}
dest: /root/kubeadm_join_master.sh
when: kubeadm_output.changed
- name: cope join master script
shell: sed -i 's/"//g' /root/kubeadm_join_master.sh
- name: copy kubernetes config
shell: mkdir -p {{ ansible_env.HOME }}/.kube && cp -i /etc/kubernetes/admin.conf {{ ansible_env.HOME }}/.kube/config
- name: enable kubectl
command: systemctl enable kubelet
- name: Create calico directory
file:
path: "{{ ansible_env.HOME }}/calico/{{ calico_version }}"
state: directory
- name: download calico tigera-operator.yaml
command: wget https://ghproxy.net/https://raw.githubusercontent.com/projectcalico/calico/{{ calico_version }}/manifests/tigera-operator.yaml -O {{ ansible_env.HOME }}/calico/{{ calico_version }}/tigera-operator.yaml
- name: download calico custom-resources.yaml
command: wget https://ghproxy.net/https://raw.githubusercontent.com/projectcalico/calico/{{ calico_version }}/manifests/custom-resources.yaml -O {{ ansible_env.HOME }}/calico/{{ calico_version }}/custom-resources.yaml
- name: set calico netwok range
replace:
path: "{{ ansible_env.HOME }}/calico/{{ calico_version }}/custom-resources.yaml"
regexp: "blockSize: 26"
replace: "blockSize: 24"
- name: set calico ip pools
replace:
path: "{{ ansible_env.HOME }}/calico/{{ calico_version }}/custom-resources.yaml"
regexp: "cidr: 192.168.0.0/16"
replace: "cidr: {{ pod_network_cidr }}"
- name: apply calico tigera-operator.yaml
command: kubectl create -f {{ ansible_env.HOME }}/calico/{{ calico_version }}/tigera-operator.yaml
- name: apply calico custom-resources.yaml
command: kubectl create -f {{ ansible_env.HOME }}/calico/{{ calico_version }}/custom-resources.yaml
- name: set crictl config
command: crictl config runtime-endpoint unix:///var/run/containerd/containerd.sock
- name: Install Kubernetes worker
hosts: worker
become: yes
vars:
kubernetes_version: "1.28.5"
tasks:
- name: Install worker kubernetes packages
apt:
name: "{{ packages }}"
state: present
vars:
packages:
- kubelet={{ kubernetes_version }}-1.1
- kubeadm={{ kubernetes_version }}-1.1
- name: copy kubeadm join script to workers
copy:
src: /root/kubeadm_join_master.sh
dest: /root/kubeadm_join_master.sh
mode: 0755
- name: worker join to cluster
command: sh /root/kubeadm_join_master.sh
- name: set crictl config
command: crictl config runtime-endpoint unix:///var/run/containerd/containerd.sock
- name: enable kubectl
command: systemctl enable kubelet
执行kubernetes脚本
- 执行脚本
root@master:~/kubernetes# ansible-playbook install-kubernetes.yml
- 集群状态
root@master:~# kubectl get node -o wide
NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME
master.k8s.local Ready control-plane 19m v1.28.5 10.22.4.11 <none> Ubuntu 22.04.3 LTS 5.15.0-91-generic containerd://1.6.27
node01.k8s.local Ready <none> 18m v1.28.5 10.22.4.12 <none> Ubuntu 22.04.3 LTS 5.15.0-91-generic containerd://1.6.27
node02.k8s.local Ready <none> 18m v1.28.5 10.22.4.13 <none> Ubuntu 22.04.3 LTS 5.15.0-91-generic containerd://1.6.27
root@master:~#
- 集群pod状态
root@master:~# kubectl get pod -A
NAMESPACE NAME READY STATUS RESTARTS AGE
calico-apiserver calico-apiserver-b897f94cd-4xz87 1/1 Running 0 16m
calico-apiserver calico-apiserver-b897f94cd-7zt28 1/1 Running 0 16m
calico-system calico-kube-controllers-57474df497-jgkmt 1/1 Running 0 19m
calico-system calico-node-mxmq6 1/1 Running 0 19m
calico-system calico-node-nqdkn 1/1 Running 0 19m
calico-system calico-node-wd5fm 1/1 Running 0 19m
calico-system calico-typha-79b8c6d4fd-tjdvm 1/1 Running 0 19m
calico-system calico-typha-79b8c6d4fd-xddmp 1/1 Running 0 19m
calico-system csi-node-driver-gxg2g 2/2 Running 0 19m
calico-system csi-node-driver-kpdxn 2/2 Running 0 19m
calico-system csi-node-driver-ttng2 2/2 Running 0 19m
kube-system coredns-66f779496c-lp8hd 1/1 Running 0 19m
kube-system coredns-66f779496c-qxcz5 1/1 Running 0 19m
kube-system etcd-master.k8s.local 1/1 Running 3 19m
kube-system kube-apiserver-master.k8s.local 1/1 Running 3 19m
kube-system kube-controller-manager-master.k8s.local 1/1 Running 3 19m
kube-system kube-proxy-7d9z4 1/1 Running 0 19m
kube-system kube-proxy-8gqbc 1/1 Running 0 19m
kube-system kube-proxy-grkdb 1/1 Running 0 19m
kube-system kube-scheduler-master.k8s.local 1/1 Running 3 19m
tigera-operator tigera-operator-7f8cd97876-dg55s 1/1 Running 0 19m
root@master:~#
