架构第四次作业-20230730
一、总结kube-apiserver、kube-controler-manager、kube-scheduler、kube-proxy、kubelet等各组件的功能
kube-apiserver:为k8s各类资源对象的增删改查及watch等提供 HTTP Rest 接口,这些对象包括 pods、services、replicationcontrollers等,API Server 为 REST 操作提供服务,并为集群的共享状态提供前端,所有其他组件都通过该前端进行交互
kube-controler-manager:是集群内部的管理控制中心,负责集群内的 Node、Pod 副本、服务端点(Endpoint)、命名空间(Namespace)、服务账号(ServiceAccount)、资源定额(ResourceQuota)的管理,当某个 Node 以外宕机时,Controller Manager 会及时发现并执行自动化修复流程,确保集群中的 pod 副本始终处于预期的工作状态
kube-scheduler:是一个控制面(管理)进程,负责 Pods 的调度
kube-proxy:运行在每个节点上,监听 API Server 中服务对象的变化,再通过管理 IPtables 或者 IPVS 规则来实现网络的转发实现 Kubernetes 服务访问
kubelet:运行在每一个 worker 节点的代理组件,它会监视已分配给节点的pod,完整某些功能:1、向 master 汇报 node 节点的状态信息 2、接受指令并在 pod 中创建 docker 容器 3、准备 pod 所需要的数据卷 4、返回 pod 的运行状态 5、在 node 节点执行容器健康性检查
二、基于Kubeadm在私有云部署高可用kubernetes环境
2.1 实验环境
# 操作系统版本
root@master1-120:~# lsb_release -a
No LSB modules are available.
Distributor ID: Ubuntu
Description: Ubuntu 22.04.2 LTS
Release: 22.04
Codename: jammy
# 各节点IP分配
master1-120: 192.168.119.120
master2-121: 192.168.119.121
VIP: 192.168.119.200
HA1-keepalived-122: 192.168.119.122
HA2-keepalived-123: 192.168.119.123
node1-124: 192.168.119.124
node2-125: 192.168.119.125
# master 节点和 node 节点上已经预先安装 containerd 环境
root@master1-120:~# containerd -v
containerd github.com/containerd/containerd v1.7.2 0cae528dd6cb557f7201036e9f43420650207b58
root@master1-120:~# runc -v
runc version 1.1.8
commit: v1.1.8-0-g82f18fe0
spec: 1.0.2-dev
go: go1.20.3
libseccomp: 2.5.4
root@master1-120:~# nerdctl -v
nerdctl version 1.4.0
2.2 实验过程
2.2.1 配置负载均衡器
# 主节点配置
root@HA1-keepalived-122:~# apt -y install keepalived
root@HA1-keepalived-122:/etc/keepalived# cat keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
acassen
}
notification_email_from Alexandre.Cassen@firewall.loc
smtp_server 192.168.200.1
smtp_connect_timeout 30
router_id LVS_DEVEL
}
vrrp_instance VI_1 {
state MASTER
interface eth0
garp_master_delay 10
smtp_alert
virtual_router_id 200 # 注意router-id不可以冲突
priority 100
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.119.200 label eth0:1 # 定义 VIP
}
}
root@HA1-keepalived-122:/etc/keepalived# systemctl restart keepalived.service
root@HA2-keepalived-122:~# systemctl enable keepalived
root@HA1-keepalived-122:/etc/keepalived# hostname -I
192.168.119.122 192.168.119.200
# 从节点配置
root@HA2-keepalived-123:~# apt -y install keepalived
# 使用主节点的配置,修改priority和state
root@HA2-keepalived-123:~# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived
global_defs {
notification_email {
acassen
}
notification_email_from Alexandre.Cassen@firewall.loc
smtp_server 192.168.200.1
smtp_connect_timeout 30
router_id LVS_DEVEL
}
vrrp_instance VI_1 {
state BACKUP
interface eth0
garp_master_delay 10
smtp_alert
virtual_router_id 200
priority 80
advert_int 1
authentication {
auth_type PASS
auth_pass 1111
}
virtual_ipaddress {
192.168.119.200 label eth0:1
}
}
root@HA2-keepalived-123:~# systemctl enable keepalived
root@HA2-keepalived-123:~# systemctl restart keepalived.service
# 默认keepalived是抢占模式,当主节点挂掉,从节点获取到VIP后,在主节点重新回复后,VIP会重新飘到主节点上
2.2.2 安装 HAproxy 做反向代理
# 在HA1节点上安装 HAproxy,HA2节点同理
root@HA1-keepalived-122:~# apt -y install haproxy
# 添加一个内核参数,使得即使我的主机没有该IP地址,但也监听,HA2节点同理
root@HA1-keepalived-122:~# vim /etc/sysctl.conf
net.ipv4.ip_nonlocal_bind = 1
root@HA1-keepalived-122:~# sysctl -p
# 更改配置,在配置文件的最后添加监听的地址和需要调度的后端真实服务器的IP地址和端口号,HA2节点同理
root@HA1-keepalived-122:~# vim /etc/haproxy/haproxy.cfg
...
listen k8s-api-6443
bind 192.168.119.200:6443
mode tcp
server server1 192.168.119.120:6443 check inter 3s fall 3 rise 3
server server2 192.168.119.121:6443 check inter 3s fall 3 rise 3
root@HA1-keepalived-122:~# systemctl restart haproxy.service
root@HA1-keepalived-122:~# systemctl enable haproxy
root@HA1-keepalived-122:~# ss -ntl | grep 6443
LISTEN 0 4096 192.168.119.200:6443 0.0.0.0:*
2.2.3 安装kubeadm、kubectl、kubelet
# 在master和node节点上安装
apt-get update && apt-get install -y apt-transport-https
curl https://mirrors.aliyun.com/kubernetes/apt/doc/apt-key.gpg | apt-key add -
cat <<EOF >/etc/apt/sources.list.d/kubernetes.list
deb https://mirrors.aliyun.com/kubernetes/apt/ kubernetes-xenial main
EOF
apt-get update
# node 节点上可不安装kubectl
apt-get install -y kubelet kubeadm kubectl
-- 注意需要关闭交换分区的使用,否则后续初始化集群会失败
2.2.4 内核参数优化
# 在master和node节点上操作
root@master1-120:~# cat /etc/sysctl.conf
net.ipv4.ip_forward=1
vm.max_map_count=262144
kernel.pid_max=4194303
fs.file-max=1000000
net.ipv4.tcp_max_tw_buckets=6000
net.netfilter.nf_conntrack_max=2097152
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
vm.swappiness=0
root@master1-120:~# sysctl -p
root@master1-120:~# vim /etc/modules-load.d/modules.conf
ip_vs
ip_vs_lc
ip_vs_lblc
ip_vs_lblcr
ip_vs_rr
ip_vs_wrr
ip_vs_sh
ip_vs_dh
ip_vs_fo
ip_vs_nq
ip_vs_sed
ip_vs_ftp
ip_vs_sh
ip_tables
ip_set
ipt_set
ipt_rpfilter
ipt_REJECT
ipip
xt_set
br_netfilter
nf_conntrack
overlay
root@master1-120:~# reboot
# 若在 sysctl -p 时遇到以下错误
sysctl: cannot stat /proc/sys/net/netfilter/nf_conntrack_max: No such file or directory
sysctl: cannot stat /proc/sys/net/bridge/bridge-nf-call-ip6tables: No such file or directory
sysctl: cannot stat /proc/sys/net/bridge/bridge-nf-call-iptables: No such file or directory
# 使用此方法解决
modprobe ip_conntrack
modprobe br_netfilter
# 检查重启后的内核模块与参数
root@master1-120:~# lsmod | grep br_netfilter
br_netfilter 32768 0
bridge 307200 1 br_netfilter
root@master1-120:~# sysctl -a | grep bridge-nf-call-iptables
net.bridge.bridge-nf-call-iptables = 1
2.2.5 下载 kubernetes镜像并进行 kubernetes 集群初始化
------ 下载镜像 ------
# 在所有 master 节点上操作
root@master1-120:~# vim images-down.sh
#!/bin/bash
nerdctl pull registry.cn-hangzhou.aliyuncs.com/google_containers/kube-apiserver:v1.27.4
nerdctl pull registry.cn-hangzhou.aliyuncs.com/google_containers/kube-controller-manager:v1.27.4
nerdctl pull registry.cn-hangzhou.aliyuncs.com/google_containers/kube-scheduler:v1.27.4
nerdctl pull registry.cn-hangzhou.aliyuncs.com/google_containers/kube-proxy:v1.27.4
nerdctl pull registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.9
nerdctl pull registry.cn-hangzhou.aliyuncs.com/google_containers/etcd:3.5.7-0
nerdctl pull registry.cn-hangzhou.aliyuncs.com/google_containers/coredns:v1.10.1
root@master1-120:~# bash images-down.sh
# 在所有node节点上安装
root@node2-125:~# cat images_node_install.sh
#!/bin/bash
nerdctl pull registry.cn-hangzhou.aliyuncs.com/google_containers/kube-proxy:v1.27.4
nerdctl pull registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.9
nerdctl pull registry.cn-hangzhou.aliyuncs.com/google_containers/etcd:3.5.7-0
nerdctl pull registry.cn-hangzhou.aliyuncs.com/google_containers/coredns:v1.10.1
root@node2-125:~# bash images_node_install.sh
------ 基于init文件初始化 ------
# 获取默认的 init 初始化文件
root@master1-120:~# kubeadm config print init-defaults > kubeadm-init.yaml
# 修改初始化文件
root@master1-120:~# cat kubeadm-init.yaml
apiVersion: kubeadm.k8s.io/v1beta3
bootstrapTokens:
- groups:
- system:bootstrappers:kubeadm:default-node-token
token: abcdef.0123456789abcdef
ttl: 24h0m0s
usages:
- signing
- authentication
kind: InitConfiguration
localAPIEndpoint:
advertiseAddress: 192.168.119.120 #当前主机监听IP
bindPort: 6443
nodeRegistration:
criSocket: unix:///var/run/containerd/containerd.sock # cri socket文件
imagePullPolicy: IfNotPresent
name: master1-120 # 当前主机名称,不可重复
taints: null
---
apiServer:
timeoutForControlPlane: 4m0s
apiVersion: kubeadm.k8s.io/v1beta3
certificatesDir: /etc/kubernetes/pki
clusterName: kubernetes
controlPlaneEndpoint: 192.168.119.200:6443 # 负载均衡
controllerManager: {}
dns: {}
etcd:
local:
dataDir: /var/lib/etcd
imageRepository: registry.cn-hangzhou.aliyuncs.com/google_containers #镜像仓库
kind: ClusterConfiguration
kubernetesVersion: 1.27.4
networking:
dnsDomain: cluster.local
podSubnet: 10.200.0.0/16 # pod子网
serviceSubnet: 10.100.0.0/16 # service 子网
scheduler: {}
--- # 指定kubelet使用systemd
kind: KubeletConfiguration
apiVersion: kubelet.config.k8s.io/v1beta1
cgroupDriver: systemd
--- # 指定kubeproxy使用ipvs
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
mode: ipvs
# 进行初始化
root@master1-120:~# kubeadm init --config kubeadm-init.yaml
···
Your Kubernetes control-plane has initialized successfully!
To start using your cluster, you need to run the following as a regular user:
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
Alternatively, if you are the root user, you can run:
export KUBECONFIG=/etc/kubernetes/admin.conf
You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
https://kubernetes.io/docs/concepts/cluster-administration/addons/
You can now join any number of control-plane nodes by copying certificate authorities
and service account keys on each node and then running the following as root:
kubeadm join 192.168.119.200:6443 --token abcdef.0123456789abcdef \
--discovery-token-ca-cert-hash sha256:b44a4d86fb5a82d5c9128bcfda69dee93daed45e0e4462fa87238d6aaa72969f \
--control-plane
Then you can join any number of worker nodes by running the following on each as root:
kubeadm join 192.168.119.200:6443 --token abcdef.0123456789abcdef \
--discovery-token-ca-cert-hash sha256:b44a4d86fb5a82d5c9128bcfda69dee93daed45e0e4462fa87238d6aaa72969f
# 初始化成功后
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
root@master1-120:~# kubectl get node
NAME STATUS ROLES AGE VERSION
master1-120 NotReady control-plane 6m56s v1.27.4
三、部署网络组件calico或flannel(上课有提供yaml文件)
# 部署calico
root@master1-120:~# kubectl apply -f calico3.26.1-ipip_ubuntu2204-k8s-1.27.x.yaml
root@master1-120:~# kubectl get pod -A
NAMESPACE NAME READY STATUS RESTARTS AGE
kube-system calico-kube-controllers-6655b6c4b-2kcq9 1/1 Running 0 2m39s
kube-system calico-node-rsmct 1/1 Running 0 2m40s
kube-system coredns-65dcc469f7-5tgr5 1/1 Running 0 15m
kube-system coredns-65dcc469f7-jmz4q 1/1 Running 0 15m
kube-system etcd-master1-120 1/1 Running 0 15m
kube-system kube-apiserver-master1-120 1/1 Running 0 15m
kube-system kube-controller-manager-master1-120 1/1 Running 0 15m
kube-system kube-proxy-sk4rd 1/1 Running 0 15m
kube-system kube-scheduler-master1-120 1/1 Running 0 15m
------ 添加worker节点,在所有node节点上操作 ------
root@node1-124:~# kubeadm join 192.168.119.200:6443 --token abcdef.0123456789abcdef --discovery-token-ca-cert-hash sha256:b44a4d86fb5a82d5c9128bcfda69dee93daed45e0e4462fa87238d6aaa72969f
# 查看node节点的状态是否为Ready,表示网络连接成功
root@master1-120:~# kubectl get nodes
NAME STATUS ROLES AGE VERSION
master1-120 Ready control-plane 20m v1.27.4
node1-124 Ready <none> 101s v1.27.4
node2-125 Ready <none> 98s v1.27.4
------ 添加 master 节点,在另一个master节点上操作 ------
# 在master1节点生成新的证书用于添加新的控制节点
root@master1-120:~# kubeadm init phase upload-certs --upload-certs
W0804 14:30:24.062364 26910 version.go:104] could not fetch a Kubernetes version from the internet: unable to get URL "https://dl.k8s.io/release/stable-1.txt": Get "https://dl.k8s.io/release/stable-1.txt": dial tcp [2600:1901:0:26f3::]:443: connect: network is unreachable
W0804 14:30:24.062466 26910 version.go:105] falling back to the local client version: v1.27.4
[upload-certs] Storing the certificates in Secret "kubeadm-certs" in the "kube-system" Namespace
[upload-certs] Using certificate key:
54a6374ae4b3fa378710c75417127e7d02f88fa423a6c72d6e280e8a9eb52176 # 获取的key
# 在 mster2 节点上操作
root@master2-121:~# kubeadm join 192.168.119.200:6443 --token abcdef.0123456789abcdef --discovery-token-ca-cert-hash sha256:b44a4d86fb5a82d5c9128bcfda69dee93daed45e0e4462fa87238d6aaa72969f --control-plane --certificate-key 54a6374ae4b3fa378710c75417127e7d02f88fa423a6c72d6e280e8a9eb52176
root@master1-120:~# kubectl get node
NAME STATUS ROLES AGE VERSION
master1-120 Ready control-plane 39m v1.27.4
master2-121 Ready control-plane 2m10s v1.27.4
node1-124 Ready <none> 21m v1.27.4
node2-125 Ready <none> 21m v1.27.4
** token 如果过期需要重新获取 **
# 查看当前的token
root@master1-120:~# kubeadm token list
TOKEN TTL EXPIRES USAGES DESCRIPTION EXTRA GROUPS
abcdef.0123456789abcdef 23h 2023-08-05T13:54:53Z authentication,signing <none> system:bootstrappers:kubeadm:default-node-token
koxmfc.pc4ir5wv24jz8zs1 1h 2023-08-04T16:30:24Z <none> Proxy for managing TTL for the kubeadm-certs secret <none>
# 重新获取
kubeadm token create
四、总结nginx的yaml文件的编写
yaml 文件的格式注意
- 大小写敏感
- 使用缩进表示层级关系
- 缩进时不允许使用 Tab 键,只允许使用空格
- 缩进的空格数量不重要,只需要相同层级的元素左侧对齐即可
- 使用 "#" 表示注释,从这个字符一直到行尾都会被解析为注释而忽略
- 相比于 json 更适用于配置文件
nginx 的 yaml 文件的详解
root@master1-120:~# cat nginx.yaml
kind: Deployment # 类型,deployment 控制器,使用 kubectl explain Deployment 进行查看
#apiVersion: extensions/v1beta1
apiVersion: apps/v1 # API 版本信息,kubectl explain Deployment.apiVersion
metadata: # pod 的元数据信息,kubectl explain Deployment.metadata
labels: # pod 标签,kubectl explain Deployment.metadata.labels
app: myserver-nginx-deployment-label # 标签的名称为 app
name: myserver-nginx-deployment # pod 的名称
namespace: myserver # pod 的Namespace,默认是default
spec: # 定义deployment中容器的详细信息,kubectl explain Deployment.spec
replicas: 1 # 创建的pod的副本数,即多少个pod
selector: # 定义标签选择器
matchLabels: # 定义匹配的标签,必须设置
app: myserver-nginx-selector # 匹配的目标标签
template: # 定义模板,描述创建的pod
metadata: # 定义模板元数据
labels: # 模板的标签,kubectl explain Deployment.spec.template.metadata.labels
app: myserver-nginx-selector # 定义标签,kubectl explain Deployment.spec.selector.matchLabels
spec: # 定义pod信息
containers: # 定义pod中容器列表,可以多个至少一个,不可动态增减容器
- name: myserver-nginx-container # 容器名称
image: registry.cn-hangzhou.aliyuncs.com/zhangshijie/nginx:1.22.0-alpine # 镜像地址
#command: ["/apps/tomcat/bin/run_tomcat.sh"] # 容器启动执行的命令或脚本
#imagePullPolicy: IfNotPresent
imagePullPolicy: Always # 拉取镜像策略
ports: # 定义容器端口列表
- containerPort: 80 # 定义端口
protocol: TCP # 端口协议
name: http # 端口名称
- containerPort: 443
protocol: TCP
name: https
env: # 配置环境标量
- name: "password"
value: "123456"
- name: "age"
value: "18"
# resources: # 对资源请求设置和限制设置
# limits:
# cpu: 2
# memory: 2Gi
# requests:
# cpu: 500m
# memory: 1Gi
---
kind: Service # 定义类型为Service
apiVersion: v1 # Service API 版本
metadata: # 定义 Service 元数据
labels: # 定义标签
app: myserver-nginx-service-label # 标签的内容
name: myserver-nginx-service # Service的名称,会被DNS解析
namespace: myserver # Service属于哪一个命名空间
spec: # 定义 Service 详细信息
type: NodePort # Service 的类型
ports: # 定义访问端口
- name: http
port: 80
protocol: TCP
targetPort: 80
nodePort: 30004
- name: https
port: 443
protocol: TCP
targetPort: 443
nodePort: 30443
selector: # Service的标签选择器,定义访问的目标pod
app: myserver-nginx-selector # 将流量转发到选择的pod上,需等于kubectl explain Deployment.spec.selector.matchLabels
lable 标签,对k8s中资源做过滤,匹配,但不起实际功能作用
service 把流量转发给pod,通常使用标签进行匹配
通过namespace隔离不同项目的容器
镜像拉取策略
- none: 不管本地有没有镜像都不拉取
- Always: 不管本地有没有镜像都从镜像仓库拉取
- IfNotPresent: 本地有就拉去本地镜像,没有也不从镜像仓库拉取
CPU 的限制按照时间的方式进行切分,1000m 表示 1 核
建议资源请求的设置和资源的限制设置相同
port: k8s 中通过 service 访问的端口
targetport: pod 节点中服务所运行的实际端口
nodeport: 宿主机所监听的端口
转发流程: 客户 -> nodeport -> port -> targetport
五、部署web服务并进行服务的访问
root@master1-120:~# mkdir /data/nginx-tomcat-case -p
root@master1-120:~# cd /data/nginx-tomcat-case/
root@master1-120:/data/nginx-tomcat-case# ls
myserver-namespace.yaml nginx.yaml tomcat.yaml
# 创建一个myserver的命名空间
root@master1-120:/data/nginx-tomcat-case# cat myserver-namespace.yaml
apiVersion: v1
kind: Namespace
metadata:
name: myserver
labels:
name: myserver
# 创建服务
root@master1-120:/data/nginx-tomcat-case# kubectl apply -f myserver-namespace.yaml -f nginx.yaml -f tomcat.yaml
# 查看容器
root@master1-120:/data/nginx-tomcat-case# kubectl get pod -n myserver
NAME READY STATUS RESTARTS AGE
myserver-nginx-deployment-548f767db4-x24fl 1/1 Running 0 4m25s
myserver-tomcat-app1-deployment-78974d9679-5njf2 1/1 Running 0 4m25s
# 使用 -o wide 选项显示更加详细的信息
root@master1-120:~# kubectl get pod -n myserver -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
myserver-nginx-deployment-548f767db4-x24fl 1/1 Running 0 13m 10.200.169.1 node2-125 <none> <none>
myserver-tomcat-app1-deployment-78974d9679-5njf2 1/1 Running 0 13m 10.200.56.1 node1-124 <none> <none>
# 查看在k8s中的service端口信息 svc
root@master1-120:~# kubectl get svc -n myserver -o wide
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE SELECTOR
myserver-nginx-service NodePort 10.100.5.185 <none> 80:30004/TCP,443:30443/TCP 16m app=myserver-nginx-selector
myserver-tomcat-app1-service NodePort 10.100.14.214 <none> 80:30005/TCP 16m app=myserver-tomcat-app1-selector
# 查看pod的端口信息
root@master1-120:~# kubectl get ep -n myserver -o wide
NAME ENDPOINTS AGE
myserver-nginx-service 10.200.169.1:443,10.200.169.1:80 18m
myserver-tomcat-app1-service 10.200.56.1:8080 18m
六、配置负载均衡器转发到后端服务
# 编辑 HA 节点,两个节点做相同操作
root@HA1-keepalived-122:~# vim /etc/haproxy/haproxy.cfg
# 在配置文件中添加监听地址和后端转发地址和端口
listen k8s-pod-server
bind 192.168.119.200:30004
mode tcp
server server1 192.168.119.124:30004 check inter 3s fall 3 rise 3
server server2 192.168.119.125:30004 check inter 3s fall 3 rise 3
root@HA1-keepalived-122:~# systemctl restart haproxy.service
# 测试结果如下
【推荐】编程新体验,更懂你的AI,立即体验豆包MarsCode编程助手
【推荐】凌霞软件回馈社区,博客园 & 1Panel & Halo 联合会员上线
【推荐】抖音旗下AI助手豆包,你的智能百科全书,全免费不限次数
【推荐】博客园社区专享云产品让利特惠,阿里云新客6.5折上折
【推荐】轻量又高性能的 SSH 工具 IShell:AI 加持,快人一步
· 清华大学推出第四讲使用 DeepSeek + DeepResearch 让科研像聊天一样简单!
· 实操Deepseek接入个人知识库
· 易语言 —— 开山篇
· 一个费力不讨好的项目,让我损失了近一半的绩效!
· 【全网最全教程】使用最强DeepSeekR1+联网的火山引擎,没有生成长度限制,DeepSeek本体