每天一点基础K8S--使用基础ingress进行灰度发布
利用ingress实现灰度发布
1、背景
ingress可以实现七层负载，可以根据请求header不同将流量代理到后端不同的service服务。
比如：后端网站进行了更新，为了不全网更换，可以利用ingress进行流量切分，先将部分流量切到新service，其余流量先不变动，等验证完成后再全网切换。
2、基于header的流量切分
# 先创建两个版本的service，version-1理解为老版本，version-2理解为新版本。
[root@master-worker-node-1 ingress]# cat my-web-version-v1.yaml
---
apiVersion: v1
kind: Service
metadata:
name: web-version-v1 # web-version-2
spec:
type: ClusterIP
selector:
version: v1 # v2
ports:
- port: 80
targetPort: 80
protocol: TCP
name: nginx-v1 # nginx-v2
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: my-web-version-1 # my-web-version-2
labels:
func: for-web-version-1 # for-web-version-2
spec:
replicas: 2
selector:
matchLabels:
version: v1 # v2
template:
metadata:
labels:
version: v1 # v2
spec:
containers:
- name: my-web-version-1 # my-web-version-2
image: nginx
imagePullPolicy: IfNotPresent
lifecycle:
postStart:
exec:
command: ["/bin/sh","-c","echo this is my web server, and the version is v1. > /usr/share/nginx/html/index.html" ] # v2
[root@master-worker-node-1 ingress]# kubectl get all
NAME READY STATUS RESTARTS AGE
pod/my-web-version-1-5786885c4c-7p5h4 1/1 Running 0 87m
pod/my-web-version-1-5786885c4c-pdsml 1/1 Running 0 87m
pod/my-web-version-2-9fdbd45cf-9m5b2 1/1 Running 0 136m
pod/my-web-version-2-9fdbd45cf-stkww 1/1 Running 0 136m
pod/tomcat-deploy-64d6489dd9-gn88s 1/1 Running 0 39h
pod/tomcat-deploy-64d6489dd9-nqf98 1/1 Running 0 39h
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
service/kubernetes ClusterIP 10.96.0.1 <none> 443/TCP 4d12h
service/tomcat ClusterIP 10.110.26.143 <none> 8080/TCP,8009/TCP 39h
service/web-version-v1 ClusterIP 10.98.172.137 <none> 80/TCP 168m
service/web-version-v2 ClusterIP 10.102.145.135 <none> 80/TCP 136m
NAME READY UP-TO-DATE AVAILABLE AGE
deployment.apps/my-web-version-1 2/2 2 2 162m
deployment.apps/my-web-version-2 2/2 2 2 136m
deployment.apps/tomcat-deploy 2/2 2 2 39h
NAME DESIRED CURRENT READY AGE
replicaset.apps/my-web-version-1-5786885c4c 2 2 2 87m
replicaset.apps/my-web-version-1-5fb7bbc67b 0 0 0 140m
replicaset.apps/my-web-version-2-9fdbd45cf 2 2 2 136m
replicaset.apps/tomcat-deploy-64d6489dd9 2 2 2 39h
# 测试集群内访问正常。
[root@master-worker-node-1 ingress]# curl 10.98.172.137
this is my web server, and the version is v1.
[root@master-worker-node-1 ingress]# curl 10.102.145.135
this is my web server, and the version is v2.
# 创建version-1的ingress规则
[root@master-worker-node-1 ingress]# cat my-web-version-v1-ingress.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: version-1-ingress
namespace: default
annotations:
kubernetes.io/ingress.class: "nginx"
spec:
rules: #定义后端转发的规则
- host: nginx.test.com #通过域名进行转发
http:
paths:
- path: / #配置访问路径，如果通过url进行转发，需要修改；空默认为访问的路径为"/"
pathType: Prefix
backend: #配置后端服务
service:
name: web-version-v1
port:
number: 80
# 在集群外访问域名，被代理到version-1的后端服务器
[root@master-worker-node-1 ingress]# curl nginx.test.com:1080 # 1080端口是因为nginx-ingress-controller进行了高可用配置。
this is my web server, and the version is v1.
# 创建version-2的ingress规则
[root@master-worker-node-1 ingress]# cat my-web-version-v2-ingress.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
annotations:
kubernetes.io/ingress.class: "nginx"
nginx.ingress.kubernetes.io/canary: "true"
nginx.ingress.kubernetes.io/canary-by-header: "Region"
nginx.ingress.kubernetes.io/canary-by-header-pattern: "CD|BJ"
name: version-2-ingress
spec:
rules:
- host: nginx.test.com
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: web-version-v2
port:
number: 80
# 创建ingress
[root@master-worker-node-1 ingress]# kubectl get ingress | grep -E "NAME|nginx.test.com"
NAME CLASS HOSTS ADDRESS PORTS AGE
version-1-ingress <none> nginx.test.com 192.168.122.132,192.168.122.182 80 13m
version-2-ingress <none> nginx.test.com 192.168.122.132,192.168.122.182 80 54s
# 测试在集群外携带相应的header访问
[root@master-worker-node-1 ingress]# curl -H "Region: CD" nginx.test.com:1080
this is my web server, and the version is v2.
[root@master-worker-node-1 ingress]# curl -H "Region: BJ" nginx.test.com:1080
this is my web server, and the version is v2.
[root@master-worker-node-1 ingress]# curl -H "Region: CQ" nginx.test.com:1080
this is my web server, and the version is v1.
# 只有当Region与ingress规则匹配时，才能访问version-2。
3、基于cookie的流量切分
# 基于header的流量切分和基于header的流量切分基本一致，
# 创建version-v3的service
[root@master-worker-node-1 ingress]# kubectl get service web-version-v3
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
web-version-v3 ClusterIP 10.110.51.93 <none> 80/TCP 104s
[root@master-worker-node-1 ingress]# kubectl describe svc web-version-v3
Name: web-version-v3
Namespace: default
Labels: <none>
Annotations: <none>
Selector: version=v3
Type: ClusterIP
IP Family Policy: SingleStack
IP Families: IPv4
IP: 10.110.51.93
IPs: 10.110.51.93
Port: nginx-v3 80/TCP
TargetPort: 80/TCP
Endpoints: 10.244.31.25:80,10.244.54.16:80
Session Affinity: None
Events: <none>
# 使用ingress代理web-version-v3 服务
[root@master-worker-node-1 ingress]# cat my-web-version-v3-ingress.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
annotations:
kubernetes.io/ingress.class: "nginx"
nginx.ingress.kubernetes.io/canary: "true"
nginx.ingress.kubernetes.io/canary-by-cookie: "user_from_cq" # 只代理请求cookie中携带user_from_CQ的流量。
name: version-2-ingress
spec:
rules:
- host: nginx.test.com
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: web-version-v3
port:
number: 80
# 此时的ingress服务关联ingress-controller-nginx成功
[root@master-worker-node-1 ingress]# kubectl get ingress
NAME CLASS HOSTS ADDRESS PORTS AGE
ingress-myapp <none> tomcat.test.com 192.168.122.132,192.168.122.182 80 6d
nginx-canary <none> myweb.com 192.168.122.132,192.168.122.182 80 4d9h
version-1-ingress <none> nginx.test.com 192.168.122.132,192.168.122.182 80 4d9h
version-2-ingress <none> nginx.test.com 192.168.122.132,192.168.122.182 80 4d8h
version-3-ingress <none> nginx.test.com 192.168.122.132,192.168.122.182 80 41s
# 测试之前需要先删除version-2-ingress这个ingress
[root@master-worker-node-1 ingress]# kubectl delete -f my-web-version-v2-ingress.yaml
ingress.networking.k8s.io "version-2-ingress" deleted
# 测试发现，只有写在user_from_cq=always的流量会被调度到service-version-v3中。
[root@master-worker-node-1 ingress]# curl nginx.test.com:1080
this is my web server, and the version is v1.
[root@master-worker-node-1 ingress]# curl --cookie 'user_from_cq=always' nginx.test.com:1080
this is my web server, and the version is v3.
4、基于权重进行流量切分
# 基于流量比例切分与其他类似。
[root@master-worker-node-1 ingress]# cat my-web-version-v3-ingress-2.yaml
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
annotations:
kubernetes.io/ingress.class: "nginx"
nginx.ingress.kubernetes.io/canary: "true"
nginx.ingress.kubernetes.io/canary-weight: "20" # 理论上，将20%的流量分担掉web-version-3中
name: version-3-ingress-2
spec:
rules:
- host: nginx.test.com
http:
paths:
- path: /
pathType: Prefix
backend:
service:
name: web-version-v3
port:
number: 80
[root@master-worker-node-1 ingress]# kubectl apply -f my-web-version-v3-ingress-2.yaml
ingress.networking.k8s.io/version-3-ingress-2 created
[root@master-worker-node-1 ingress]#
[root@master-worker-node-1 ingress]# kubectl get ingress
NAME CLASS HOSTS ADDRESS PORTS AGE
ingress-myapp <none> tomcat.test.com 192.168.122.132,192.168.122.182 80 6d1h
nginx-canary <none> myweb.com 192.168.122.132,192.168.122.182 80 4d9h
version-1-ingress <none> nginx.test.com 192.168.122.132,192.168.122.182 80 4d9h
version-3-ingress-2 <none> nginx.test.com 192.168.122.132,192.168.122.182 80 66s
# 测试发现流量基本满足20%
[root@master-worker-node-1 ingress]# while true; do curl nginx.test.com:1080; sleep 2 ; done
this is my web server, and the version is v3.
this is my web server, and the version is v1.
this is my web server, and the version is v3.
this is my web server, and the version is v1.
this is my web server, and the version is v3.
this is my web server, and the version is v1.
this is my web server, and the version is v1.
this is my web server, and the version is v1.
this is my web server, and the version is v1.
this is my web server, and the version is v1.
this is my web server, and the version is v1.
this is my web server, and the version is v1.
this is my web server, and the version is v1.
this is my web server, and the version is v1.
this is my web server, and the version is v1.
this is my web server, and the version is v1.
5、小结
1、通过配置灰度发布版本不同的header或者cookie或者权重等，可以将流量负载到新版本中。
2、测试的时候发现，使用灰度发布进行测试的时候，只能有两个ingress规则存在，否则将代理失败。
