nginx-Ingress-k8s内部7层负载均衡器
root@slave001:~/ingress# cat default-backend.yaml apiVersion: apps/v1 kind: Deployment metadata: name: default-http-backend labels: k8s-app: default-http-backend namespace: kube-system spec: replicas: 1 selector: matchLabels: k8s-app: default-http-backend template: metadata: labels: k8s-app: default-http-backend spec: terminationGracePeriodSeconds: 60 containers: - name: default-http-backend # Any image is permissable as long as: # 1. It serves a 404 page at / # 2. It serves 200 on a /healthz endpoint image: registry.cn-hangzhou.aliyuncs.com/hachikou/defaultbackend:1.0 livenessProbe: httpGet: path: /healthz #这个URI是 nginx-ingress-controller中nginx里配置好的localtion port: 8080 scheme: HTTP initialDelaySeconds: 30 #30s检测一次/healthz timeoutSeconds: 5 ports: - containerPort: 8080 # resources: # limits: # cpu: 10m # memory: 20Mi # requests: # cpu: 10m # memory: 20Mi nodeName: 192.168.192.152 --- apiVersion: v1 kind: Service #为default backend 创建一个service metadata: name: default-http-backend namespace: kube-system labels: k8s-app: default-http-backend spec: ports: - port: 80 targetPort: 8080 selector: k8s-app: default-http-backend
root@slave001:~/ingress# cat nginx-ingress-controller-rbac.yml --- apiVersion: v1 kind: ServiceAccount metadata: name: nginx-ingress-serviceaccount #创建一个serveerAcount namespace: kube-system --- apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRole metadata: name: nginx-ingress-clusterrole #这个ServiceAcount所绑定的集群角色 rules: - apiGroups: - "" resources: #此集群角色的权限,它能操作的API资源 - configmaps - endpoints - nodes - pods - secrets verbs: - list - watch - apiGroups: - "" resources: - nodes verbs: - get - apiGroups: - "" resources: - services verbs: - get - list - watch - apiGroups: - "extensions" resources: - ingresses verbs: - get - list - watch - apiGroups: - "" resources: - events verbs: - create - patch - apiGroups: - "extensions" resources: - ingresses/status verbs: - update --- apiVersion: rbac.authorization.k8s.io/v1beta1 kind: Role metadata: name: nginx-ingress-role #这是一个角色,而非集群角色 namespace: kube-system rules: #角色的权限 - apiGroups: - "" resources: - configmaps - pods - secrets - namespaces verbs: - get - apiGroups: - "" resources: - configmaps resourceNames: # Defaults to "<election-id>-<ingress-class>" # Here: "<ingress-controller-leader>-<nginx>" # This has to be adapted if you change either parameter # when launching the nginx-ingress-controller. - "ingress-controller-leader-nginx" verbs: - get - update - apiGroups: - "" resources: - configmaps verbs: - create - apiGroups: - "" resources: - endpoints verbs: - get - create - update --- apiVersion: rbac.authorization.k8s.io/v1beta1 kind: RoleBinding #角色绑定 metadata: name: nginx-ingress-role-nisa-binding namespace: kube-system roleRef: apiGroup: rbac.authorization.k8s.io kind: Role name: nginx-ingress-role subjects: - kind: ServiceAccount name: nginx-ingress-serviceaccount #绑定在这个用户 namespace: kube-system --- apiVersion: rbac.authorization.k8s.io/v1beta1 kind: ClusterRoleBinding #集群绑定 metadata: name: nginx-ingress-clusterrole-nisa-binding roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: nginx-ingress-clusterrole subjects: - kind: ServiceAccount name: nginx-ingress-serviceaccount #集群绑定到这个serviceacount namespace: kube-system #集群角色是可以跨namespace,但是这里只指明给这个namespce来使用
root@slave001:~/ingress# cat nginx-ingress-controller.yaml apiVersion: apps/v1 kind: Deployment metadata: name: nginx-ingress-controller labels: k8s-app: nginx-ingress-controller namespace: kube-system spec: replicas: 1 selector: matchLabels: k8s-app: nginx-ingress-controller template: metadata: labels: k8s-app: nginx-ingress-controller spec: # hostNetwork makes it possible to use ipv6 and to preserve the source IP correctly regardless of docker configuration # however, it is not a hard dependency of the nginx-ingress-controller itself and it may cause issues if port 10254 already is taken on the host # that said, since hostPort is broken on CNI (https://github.com/kubernetes/kubernetes/issues/31307) we have to use hostNetwork where CNI is used # like with kubeadm # hostNetwork: true #注释表示不使用宿主机的80口, terminationGracePeriodSeconds: 60 hostNetwork: true #表示容器使用和宿主机一样的网络 serviceAccountName: nginx-ingress-serviceaccount #引用前面创建的serviceacount containers: - image: registry.cn-hangzhou.aliyuncs.com/peter1009/nginx-ingress-controller:0.20.0 #容器使用的镜像 name: nginx-ingress-controller #容器名 #readinessProbe: #启动这个服务时要验证/healthz 端口10254会在运行的node上监听。 # httpGet: # path: /healthz # port: 10254 # scheme: HTTP #livenessProbe: # httpGet: # path: /healthz # port: 10254 # scheme: HTTP # initialDelaySeconds: 60 #每隔10做健康检查 # timeoutSeconds: 1 ports: - containerPort: 80 hostPort: 80 #80映射到80 # - containerPort: 443 # hostPort: 443 env: - name: POD_NAME valueFrom: fieldRef: fieldPath: metadata.name - name: POD_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace args: - /nginx-ingress-controller - --default-backend-service=$(POD_NAMESPACE)/default-http-backend # - --default-ssl-certificate=$(POD_NAMESPACE)/ingress-secret #这是启用Https时用的 # nodeSelector: #指明运行在哪,此IP要和default backend是同一个IP # kubernetes.io/hostname: 10.3.1.17 #上面映射到了hostport80,确保此IP80,443没有占用. # nodeName: 192.168.192.152
root@slave001:~/ingress# cat demo-tomcat.yaml apiVersion: v1 kind: Service metadata: name: tomcat namespace: default spec: selector: app: tomcat release: canary ports: - name: http targetPort: 8080 port: 8080 - name: ajp targetPort: 8009 port: 8009 --- apiVersion: apps/v1 kind: Deployment metadata: name: tomcat-deploy namespace: default spec: replicas: 2 selector: matchLabels: app: tomcat release: canary template: metadata: labels: app: tomcat release: canary spec: containers: - name: tomcat image: 192.168.192.155:80/chuan/tomcat:8.5.34-jre8-alpine #tomcat:8.5.34-jre8-alpine ports: - name: http containerPort: 8080 name: ajp containerPort: 8009
root@slave001:~/ingress# cat ingress-myapp.yaml apiVersion: extensions/v1beta1 #api版本 kind: Ingress #清单类型 metadata: #元数据 name: ingress-myapp #ingress的名称 namespace: default #所属名称空间 annotations: #注解信息 kubernetes.io/ingress.class: "nginx" # 将配置传到ingress controller中的nginx配置中 spec: #规格 rules: #定义后端转发的规则 - host: www.chuan.com #通过域名进行转发 http: paths: - path: #配置访问路径,如果通过url进行转发,需要修改;空默认为访问的路径为"/" backend: #配置后端服务 serviceName: tomcat servicePort: 8080
root@slave001:~/ingress# kubectl describe ingress ingress-myapp Name: ingress-myapp Namespace: default Address: Default backend: default-http-backend:80 (10.200.122.229:8080) Rules: Host Path Backends ---- ---- -------- www.chuan.com tomcat:8080 (10.200.111.38:8080,10.200.163.106:8080) Annotations: kubernetes.io/ingress.class: nginx Events: <none>
root@slave001:~/ingress# kubectl exec -it nginx-ingress-controller-559847d7f9-fpkq4 -n kube-system -- cat nginx.conf .... ## start server www.chuan.com server { server_name www.chuan.com ; listen 80; listen [::]:80; set $proxy_upstream_name "-"; location / { set $namespace "default"; set $ingress_name "ingress-myapp"; set $service_name "tomcat"; set $service_port "8080"; set $location_path "/"; rewrite_by_lua_block { balancer.rewrite() } log_by_lua_block { balancer.log() monitor.call() } port_in_redirect off; set $proxy_upstream_name "default-tomcat-8080"; client_max_body_size 1m; proxy_set_header Host $best_http_host; # Pass the extracted client certificate to the backend # Allow websocket connections proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection $connection_upgrade; proxy_set_header X-Request-ID $req_id; proxy_set_header X-Real-IP $the_real_ip; proxy_set_header X-Forwarded-For $the_real_ip; proxy_set_header X-Forwarded-Host $best_http_host; proxy_set_header X-Forwarded-Port $pass_port; proxy_set_header X-Forwarded-Proto $pass_access_scheme; proxy_set_header X-Original-URI $request_uri; proxy_set_header X-Scheme $pass_access_scheme; # Pass the original X-Forwarded-For proxy_set_header X-Original-Forwarded-For $http_x_forwarded_for; # mitigate HTTPoxy Vulnerability # https://www.nginx.com/blog/mitigating-the-httpoxy-vulnerability-with-nginx/ proxy_set_header Proxy ""; # Custom headers to proxied server proxy_connect_timeout 5s; proxy_send_timeout 60s; proxy_read_timeout 60s; proxy_buffering off; proxy_buffer_size 4k; proxy_buffers 4 4k; proxy_request_buffering on; proxy_http_version 1.1; proxy_cookie_domain off; proxy_cookie_path off; # In case of errors try the next upstream server before returning an error proxy_next_upstream error timeout; proxy_next_upstream_tries 3; proxy_pass http://upstream_balancer; proxy_redirect off; } } ## end server www.chuan.com ....
root@slave001:~/ingress# cat demo-tomcat2.yaml apiVersion: v1 kind: Service metadata: name: tomcat2 namespace: default spec: selector: app: tomcat2 release: canary2 ports: - name: http targetPort: 8080 port: 8080 - name: ajp targetPort: 8009 port: 8009 --- apiVersion: apps/v1 kind: Deployment metadata: name: tomcat-deploy2 namespace: default spec: replicas: 2 selector: matchLabels: app: tomcat2 release: canary2 template: metadata: labels: app: tomcat2 release: canary2 spec: containers: - name: tomcat2 image: 192.168.192.155:80/chuan/tomcat:8.5.32-jre8-alpine ports: - name: http containerPort: 8080 name: ajp containerPort: 8009
root@slave001:~/ingress# cat ingress_multi-host.yaml apiVersion: extensions/v1beta1 #api版本 kind: Ingress metadata: name: nginx-web namespace: default annotations: kubernetes.io/ingress.class: "nginx" ##将配置传到ingress controller中的nginx配置中 spec: rules: - host: www.chuan.com http: paths: - path: backend: serviceName: tomcat servicePort: 8080 - host: mobile.chuan.com http: paths: - path: backend: serviceName: tomcat2 servicePort: 8080
openssl genrsa -out tls.key 2048 openssl req -new -x509 -key tls.key -out tls.crt -subj /C=CN/ST=Beijing/L=Beijing/O=DevOps/CN=www.chuan.com kubectl create secret tls tomcat-ingress-secret --cert=tls.crt --key=tls.key kubectl get secret kubectl describe secret tomcat-ingress-secret
root@slave001:~/ingress/tls# cat ingress-tomcat-tls.yaml apiVersion: extensions/v1beta1 kind: Ingress metadata: name: ingress-tomcat-tls namespace: default annotations: kubernetes.io/ingress.class: "nginx" # 将配置传到ingress controller中的nginx配置中 spec: tls: - hosts: - www.chuan.com secretName: tomcat-ingress-secret rules: - host: www.chuan.com http: paths: - path: backend: serviceName: tomcat servicePort: 8080
openssl genrsa -out tls.key 2048 openssl req -new -x509 -key tls.key -out tls.crt -subj /C=CN/ST=Beijing/L=Beijing/O=DevOps/CN=mobile.chuan.com kubectl create secret tls tomcat-ingress-secret2 --cert=tls.crt --key=tls.key kubectl get secret kubectl describe secret tomcat-ingress-secret
root@slave001:~/ingress/tls/tls2# cat ingress-tomcat-tls.yaml apiVersion: extensions/v1beta1 kind: Ingress metadata: name: ingress-tomcat-tls namespace: default annotations: kubernetes.io/ingress.class: "nginx" # 将配置传到ingress controller中的nginx配置中 spec: tls: - hosts: - www.chuan.com secretName: tomcat-ingress-secret rules: - host: www.chuan.com http: paths: - path: backend: serviceName: tomcat servicePort: 8080 tls: - hosts: - mobile.chuan.com secretName: tomcat-ingress-secret2 rules: - host: mobile.chuan.com http: paths: - path: backend: serviceName: tomcat2 servicePort: 8080
root@slave001:~/ingress# cat ingress-url.yaml apiVersion: networking.k8s.io/v1beta1 kind: Ingress metadata: name: nginx-web namespace: chuan annotations: kubernetes.io/ingress.class: "nginx" ##指定Ingress Controller的类型 nginx.ingress.kubernetes.io/use-regex: "true" ##指定后面rules定义的path可以使用正则表达式 nginx.ingress.kubernetes.io/proxy-connect-timeout: "600" ##连接超时时间,默认为5s nginx.ingress.kubernetes.io/proxy-send-timeout: "600" ##后端服务器回转数据超时时间,默认为60s nginx.ingress.kubernetes.io/proxy-read-timeout: "600" ##后端服务器响应超时时间,默认为60s nginx.ingress.kubernetes.io/proxy-body-size: "10m" ##客户端上传文件,最大大小,默认为20m #nginx.ingress.kubernetes.io/rewrite-target: / ##URL重写 nginx.ingress.kubernetes.io/app-root: /index.html spec: rules: - host: www.chuan.com http: paths: - path: /url1 backend: serviceName: tomcat1-service servicePort: 8080 - path: /url2 backend: serviceName: tomcat2-service servicePort: 8080 - host: mobile.chuan.com http: paths: - path: /url1 backend: serviceName: tomcat1-service servicePort: 8080 - path: /url2 backend: serviceName: tomcat2-service servicePort: 8080