k8s-ingress部署测试以及深入理解
1、ingress 部署有两种方式。本次采用DaemonSet部署。
apiVersion: v1
kind: Namespace
metadata:
name: ingress-nginx
---
apiVersion: extensions/v1beta1
kind: Deployment
metadata:
name: default-http-backend
labels:
app.kubernetes.io/name: default-http-backend
app.kubernetes.io/part-of: ingress-nginx
namespace: ingress-nginx
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: default-http-backend
app.kubernetes.io/part-of: ingress-nginx
template:
metadata:
labels:
app.kubernetes.io/name: default-http-backend
app.kubernetes.io/part-of: ingress-nginx
spec:
terminationGracePeriodSeconds: 60
containers:
- name: default-http-backend
# Any image is permissible as long as:
# 1. It serves a 404 page at /
# 2. It serves 200 on a /healthz endpoint
image: k8s.gcr.io/defaultbackend-amd64:1.5
livenessProbe:
httpGet:
path: /healthz
port: 8080
scheme: HTTP
initialDelaySeconds: 30
timeoutSeconds: 5
ports:
- containerPort: 8080
resources:
limits:
cpu: 10m
memory: 20Mi
requests:
cpu: 10m
memory: 20Mi
---
apiVersion: v1
kind: Service
metadata:
name: default-http-backend
namespace: ingress-nginx
labels:
app.kubernetes.io/name: default-http-backend
app.kubernetes.io/part-of: ingress-nginx
spec:
ports:
- port: 80
targetPort: 8080
selector:
app.kubernetes.io/name: default-http-backend
app.kubernetes.io/part-of: ingress-nginx
---
kind: ConfigMap
apiVersion: v1
metadata:
name: nginx-configuration
namespace: ingress-nginx
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
---
kind: ConfigMap
apiVersion: v1
metadata:
name: tcp-services
namespace: ingress-nginx
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
---
kind: ConfigMap
apiVersion: v1
metadata:
name: udp-services
namespace: ingress-nginx
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: nginx-ingress-serviceaccount
namespace: ingress-nginx
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
name: nginx-ingress-clusterrole
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
rules:
- apiGroups:
- ""
resources:
- configmaps
- endpoints
- nodes
- pods
- secrets
verbs:
- list
- watch
- apiGroups:
- ""
resources:
- nodes
verbs:
- get
- apiGroups:
- ""
resources:
- services
verbs:
- get
- list
- watch
- apiGroups:
- "extensions"
resources:
- ingresses
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- events
verbs:
- create
- patch
- apiGroups:
- "extensions"
resources:
- ingresses/status
verbs:
- update
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: Role
metadata:
name: nginx-ingress-role
namespace: ingress-nginx
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
rules:
- apiGroups:
- ""
resources:
- configmaps
- pods
- secrets
- namespaces
verbs:
- get
- apiGroups:
- ""
resources:
- configmaps
resourceNames:
# Defaults to "<election-id>-<ingress-class>"
# Here: "<ingress-controller-leader>-<nginx>"
# This has to be adapted if you change either parameter
# when launching the nginx-ingress-controller.
- "ingress-controller-leader-nginx"
verbs:
- get
- update
- apiGroups:
- ""
resources:
- configmaps
verbs:
- create
- apiGroups:
- ""
resources:
- endpoints
verbs:
- get
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: RoleBinding
metadata:
name: nginx-ingress-role-nisa-binding
namespace: ingress-nginx
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: nginx-ingress-role
subjects:
- kind: ServiceAccount
name: nginx-ingress-serviceaccount
namespace: ingress-nginx
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
name: nginx-ingress-clusterrole-nisa-binding
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: nginx-ingress-clusterrole
subjects:
- kind: ServiceAccount
name: nginx-ingress-serviceaccount
namespace: ingress-nginx
---
apiVersion: extensions/v1beta1
#apiVersion: v1
kind: DaemonSet
metadata:
name: nginx-ingress-controller
namespace: ingress-nginx
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
spec:
# replicas: 1 注意使用DaemonSet部署不要设置re.默认为会给标签上面会有一个副本
# selector: #注释掉
# matchLabels:
# app.kubernetes.io/name: ingress-nginx #注释掉
# app.kubernetes.io/part-of: ingress-nginx #注释掉
# sIngress: true #主调式
template:
metadata:
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
annotations:
prometheus.io/port: "10254"
prometheus.io/scrape: "true"
spec:
serviceAccountName: nginx-ingress-serviceaccount
nodeSelector:
isIngress: "true"
hostNetwork: true ####启用主机网络 。会霸占主机80端口，如果主机80端口已经被使用。则会启动失败。
containers:
- name: nginx-ingress-controller
image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.20.0
args:
- /nginx-ingress-controller
- --default-backend-service=$(POD_NAMESPACE)/default-http-backend
- --configmap=$(POD_NAMESPACE)/nginx-configuration
- --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services
- --udp-services-configmap=$(POD_NAMESPACE)/udp-services
- --publish-service=$(POD_NAMESPACE)/ingress-nginx
- --annotations-prefix=nginx.ingress.kubernetes.io
securityContext:
capabilities:
drop:
- ALL
add:
- NET_BIND_SERVICE
# www-data -> 33
runAsUser: 33
env:
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
ports:
- name: http
containerPort: 80
- name: https
containerPort: 443
livenessProbe:
failureThreshold: 3
httpGet:
path: /healthz
port: 10254
scheme: HTTP
initialDelaySeconds: 10
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
readinessProbe:
failureThreshold: 3
httpGet:
path: /healthz
port: 10254
scheme: HTTP
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
---
2、部署结果
kubectl apply -f mandatory.yaml [root@VM_0_48_centos ingress-nginx]# kubectl get pods -n ingress-nginx NAME READY STATUS RESTARTS AGE default-http-backend-85b8b595f9-j5twg 1/1 Running 1 30h nginx-ingress-controller-mrvzh 1/1 Running 0 104m nginx-ingress-controller-pgp9t 1/1 Running 0 105m nginx-ingress-controller-vd7v6 1/1 Running 0 104m [root@VM_0_48_centos ingress-nginx]#
3、测试
[root@VM_0_48_centos ingress-nginx]# cat test-deploy-demon.yaml apiVersion: v1 kind: Service metadata: name: myapp spec: selector: app: myapp release: canary ports: - name: http port: 80 targetPort: 80 --- apiVersion: apps/v1 kind: Deployment metadata: name: myapp-deploy spec: replicas: 1 selector: matchLabels: app: myapp release: canary template: metadata: labels: app: myapp release: canary spec: containers: - name: myapp image: ikubernetes/myapp:v2 ports: - name: httpd containerPort: 80
[root@VM_0_48_centos ingress-nginx]# cat test.yaml apiVersion: extensions/v1beta1 kind: Ingress metadata: name: ingress-myapp namespace: default #######注意和service 保持一致。 annotations: kubernetes.io/ingress.class: "" spec: tls: #是否启用https证书。 - hosts: - test.xiajq.com secretName: tomcat-ingress-secret ## rules: - host: test.xiajq.com #生产中该域名应当可以被公网解析 http: paths: - path: backend: serviceName: myapp servicePort: 80
4、如果不使用https，可以直接本地hosts解析后，访问。如果https还需创建证书
openssl genrsa -out tls.key 2048 openssl req -new -x509 -key tls.key -out tls.crt -subj /C=CN/ST=Beijing/L=Beijing/O=DevOps/CN=test.xiajq.com kubectl create secret tls tomcat-ingress-secret --cert=tls.crt --key=tls.key #创建secret
5、本地hosts解析后。测试访问结果：
https://test.xiajq.com/
6、ingress 解析原理。本质上还是使用了nginx作为代理服务器进行解析。只是nginx采用node方式部署。同时授权了动态获取k8s ep权限。可以动态感知后端ep变化。从而动态更新nginx代理到后端pod配置。也就nginx.conf的配置。
--查看node端口 [root@VM_0_2_centos ~]# netstat -ntlup|grep 80 tcp 0 0 172.19.0.2:2380 0.0.0.0:* LISTEN 1187/etcd tcp 0 0 0.0.0.0:80 0.0.0.0:* LISTEN 10193/nginx: master --进入pod kubectl exec -it nginx-ingress-controller-mrvzh /bin/bash -n ingress-nginx -- include /etc/nginx/mime.types; default_type text/html; gzip on; gzip_comp_level 5; gzip_http_version 1.1; gzip_min_length 256; gzip_types application/atom+xml application/javascript application/x-javascript application/json application/rss+xml application/vnd.ms-fontobject application/x-font-ttf application/x-web-app-manifest+json application/xhtml+xml application/xml font/opentype image/svg+xml image/x-icon text/css text/plain text/x-component; gzip_proxied any; gzip_vary on; # Custom headers for response server_tokens on; # disable warnings uninitialized_variable_warn off; # Additional available variables: # $namespace # $ingress_name # $service_name # $service_port log_format upstreaminfo '$the_real_ip - [$the_real_ip] - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" $request_length $request_time [$proxy_upstream_name] $upstream_addr $upstream_response_length $upstream_response_time $upstream_status $req_id'; map $request_uri $loggable { default 1; } access_log /var/log/nginx/access.log upstreaminfo if=$loggable; error_log /var/log/nginx/error.log notice; resolver 183.60.83.19 183.60.82.98 valid=30s ipv6=off; # See https://www.nginx.com/blog/websocket-nginx map $http_upgrade $connection_upgrade { default upgrade; # See http://nginx.org/en/docs/http/ngx_http_upstream_module.html#keepalive '' ''; } # The following is a sneaky way to do "set $the_real_ip $remote_addr" # Needed because using set is not allowed outside server blocks. map '' $the_real_ip { default $remote_addr; } # trust http_x_forwarded_proto headers correctly indicate ssl offloading map $http_x_forwarded_proto $pass_access_scheme { default $http_x_forwarded_proto; '' $scheme; } map $http_x_forwarded_port $pass_server_port { default $http_x_forwarded_port; '' $server_port; } # Obtain best http host map $http_host $this_host { default $http_host; '' $host; } map $http_x_forwarded_host $best_http_host { default $http_x_forwarded_host; '' $this_host; } # validate $pass_access_scheme and $scheme are http to force a redirect map "$scheme:$pass_access_scheme" $redirect_to_https { default 0; "http:http" 1; "https:http" 1; } map $pass_server_port $pass_port { 443 443; default $pass_server_port; } # Reverse proxies can detect if a client provides a X-Request-ID header, and pass it on to the backend server. # If no such header is provided, it can provide a random value. map $http_x_request_id $req_id { default $http_x_request_id; "" $request_id; } server_name_in_redirect off; port_in_redirect off; ssl_protocols TLSv1.2; # turn on session caching to drastically improve performance ssl_session_cache builtin:1000 shared:SSL:10m; ssl_session_timeout 10m; # allow configuring ssl session tickets ssl_session_tickets on; # slightly reduce the time-to-first-byte ssl_buffer_size 4k; # allow configuring custom ssl ciphers ssl_ciphers 'ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256'; ssl_prefer_server_ciphers on; ssl_ecdh_curve auto; proxy_ssl_session_reuse on; upstream upstream_balancer { server 0.0.0.1; # placeholder balancer_by_lua_block { balancer.balance() } keepalive 32; } # Global filters ## start server _ server { server_name _ ; listen 80 default_server reuseport backlog=511; set $proxy_upstream_name "-"; listen 443 default_server reuseport backlog=511 ssl http2; # PEM sha: 472974aa2c90732bd8c67a1e5ef601e2f5915003 ssl_certificate /etc/ingress-controller/ssl/default-fake-certificate.pem; ssl_certificate_key /etc/ingress-controller/ssl/default-fake-certificate.pem; location / { set $namespace ""; set $ingress_name ""; set $service_name ""; set $service_port "0"; set $location_path "/"; rewrite_by_lua_block { balancer.rewrite() } log_by_lua_block { balancer.log() monitor.call() } if ($scheme = https) { more_set_headers "Strict-Transport-Security: max-age=15724800; includeSubDomains"; } access_log off; port_in_redirect off; set $proxy_upstream_name "upstream-default-backend"; client_max_body_size 1m; proxy_set_header Host $best_http_host; # Pass the extracted client certificate to the backend # Allow websocket connections proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection $connection_upgrade; proxy_set_header X-Request-ID $req_id; proxy_set_header X-Real-IP $the_real_ip; proxy_set_header X-Forwarded-For $the_real_ip; proxy_set_header X-Forwarded-Host $best_http_host; proxy_set_header X-Forwarded-Port $pass_port; proxy_set_header X-Forwarded-Proto $pass_access_scheme; proxy_set_header X-Original-URI $request_uri; proxy_set_header X-Scheme $pass_access_scheme; # Pass the original X-Forwarded-For proxy_set_header X-Original-Forwarded-For $http_x_forwarded_for; # mitigate HTTPoxy Vulnerability # https://www.nginx.com/blog/mitigating-the-httpoxy-vulnerability-with-nginx/ proxy_set_header Proxy ""; # Custom headers to proxied server proxy_connect_timeout 5s; proxy_send_timeout 60s; proxy_read_timeout 60s; proxy_buffering off; proxy_buffer_size 4k; proxy_buffers 4 4k; proxy_request_buffering on; proxy_http_version 1.1; proxy_cookie_domain off; proxy_cookie_path off; # In case of errors try the next upstream server before returning an error proxy_next_upstream error timeout; proxy_next_upstream_tries 3; proxy_pass http://upstream_balancer; proxy_redirect off; } # health checks in cloud providers require the use of port 80 location /healthz { access_log off; return 200; } # this is required to avoid error if nginx is being monitored # with an external software (like sysdig) location /nginx_status { allow 127.0.0.1; deny all; access_log off; stub_status on; } } ## end server _ ## start server dashboard.xiajq.com server { server_name dashboard.xiajq.com ; listen 80; set $proxy_upstream_name "-"; listen 443 ssl http2; # PEM sha: 472974aa2c90732bd8c67a1e5ef601e2f5915003 ssl_certificate /etc/ingress-controller/ssl/default-fake-certificate.pem; ssl_certificate_key /etc/ingress-controller/ssl/default-fake-certificate.pem; location / { set $namespace "ingress-nginx"; set $ingress_name "ingress-myapp"; set $service_name "kubernetes-dashboard"; set $service_port "443"; set $location_path "/"; rewrite_by_lua_block { balancer.rewrite() } log_by_lua_block { balancer.log() monitor.call() } if ($scheme = https) { more_set_headers "Strict-Transport-Security: max-age=15724800; includeSubDomains"; } port_in_redirect off; set $proxy_upstream_name ""; # enforce ssl on server side if ($redirect_to_https) { return 308 https://$best_http_host$request_uri; } client_max_body_size 1m; proxy_set_header Host $best_http_host; # Pass the extracted client certificate to the backend # Allow websocket connections proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection $connection_upgrade; proxy_set_header X-Request-ID $req_id; proxy_set_header X-Real-IP $the_real_ip; proxy_set_header X-Forwarded-For $the_real_ip; proxy_set_header X-Forwarded-Host $best_http_host; proxy_set_header X-Forwarded-Port $pass_port; proxy_set_header X-Forwarded-Proto $pass_access_scheme; proxy_set_header X-Original-URI $request_uri; proxy_set_header X-Scheme $pass_access_scheme; # Pass the original X-Forwarded-For proxy_set_header X-Original-Forwarded-For $http_x_forwarded_for; # mitigate HTTPoxy Vulnerability # https://www.nginx.com/blog/mitigating-the-httpoxy-vulnerability-with-nginx/ proxy_set_header Proxy ""; # Custom headers to proxied server proxy_connect_timeout 5s; proxy_send_timeout 60s; proxy_read_timeout 60s; proxy_buffering off; proxy_buffer_size 4k; proxy_buffers 4 4k; proxy_request_buffering on; proxy_http_version 1.1; proxy_cookie_domain off; proxy_cookie_path off; # In case of errors try the next upstream server before returning an error proxy_next_upstream error timeout; proxy_next_upstream_tries 3; # No endpoints available for the request return 503; } } ## end server dashboard.xiajq.com ## start server test.xiajq.com ############################## 在创建一个ingress后，会自动在ng.conf中默认配置 server { server_name test.xiajq.com ; listen 80; set $proxy_upstream_name "-"; listen 443 ssl http2; ##启用443端口 # PEM sha: 6b96e02eb5f1eaa3d45486073017e7fd8ce3a40e ssl_certificate /etc/ingress-controller/ssl/default-tomcat-ingress-secret.pem; #证书位置。这个为我们创建的证书 ssl_certificate_key /etc/ingress-controller/ssl/default-tomcat-ingress-secret.pem; ssl_trusted_certificate /etc/ingress-controller/ssl/default-tomcat-ingress-secret-full-chain.pem; ## ssl_stapling on; ssl_stapling_verify on; location / { set $namespace "default"; ####设置变量 set $ingress_name "ingress-myapp"; ### set $service_name "myapp"; ###服务名字 set $service_port "80"; ##端口 set $location_path "/"; ###路径 rewrite_by_lua_block { balancer.rewrite() } log_by_lua_block { balancer.log() monitor.call() } if ($scheme = https) { more_set_headers "Strict-Transport-Security: max-age=15724800; includeSubDomains"; } port_in_redirect off; set $proxy_upstream_name "default-myapp-80"; # enforce ssl on server side if ($redirect_to_https) { return 308 https://$best_http_host$request_uri; } client_max_body_size 1m; proxy_set_header Host $best_http_host; # Pass the extracted client certificate to the backend # Allow websocket connections proxy_set_header Upgrade $http_upgrade; proxy_set_header Connection $connection_upgrade; proxy_set_header X-Request-ID $req_id; proxy_set_header X-Real-IP $the_real_ip; proxy_set_header X-Forwarded-For $the_real_ip; proxy_set_header X-Forwarded-Host $best_http_host; proxy_set_header X-Forwarded-Port $pass_port; proxy_set_header X-Forwarded-Proto $pass_access_scheme; proxy_set_header X-Original-URI $request_uri; proxy_set_header X-Scheme $pass_access_scheme; # Pass the original X-Forwarded-For proxy_set_header X-Original-Forwarded-For $http_x_forwarded_for; # mitigate HTTPoxy Vulnerability # https://www.nginx.com/blog/mitigating-the-httpoxy-vulnerability-with-nginx/ proxy_set_header Proxy ""; # Custom headers to proxied server proxy_connect_timeout 5s; proxy_send_timeout 60s; proxy_read_timeout 60s; proxy_buffering off; proxy_buffer_size 4k; proxy_buffers 4 4k; proxy_request_buffering on; proxy_http_version 1.1; proxy_cookie_domain off; proxy_cookie_path off; # In case of errors try the next upstream server before returning an error proxy_next_upstream error timeout; proxy_next_upstream_tries 3; proxy_pass http://upstream_balancer; proxy_redirect off; } } ## end server test.xiajq.com # backend for when default-backend-service is not configured or it does not have endpoints server { listen 8181 default_server reuseport backlog=511; set $proxy_upstream_name "-"; location / { return 404; } } # default server, used for NGINX healthcheck and access to nginx stats server { listen 18080 default_server reuseport backlog=511; set $proxy_upstream_name "-"; location /healthz { access_log off; return 200; } location /is-dynamic-lb-initialized { access_log off; content_by_lua_block { local configuration = require("configuration") local backend_data = configuration.get_backends_data() if not backend_data then ngx.exit(ngx.HTTP_INTERNAL_SERVER_ERROR) return end ngx.say("OK") ngx.exit(ngx.HTTP_OK) } } location /nginx_status { set $proxy_upstream_name "internal"; access_log off; stub_status on; } location /configuration { access_log off; allow 127.0.0.1; deny all; # this should be equals to configuration_data dict client_max_body_size 10m; proxy_buffering off; content_by_lua_block { configuration.call() } } location / { set $proxy_upstream_name "upstream-default-backend"; proxy_set_header Host $best_http_host; proxy_pass http://upstream_balancer; } } } stream { log_format log_stream [$time_local] $protocol $status $bytes_sent $bytes_received $session_time; access_log /var/log/nginx/access.log log_stream; error_log /var/log/nginx/error.log; # TCP services # UDP services }
7、ingress 使用案例，添加白名单
igress控制ip访问 apiVersion: extensions/v1beta1 kind: Ingress metadata: name: whitelist annotations: ingress.kubernetes.io/whitelist-source-range: "1.1.1.1/24" ingress.kubernetes.io/whitelist-source-range: 192.168**/,192.168**/32 spec: rules: - host: whitelist.test.net http: paths: - path: / backend: serviceName: webserver servicePort: 80
yaml下载地址: https://github.com/kubernetes/ingress-nginx/tree/nginx-0.20.0/deploy
良禽择木而栖 贤臣择主而侍
