k8s Ingress-nginx 部署使用

 

k8s版本信息：v1.18.0

官方文档中，部署只要简单的执行一个yaml

https://github.com/kubernetes/ingress-nginx/blob/nginx-0.20.0/deploy/mandatory.yaml

mandatory.yaml这一个yaml中包含了部署ingress需要的资源的创建，包括namespace、ConfigMap、role，ServiceAccount等等所有部署ingress-controller需要的资源，修改真正修改的就是deployment这部分部分：

我们需要使用daemonset 部署到特定node，需要修改部分配置：先给要部署nginx-ingress 的 node打上特定标签,这里测试部署在"node-1"和"node-2"这两个节点。

# kubectl label node k8s-node01 isIngress="true"
# kubectl label node k8s-node02 isIngress="true"

这个是执行时已经改好的配置文件，主要是删除了deploment 和service的部分，添加了daemonset部分.

# cat mandatory.yaml 
apiVersion: v1
kind: Namespace
metadata:
  name: ingress-nginx

---

kind: ConfigMap
apiVersion: v1
metadata:
  name: nginx-configuration
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx

---

kind: ConfigMap
apiVersion: v1
metadata:
  name: tcp-services
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx

---

kind: ConfigMap
apiVersion: v1
metadata:
  name: udp-services
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx

---

apiVersion: v1
kind: ServiceAccount
metadata:
  name: nginx-ingress-serviceaccount
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx

---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
  name: nginx-ingress-clusterrole
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
rules:
  - apiGroups:
      - ""
    resources:
      - configmaps
      - endpoints
      - nodes
      - pods
      - secrets
    verbs:
      - list
      - watch
  - apiGroups:
      - ""
    resources:
      - nodes
    verbs:
      - get
  - apiGroups:
      - ""
    resources:
      - services
    verbs:
      - get
      - list
      - watch
  - apiGroups:
      - "extensions"
    resources:
      - ingresses
    verbs:
      - get
      - list
      - watch
  - apiGroups:
      - ""
    resources:
      - events
    verbs:
      - create
      - patch
  - apiGroups:
      - "extensions"
    resources:
      - ingresses/status
    verbs:
      - update

---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: Role
metadata:
  name: nginx-ingress-role
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
rules:
  - apiGroups:
      - ""
    resources:
      - configmaps
      - pods
      - secrets
      - namespaces
    verbs:
      - get
  - apiGroups:
      - ""
    resources:
      - configmaps
    resourceNames:
      # Defaults to "<election-id>-<ingress-class>"
      # Here: "<ingress-controller-leader>-<nginx>"
      # This has to be adapted if you change either parameter
      # when launching the nginx-ingress-controller.
      - "ingress-controller-leader-nginx"
    verbs:
      - get
      - update
  - apiGroups:
      - ""
    resources:
      - configmaps
    verbs:
      - create
  - apiGroups:
      - ""
    resources:
      - endpoints
    verbs:
      - get

---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: RoleBinding
metadata:
  name: nginx-ingress-role-nisa-binding
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: Role
  name: nginx-ingress-role
subjects:
  - kind: ServiceAccount
    name: nginx-ingress-serviceaccount
    namespace: ingress-nginx

---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
  name: nginx-ingress-clusterrole-nisa-binding
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: nginx-ingress-clusterrole
subjects:
  - kind: ServiceAccount
    name: nginx-ingress-serviceaccount
    namespace: ingress-nginx

---
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: nginx-ingress-controller
  namespace: ingress-nginx
  labels:
    app.kubernetes.io/name: ingress-nginx
    app.kubernetes.io/part-of: ingress-nginx
spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: ingress-nginx
      app.kubernetes.io/part-of: ingress-nginx
  template:
    metadata:
      labels:
        app.kubernetes.io/name: ingress-nginx
        app.kubernetes.io/part-of: ingress-nginx
      annotations:
        prometheus.io/port: "10254"
        prometheus.io/scrape: "true"
    spec:
      serviceAccountName: nginx-ingress-serviceaccount
      nodeSelector:
        isIngress: "true"
      hostNetwork: true
      containers:
        - name: nginx-ingress-controller
          image: registry.cn-qingdao.aliyuncs.com/kubernetes_xingej/nginx-ingress-controller:0.20.0
          args:
            - /nginx-ingress-controller
            - --configmap=$(POD_NAMESPACE)/nginx-configuration
            - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services
            - --udp-services-configmap=$(POD_NAMESPACE)/udp-services
            - --publish-service=$(POD_NAMESPACE)/ingress-nginx
            - --annotations-prefix=nginx.ingress.kubernetes.io
          securityContext:
            allowPrivilegeEscalation: true
            capabilities:
              drop:
                - ALL
              add:
                - NET_BIND_SERVICE
            # www-data -> 33
            runAsUser: 33
          env:
            - name: POD_NAME
              valueFrom:
                fieldRef:
                  fieldPath: metadata.name
            - name: POD_NAMESPACE
              valueFrom:
                fieldRef:
                  fieldPath: metadata.namespace
          ports:
            - name: http
              containerPort: 80
            - name: https
              containerPort: 443
          livenessProbe:
            failureThreshold: 3
            httpGet:
              path: /healthz
              port: 10254
              scheme: HTTP
            initialDelaySeconds: 10
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 10
          readinessProbe:
            failureThreshold: 3
            httpGet:
              path: /healthz
              port: 10254
              scheme: HTTP
            periodSeconds: 10
            successThreshold: 1
            timeoutSeconds: 10
---
[root@k8s-master ~]#
mandatory.yaml

修改完后执行apply,并检查服务

# kubectl apply -f mandatory.yaml
[root@k8s-master ~]# kubectl get daemonset -n ingress-nginx
NAME                       DESIRED   CURRENT   READY   UP-TO-DATE   AVAILABLE   NODE SELECTOR    AGE
nginx-ingress-controller   2         2         2       2            2           isIngress=true   21h
[root@k8s-master ~]# kubectl get po -n ingress-nginx -o wide
NAME                             READY   STATUS    RESTARTS   AGE   IP              NODE         NOMINATED NODE   READINESS GATES
nginx-ingress-controller-qnj2t   1/1     Running   0          21h   192.168.10.30   k8s-node01   <none>           <none>
nginx-ingress-controller-z45z7   1/1     Running   10         21h   192.168.10.40   k8s-node02   <none>           <none>
[root@k8s-master ~]#

可以看到，nginx-controller的pod已经部署在在node01和node02上了。

到node02上看下本地端口：

# netstat -lntup | grep nginx
tcp        0      0 0.0.0.0:80              0.0.0.0:*               LISTEN      25040/nginx: master 
tcp        0      0 0.0.0.0:8181            0.0.0.0:*               LISTEN      25040/nginx: master 
tcp        0      0 0.0.0.0:443             0.0.0.0:*               LISTEN      25040/nginx: master 
tcp        0      0 0.0.0.0:18080           0.0.0.0:*               LISTEN      25040/nginx: master 
tcp6       0      0 :::10254                :::*                    LISTEN      24997/nginx-ingress 
tcp6       0      0 :::80                   :::*                    LISTEN      25040/nginx: master 
tcp6       0      0 :::8181                 :::*                    LISTEN      25040/nginx: master 
tcp6       0      0 :::443                  :::*                    LISTEN      25040/nginx: master 
tcp6       0      0 :::18080                :::*                    LISTEN      25040/nginx: master 
[root@k8s-node01 ~]#

由于配置了hostNetwork，nginx已经在node主机本地监听80/443/8181端口。其中8181是nginx-controller默认配置的一个default backend。这样，只要访问node主机的公网IP，就可以直接映射域名来对外网暴露服务了。

配置ingress资源
部署完ingress-controller，接下来就按照测试的需求来创建ingress资源，创建ingress资源之前，需要创建好pod及service服务.

1.创建nginx pod及service服务，service名称为 ngx-service

# cat deploy.yaml 
apiVersion: apps/v1
kind: Deployment
metadata:
  name: nginx-deploy
  namespace: default
spec:
  replicas: 3
  selector:
    matchLabels:
      app: nginx
  template:
    metadata:
      labels:
        app: nginx
    spec:
      containers:
      - name: nginx
        #image: nginx:latest
        image: ikubernetes/myapp:v2
        ports:
        - containerPort: 80
---
apiVersion: v1   
kind: Service
metadata:
  name: ngx-service
  labels:
    app: nginx
spec:
  type: NodePort
  selector:
    app: nginx
  ports:
  - port: 80
    targetPort: 80
    nodePort: 32500
[root@k8s-master ~]#

2.创建ingress资源。

# cat ingresstest.yaml 
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  name: ingress-test
  annotations:
    kubernetes.io/ingress.class: "nginx"
    nginx.ingress.kubernetes.io/use-regex: "true"
spec:
  rules:
    # 定义域名
    - host: test.heheda.io
      http:
        paths:
        # 不同path转发到不同端口
          - path: /
            backend:
              serviceName: ngx-service
              servicePort: 80
          - path: /xxx
            backend:
              serviceName: ngx-service
              servicePort: 80
[root@k8s-master ~]#

部署资源

# kubectl apply -f ingresstest.yaml

测试访问

部署好以后，做一条本地host来模拟解析 test.heheda.io 到node的ip地址。测试访问

增加后端服务tomcat

1.创建tomcat pod及service服务

# cat tomcat-deploy.yaml 
apiVersion: v1
kind: Service
metadata:
  name: tomcat-svc
  namespace: default
spec:
  selector:
    app: tomcat
  ports:
  - name: http
    port: 8080
    targetPort: 8080
  - name: ajp
    port: 8009
    targetPort: 8009
 
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: tomcat-deploy
spec:
  replicas: 3
  selector:
    matchLabels:
      app: tomcat
  template:
    metadata:
      labels:
        app: tomcat
    spec:
      containers:
      - name: tomcat
        image: tomcat:7-alpine
        ports:
        - name: httpd
          containerPort: 8080
        - name: ajp
          containerPort: 8009
[root@k8s-master ~]#

2.创建tomcat ingress资源。

# cat ingress-tomcat.yaml 
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  name: ingress-tomcat
  namespace: default
  annotations:
    kubernets.io/ingress.class: "nginx"
spec:
  rules:
  - host: test.tomcat.io
    http:
      paths:
      - path:
        backend:
          serviceName: tomcat-svc
          servicePort: 8080
[root@k8s-master ~]#

3.主机添加host绑定解析即可访问测试.

 

 

下面我们对tomcat服务添加httpds服务
创建私有证书及secret

# openssl req -x509 -sha256 -nodes -days 365 -newkey rsa:2048 -keyout tls.key -out tls.crt -subj "/CN=nginxsvc/O=test.tomcat.io"
# kubectl create secret tls tls-secret --key=tls.key --cert tls.crt
# kubectl get secret
NAME                  TYPE                                  DATA   AGE
default-token-t55js   kubernetes.io/service-account-token   3      5d22h
tls-secret            kubernetes.io/tls                     2      68m
[root@k8s-master ~]# 
[root@k8s-master ~]# kubectl describe secret tls-secret
Name:         tls-secret
Namespace:    default
Labels:       <none>
Annotations:  <none>

Type:  kubernetes.io/tls

Data
====
tls.key:  1704 bytes
tls.crt:  1143 bytes
[root@k8s-master ~]#

将证书应用至tomcat服务中

# cat ingress-tomcat.yaml 
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  name: ingress-tomcat
  namespace: default
  annotations:
    kubernets.io/ingress.class: "nginx"
spec:
  tls:
    - hosts:
      - test.tomcat.io
      secretName: tls-secret  
  rules:
  - host: test.tomcat.io
    http:
      paths:
      - path:
        backend:
          serviceName: tomcat-svc
          servicePort: 8080
[root@k8s-master ~]#

访问服务：

 创建pod-svc-ingress服务：

apiVersion: apps/v1
kind: Deployment
metadata:
  name: my-nginx
spec:
  replicas: 3
  selector:
    matchLabels:
      app: nginx
  template:
    metadata:
      labels:
        app: nginx
    spec:
      containers:
      - name: nginx
        #image: nginx:latest
        image: ikubernetes/myapp:v2
        ports:
        - containerPort: 80
---
apiVersion: v1
kind: Service
metadata:
  name: ngx-service
  labels:
    app: nginx
spec:
  type: NodePort
  selector:
    app: nginx
  ports:
  - port: 80
    targetPort: 80
    nodePort: 32500

---

apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  name: ingress-test
  annotations:
    kubernetes.io/ingress.class: "nginx"
    nginx.ingress.kubernetes.io/use-regex: "true"
spec:
  rules:
    # 定义域名
    - host: test.saneri.io
      http:
        paths:
        # 不同path转发到不同端口
          - path: /
            backend:
              serviceName: ngx-service
              servicePort: 80
          - path: /xxx
            backend:
              serviceName: ngx-service
              servicePort: 80
deploy-nginx.yaml

 

参考文档：　　　

　　　　https://www.cnblogs.com/panwenbin-logs/p/9915927.html

　　　　https://github.com/kubernetes/ingress-nginx/blob/nginx-0.20.0/deploy/mandatory.yaml

　　　　https://blog.csdn.net/zhangjunli/article/details/107545984

　　　　https://blog.csdn.net/yucaifu1989/article/details/106898901/

 

posted @ 2021-03-04 14:21  梦徒  阅读(2739)  评论(0)  编辑  收藏  举报