解决k8s故障，eureka处于unknow的问题

 

//pod Unknown状态，开始强制删除eureka-server-node-0,eureka-server-node-1等,强制删除不了

[root@fp158 sts]# kubectl delete pods eureka-server-node-0 --grace-period=0 --force

//那就用kubectl scale扩展吧，发现也不行，后用 kubectl describe pod podname 查看发现标签有问题, 这是pod状态是pending

 //同时采用在线编辑yaml，保存,本来强制删除，但是用--grace-period=0 --force 删除都删不了，也是第一回遇到，那就先用edit命令吧，
  注意 不能只是kubectl edit pod podname形式。

 [root@f158 sts]# kubectl edit sts eureka-server-node -o yaml --save-config  //eureka-server-node是pod名

//覆盖某个node上的role标签
kubectl label nodes k2.node role=htt --overwrite

//查看显示某个node的标签

[root@f158 sts]# kubectl get nodes -o wide --show-labels | grep k2  //k2调度到的节点

//查看eureka-server-node

[root@f158 sts]# kubectl get pod -o wide | grep eureka-server-node

//进入pod查看

[root@f158 sts]# kubectl exec -ti ut-node-0 -- /bin/sh
# ping www.baidu.com   //无反应，怀疑网络有问题，首先必然是calico, 然后上racher 管理平台上查看node的一些上报状态情况.

//同时在宿主机上进行了一系列操作

[root@fp-web-187 ~]# docker ps| grep calico-node   //看看网络情况

[root@fp-web-187 ~]# docker ps -a | grep calico-node  //查看未启动的，可恢复

[root@fp-web-187 ~]# systemctl restart calico-node  //重启calico网络

[root@fp-web-187 ~]# systemctl restart kubectl   //重启 k8s 节点

[root@fp-web-187 ~]# systemctl restart kube-proxy  //重启代理

[root@fp-web-187 ~]# systemctl restart docker  //重启pod

[root@fp-web-187 ~]# ip a  //查ip,网卡情况

[root@fp-web-187 ~]# route -n   //查看 tunl0（ip ip ）情况

[root@fp-web-187 ~]# docker exec -it 973ad176419c /bin/bash  //进入容器里面看一下，这时不能叫pod，因为pod可以有多个容器，我们进入的是当前的eureka容器

[root@fp-web-187 ~]# docker ps | grep eu  //查看eureka,服务在线情况.

同时为方便查找出错误，配置在web浏览器查看

/usr/local/nginx/conf/extconf/eu.conf

server {
listen 80;
listen 443 ssl;
listen [::]:80;
listen [::]:443 ssl;
server_name a.b.com.cn;
#ssl on;
ssl_certificate ssl/server.pem;
ssl_certificate_key ssl/server.key;
ssl_session_cache shared:SSL:1m;
ssl_session_timeout 5m;
ssl_protocols TLSv1 TLSv1.1 TLSv1.2;
ssl_ciphers HIGH:!aNULL:!MD5;
ssl_prefer_server_ciphers on;
large_client_header_buffers 4 1m;
fastcgi_intercept_errors on;

location / {

proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header REMOTE-HOST $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_pass http://eu;
proxy_connect_timeout 600s;
proxy_read_timeout 900s;
proxy_redirect off;
client_max_body_size 1000m;
}
}

主nginx.conf

user  weblogic;
worker_processes  auto;
worker_cpu_affinity auto;
worker_rlimit_nofile 65535;
#error_log  logs/error.log;
#error_log  logs/error.log  notice;
error_log  logs/error.log  error;
pid        logs/nginx.pid;
events {
    worker_connections  65535;
    accept_mutex off;
        use epoll;
        multi_accept on;
}
http {
  #autoindex off;
        server_tokens  off;
    include       mime.types;
    include /usr/local/nginx/conf/extconf/*.conf;
    default_type  application/octet-stream;
    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" "$request_body" '
                       '$status $body_bytes_sent "$http_referer" '
                       '"$http_user_agent" "$http_x_forwarded_for" '
                       '$upstream_addr'
                        ' $upstream_response_time';
    access_log  /var/log/nginx/access.log  main;
    fastcgi_intercept_errors on;
    sendfile        on;
    keepalive_timeout  65;
    gzip  on;
    gzip_min_length 1k;
    gzip_buffers 4 16k;
    gzip_http_version 1.0;
    gzip_comp_level 2;
    gzip_types text/plain application/x-javascript text/css application/xml;
    gzip_vary on;

    upstream bc {
                server 10.121.51.11:38761        max_fails=3 fail_timeout=120s;  //master ip
                server 10.121.51.12:38761       max_fails=3 fail_timeout=120s;   //master2 ip
                keepalive 32;
                keepalive_timeout 120s;
                }
}

[root@fp-Nignx01 ~]# /usr/local/nginx/sbin/nginx -t
[root@fp-Nignx01 ~]# /usr/local/nginx/sbin/nginx -s reload
 访问
 http://a.b.com.cn/lastn //即可访问

配置eureka在web页面上访问，可以通过 
1、查看暴露的端口

[root@fp-web-158 sts]# kubectl get svc -o wide  | grep eu
eureka-server         ClusterIP   None             <none>        8761/TCP                      3y        name=eureka-server-node
eureka-server-np      NodePort    172.17.65.89     <none>        8761:38761/TCP                3y        name=eureka-server-node

2、在域名里面进行相关的配置 

               upstream bc {
                server 10.121.51.11:38761        max_fails=3 fail_timeout=120s;  //master ip
                keepalive 32;
                keepalive_timeout 120s;
                }

3、访问 http://a.b.com.cn/  （这个域名根据公司申请的域名设置二级域名即可）

 

 

//pod正常应该可以ping通eu
[root@f158 sts]# kubectl exec -ti ts-node-0 -- /bin/sh
# ping eureka-server-node-0.eureka-server
PING eureka-server-node-0.eureka-server.default.svc.cluster.local (172.17.94.145): 56 data bytes
64 bytes from 172.17.94.145: icmp_seq=0 ttl=62 time=0.273 ms
64 bytes from 172.17.94.145: icmp_seq=1 ttl=62 time=0.266 ms
64 bytes from 172.17.94.145: icmp_seq=2 ttl=62 time=0.310 ms
64 bytes from 172.17.94.145: icmp_seq=3 ttl=62 time=0.256 ms
64 bytes from 172.17.94.145: icmp_seq=4 ttl=62 time=0.308 ms
64 bytes from 172.17.94.145: icmp_seq=5 ttl=62 time=0.233 ms
64 bytes from 172.17.94.145: icmp_seq=6 ttl=62 time=0.507 ms
^C--- eureka-server-node-0.eureka-server.default.svc.cluster.local ping statistics ---
7 packets transmitted, 7 packets received, 0% packet loss
round-trip min/avg/max/stddev = 0.233/0.308/0.507/0.085 ms