centos7安装harbor、配置私有镜像仓库
准备工作
#关闭防火墙 systemctl stop firewalld systemctl disable firewalld #关闭SELinux和取消swap sed -i 's/enforcing/disabled/' /etc/selinux/config sed -ri 's/.*swap.*/#&/' /etc/fstab #主机名，根据你自己的情况来设置 echo -e "192.168.50.10 centos-k8s-master0\n192.168.50.11 centos-k8s-master1\n192.168.50.12 centos-k8s-master2\n192.168.50.16 centos-k8s-node0\n192.168.50.17 centos-k8s-node1\n192.168.50.18 centos-k8s-node2\n" >> /etc/hosts #内核参数 echo -e "net.bridge.bridge-nf-call-ip6tables = 1\nnet.bridge.bridge-nf-call-iptables = 1" >/etc/sysctl.d/k8s.conf sysctl --system yum install ntpdate wget -y ntpdate time.windows.com #修改主机名为harbor hostnamectl set-hostname harbor #docker源 wget https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo -O /etc/yum.repos.d/docker-ce.repo yum install docker-ce -y echo '{"registry-mirrors": ["https://registry.docker-cn.com","https://gg3gwnry.mirror.aliyuncs.com"]}'>/etc/docker/daemon.json systemctl enable docker.service && systemctl start docker.service #设置科学代理，请参考《让局域网中的其他主机使用clash的代理》 cat /etc/environment https_proxy=http://192.168.24.1:7890 http_proxy=http://192.168.24.1:7890 source /etc/environment #此时应该要重连当前ssh连接 #安装docker-compose #官网：https://link.zhihu.com/?target=https%3A//github.com/docker/compose/releases wget https://github.com/docker/compose/releases/download/v2.24.0/docker-compose-linux-x86_64 mv docker-compose-Linux-x86_64 /usr/bin/docker-compose chmod a+x /usr/bin/docker-compose #检查信息以及确定命令可用 docker-compose --version #离线安装harbor #官网：https://github.com/goharbor/harbor/releases #比较大，有600多兆 wget https://github.com/goharbor/harbor/releases/download/v2.10.0/harbor-offline-installer-v2.10.0.tgz [root@localhost ~]# mkdir harbor [root@localhost ~]# tar -xvf harbor-offline-installer-v2.10.0.tgz -C harbor harbor/harbor.v2.10.0.tar.gz harbor/prepare harbor/LICENSE harbor/install.sh harbor/common.sh harbor/harbor.yml.tmpl [root@localhost ~]# #创建https证书 mkdir -p /harbor/https/ca/test cd /harbor/https/ca/test openssl genrsa -out ca.key 4096 openssl req -x509 -new -nodes -sha512 -days 3650 -subj "/C=CN/ST=Beijing/L=Beijing/O=example/OU=Personal/CN=192.168.24.5" -key ca.key -out ca.crt openssl genrsa -out myharbor.com.key 4096 openssl req -sha512 -new -subj "/C=CN/ST=Beijing/L=Beijing/O=example/OU=Personal/CN=192.168.24.5" -key myharbor.com.key -out myharbor.com.csr #这一步比较重要，不然会导致后面docker登录的时候报错：x509: cannot validate certificate for 192.168.24.5 because it doesn't contain any IP SANs echo subjectAltName = IP:192.168.24.5 > v3.ext openssl x509 -req -sha512 -days 3650 -extfile v3.ext -CA ca.crt -CAkey ca.key -CAcreateserial -in myharbor.com.csr -out myharbor.com.crt openssl x509 -inform PEM -in myharbor.com.crt -out myharbor.com.cert #这一步也必须要做，不然上传镜像会报错；Get "https://192.168.24.5/v2/": tls: failed to verify certificate: x509: certificate signed by unknown authority mkdir -p /etc/docker/certs.d/192.168.24.5/ cp myharbor.com.cert /etc/docker/certs.d/192.168.24.5/ cp myharbor.com.key /etc/docker/certs.d/192.168.24.5/ cp ca.crt /etc/docker/certs.d/192.168.24.5/ systemctl restart docker #初始配置配置文件： [root@localhost harbor]# pwd /root/harbor/harbor #备份下原始的配置文件 [root@localhost harbor]# cp harbor.yml.tmpl harbor.yml.tmpl.bak20240120 #配置之后的配置文件，实际上我只配置了hostname，certificate，private_key，harbor_admin_password；其他的很多都是保持默认的 [root@localhost harbor]# cat harbor.yml.tmpl |grep -v '^\s*#'|grep -v "^#"|grep -v '^$' hostname: 192.168.24.5 #harbor主机的地址或者dns域名 http: port: 80 https: port: 443 certificate: /harbor/https/ca/test/myharbor.com.crt #证书 private_key: /harbor/https/ca/test/myharbor.com.key #私钥 harbor_admin_password: Lovedan@971220 #web页面的管理员密码 database: password: root123 #数据库的root密码 max_idle_conns: 100 max_open_conns: 900 conn_max_lifetime: 5m conn_max_idle_time: 0 data_volume: /data #数据存储的目录 trivy: ignore_unfixed: false skip_update: false offline_scan: false security_check: vuln insecure: false jobservice: max_job_workers: 10 job_loggers: - STD_OUTPUT - FILE logger_sweeper_duration: 1 #days notification: webhook_job_max_retry: 3 webhook_job_http_client_timeout: 3 #seconds log: level: info local: rotate_count: 50 rotate_size: 200M location: /var/log/harbor _version: 2.10.0 proxy: http_proxy: https_proxy: no_proxy: components: - core - jobservice - trivy upload_purging: enabled: true age: 168h interval: 24h dryrun: false cache: enabled: false expire_hours: 24 [root@localhost harbor]#
安装
#安装harbor [root@localhost harbor]# mv harbor.yml.tmpl harbor.yml [root@localhost harbor]# ./install.sh [Step 0]: checking if docker is installed ... Note: docker version: 25.0.0 [Step 1]: checking docker-compose is installed ... Note: Docker Compose version v2.24.1 [Step 2]: loading Harbor images ... ad00c0c1d948: Loading layer [==================================================>] 40.11MB/40.11MB 7570a78aea36: Loading layer [==================================================>] 9.08MB/9.08MB 04774e0e84f2: Loading layer [==================================================>] 4.096kB/4.096kB 8a42710a9f5a: Loading layer [==================================================>] 3.072kB/3.072kB f1990e77f8e5: Loading layer [==================================================>] 197MB/197MB 172f5af926be: Loading layer [==================================================>] 17.6MB/17.6MB 2bef5dd17a10: Loading layer [==================================================>] 215.4MB/215.4MB Loaded image: goharbor/trivy-adapter-photon:v2.10.0 8c10ac3a40a9: Loading layer [==================================================>] 89.84MB/89.84MB cc741ba6af65: Loading layer [==================================================>] 65.11MB/65.11MB 7877ea7046dd: Loading layer [==================================================>] 13.2MB/13.2MB 3de91a0984fe: Loading layer [==================================================>] 65.54kB/65.54kB 4d2cda613456: Loading layer [==================================================>] 2.56kB/2.56kB 625b81c9e514: Loading layer [==================================================>] 1.536kB/1.536kB 6496ec4ce84e: Loading layer [==================================================>] 12.29kB/12.29kB 2b332fb0075b: Loading layer [==================================================>] 5.322MB/5.322MB f8976ae46d77: Loading layer [==================================================>] 457.7kB/457.7kB Loaded image: goharbor/prepare:v2.10.0 a36cb8a4e510: Loading layer [==================================================>] 126.1MB/126.1MB fd72ef63aacc: Loading layer [==================================================>] 3.584kB/3.584kB 162d9960a2b9: Loading layer [==================================================>] 3.072kB/3.072kB b71508b0d586: Loading layer [==================================================>] 2.56kB/2.56kB 18c685e189cb: Loading layer [==================================================>] 3.072kB/3.072kB 84690af2e82b: Loading layer [==================================================>] 3.584kB/3.584kB 8d7eb73e8207: Loading layer [==================================================>] 20.48kB/20.48kB Loaded image: goharbor/harbor-log:v2.10.0 001da4979db8: Loading layer [==================================================>] 8.562MB/8.562MB 275281f671bf: Loading layer [==================================================>] 4.096kB/4.096kB f6856e2d539f: Loading layer [==================================================>] 17.4MB/17.4MB 557317f3c1c5: Loading layer [==================================================>] 3.072kB/3.072kB 942b8c3060c6: Loading layer [==================================================>] 32.81MB/32.81MB b2bc30e737e7: Loading layer [==================================================>] 51MB/51MB Loaded image: goharbor/harbor-registryctl:v2.10.0 d5e5478da184: Loading layer [==================================================>] 116.8MB/116.8MB Loaded image: goharbor/nginx-photon:v2.10.0 74a7f6a8de8c: Loading layer [==================================================>] 11.58MB/11.58MB 46523ccaf371: Loading layer [==================================================>] 27.6MB/27.6MB 3cea6b428022: Loading layer [==================================================>] 4.608kB/4.608kB 9815d4ae0f06: Loading layer [==================================================>] 28.39MB/28.39MB Loaded image: goharbor/harbor-exporter:v2.10.0 7d7e15ae8ca1: Loading layer [==================================================>] 15.93MB/15.93MB e13b0ff80947: Loading layer [==================================================>] 111.8MB/111.8MB da74a4230588: Loading layer [==================================================>] 3.072kB/3.072kB 89240f6f343b: Loading layer [==================================================>] 59.9kB/59.9kB 7b00214da46a: Loading layer [==================================================>] 61.95kB/61.95kB Loaded image: goharbor/redis-photon:v2.10.0 345fdbd05997: Loading layer [==================================================>] 8.562MB/8.562MB 5c4443929555: Loading layer [==================================================>] 4.096kB/4.096kB 605c9788ef17: Loading layer [==================================================>] 3.072kB/3.072kB b2ab56de0e45: Loading layer [==================================================>] 17.4MB/17.4MB 950eb2734789: Loading layer [==================================================>] 18.19MB/18.19MB Loaded image: goharbor/registry-photon:v2.10.0 7693164e30e0: Loading layer [==================================================>] 116.8MB/116.8MB 6b50b5d516aa: Loading layer [==================================================>] 6.531MB/6.531MB 5aea217650ad: Loading layer [==================================================>] 246.8kB/246.8kB 0b3de92ff70b: Loading layer [==================================================>] 1.477MB/1.477MB Loaded image: goharbor/harbor-portal:v2.10.0 b79ac58f353c: Loading layer [==================================================>] 11.58MB/11.58MB 18c4c015e339: Loading layer [==================================================>] 3.584kB/3.584kB c6eef6a39935: Loading layer [==================================================>] 2.56kB/2.56kB 03db56130352: Loading layer [==================================================>] 58.57MB/58.57MB 78c9748f2d29: Loading layer [==================================================>] 5.632kB/5.632kB e1732f90232a: Loading layer [==================================================>] 123.4kB/123.4kB 6b733e4833c8: Loading layer [==================================================>] 80.38kB/80.38kB fe6828cc147a: Loading layer [==================================================>] 59.56MB/59.56MB 8e1349c44768: Loading layer [==================================================>] 2.56kB/2.56kB Loaded image: goharbor/harbor-core:v2.10.0 02b0385778eb: Loading layer [==================================================>] 15.93MB/15.93MB ac2be26232ee: Loading layer [==================================================>] 175MB/175MB 1e3e9dba5eaa: Loading layer [==================================================>] 25.47MB/25.47MB c7fa85d79c1c: Loading layer [==================================================>] 18.14MB/18.14MB f9ef6596c7ce: Loading layer [==================================================>] 5.12kB/5.12kB e35036cc139f: Loading layer [==================================================>] 6.144kB/6.144kB 0cb7ded8041a: Loading layer [==================================================>] 3.072kB/3.072kB 5d178abacde0: Loading layer [==================================================>] 2.048kB/2.048kB 67fe09fd709b: Loading layer [==================================================>] 2.56kB/2.56kB d0b4aac335c4: Loading layer [==================================================>] 7.68kB/7.68kB Loaded image: goharbor/harbor-db:v2.10.0 fced8697b2c7: Loading layer [==================================================>] 11.58MB/11.58MB af968b902c26: Loading layer [==================================================>] 3.584kB/3.584kB 7caaebde63bc: Loading layer [==================================================>] 2.56kB/2.56kB d6e3c7a6ef36: Loading layer [==================================================>] 44.96MB/44.96MB bc8a8ef654ea: Loading layer [==================================================>] 45.75MB/45.75MB Loaded image: goharbor/harbor-jobservice:v2.10.0 [Step 3]: preparing environment ... [Step 4]: preparing harbor configs ... prepare base dir is set to /root/harbor/harbor Generated configuration file: /config/portal/nginx.conf Generated configuration file: /config/log/logrotate.conf Generated configuration file: /config/log/rsyslog_docker.conf Generated configuration file: /config/nginx/nginx.conf Generated configuration file: /config/core/env Generated configuration file: /config/core/app.conf Generated configuration file: /config/registry/config.yml Generated configuration file: /config/registryctl/env Generated configuration file: /config/registryctl/config.yml Generated configuration file: /config/db/env Generated configuration file: /config/jobservice/env Generated configuration file: /config/jobservice/config.yml Generated and saved secret to file: /data/secret/keys/secretkey Successfully called func: create_root_cert Generated configuration file: /compose_location/docker-compose.yml Clean up the input dir Note: stopping existing Harbor instance ... [Step 5]: starting Harbor ... [+] Running 9/10 ⠸ Network harbor_harbor Created 2.3s ✔ Container harbor-log Started 0.5s ✔ Container redis Started 1.4s ✔ Container harbor-portal Started 1.1s ✔ Container registryctl Started 1.3s ✔ Container harbor-db Started 1.2s ✔ Container registry Started 1.0s ✔ Container harbor-core Started 1.7s ✔ Container nginx Started 2.1s ✔ Container harbor-jobservice Started 2.0s ✔ ----Harbor has been installed and started successfully.---- [root@localhost harbor]#
看到最后输出的【✔ ----Harbor has been installed and started successfully.----】就是成功了。
服务正常但是访问不了：
可能是ip转发没有开启：
[root@harbor ~]# sysctl net.ipv4.ip_forward net.ipv4.ip_forward = 0 [root@harbor ~]#
需要将其开启：
[root@harbor ~]# cat /etc/sysctl.conf |grep -v '^#'|grep -v '^$' net.ipv4.ip_forward = 1 [root@harbor ~]# systemctl restart network.service [root@harbor ~]#
配置私有仓库
登录harbor：
创建用户
专门用来拉取上传的用户，等会要用到，不然总不能使用admin去干这事吧。
点击系统管理>>用户管理>>创建用户：
创建项目
点击项目>>新建项目
添加成员，查看镜像拉取命令
在项目里面，添加成员 ，角色为开发人员，具有推送拉取镜像的权限
查看命令
上传和拉取私有镜像
配置镜像仓库地址并重启docker和harbor服务
#添加私有仓库地址 [root@localhost harbor]# cat /etc/docker/daemon.json { "registry-mirrors": ["https://registry.docker-cn.com","https://gg3gwnry.mirror.aliyuncs.com"], "insecure-registries": ["http://192.168.24.5:80"] } [root@localhost harbor]# #cd到harbor所在的目录 [root@localhost harbor]# pwd /root/harbor/harbor [root@localhost harbor]# #重启docker，重启docker会导致harbor停止，所以docker重启完了后也要启动harbor [root@localhost harbor]# systemctl restart docker #harbor用docker-compose启动，docker-compose启动的时候要在harbor目录中使用docker-compse命令 [root@localhost harbor]# docker-compose up -d [+] Running 9/9 ✔ Container harbor-log Running 0.0s ✔ Container harbor-db Running 0.0s ✔ Container registry Running 0.0s ✔ Container registryctl Running 0.0s ✔ Container harbor-portal Running 0.0s ✔ Container redis Running 0.0s ✔ Container harbor-core Started 0.0s ✔ Container nginx Started 0.0s ✔ Container harbor-jobservice Started 0.0s [root@localhost harbor]# [root@localhost harbor]# docker login http://192.168.24.5:80 Username: k8s Password: WARNING! Your password will be stored unencrypted in /root/.docker/config.json. Configure a credential helper to remove this warning. See https://docs.docker.com/engine/reference/commandline/login/#credentials-store #上面的警告是在说密码将以明文的方式存储在/root/.docker/config.json文件中。 Login Succeeded [root@localhost harbor]#
拉取镜像、打标签、上传镜像和下载镜像
[root@localhost ~]# docker pull busybox:latest latest: Pulling from library/busybox 5cc84ad355aa: Pull complete Digest: sha256:5acba83a746c7608ed544dc1533b87c737a0b0fb730301639a0179f9344b1678 Status: Downloaded newer image for busybox:latest docker.io/library/busybox:latest [root@localhost ~]# [root@localhost harbor]# docker images busybox REPOSITORY TAG IMAGE ID CREATED SIZE busybox latest beae173ccac6 2 years ago 1.24MB [root@localhost harbor]# #需要注意的是，打标签的时候需要将端口带上，不然会被拒绝： #unauthorized: unauthorized to access repository: k8s/busybox, action: push: unauthorized to access repository: k8s/busybox, action: push [root@localhost harbor]# docker image tag busybox:latest 192.168.24.5:80/k8s/busybox:my [root@localhost harbor]# docker push 192.168.24.5:80/k8s/busybox:my The push refers to repository [192.168.24.5:80/k8s/busybox] 01fd6df81c8e: Pushed my: digest: sha256:62ffc2ed7554e4c6d360bce40bbcf196573dd27c4ce080641a2c59867e732dee size: 527 [root@localhost harbor]#
查看
harbor卸载重装
[root@localhost harbor]# pwd /root/harbor/harbor [root@localhost harbor]# #我是直接删除所有的容器了，但是实际使用时需要确保目前只有harbor相关的容器 #并且，相关的数据都是通过挂载的方式存储到本地文件系统的，你可能也需要一并删除：$ rm -rf `find / -name harbor` [root@localhost harbor]# docker rm -f $(docker ps -qa) cb5833e4fe17 783cb8ecbfc8 c57fe2f54699 25e4a0da5571 feafadba960a a92e81bf9077 270dbd3ea3d2 a9de5ae0da83 b54d796a6911 [root@localhost harbor]# #按照需要修改好配置文件后，重新安装即可： [root@localhost harbor]# ./install.sh
使用systemctl管理harbor
#编写harbor.service文件 cat /etc/systemd/system/harbor.service #- Unit部分用于描述服务单元的基本信息，包括描述和依赖关系。在这里，描述了Harbor服务，并指定它在docker.service之后启动，并且需要docker.service。 [Unit] Description=Harbor Service After=docker.service Requires=docker.service #- Service部分定义了实际的服务配置。指定服务的类型为oneshot，这意味着它只会运行一次，并且在退出后仍然保持激活状态。还指定了服务的工作目录和启动/停止命令。 [Service] Type=oneshot RemainAfterExit=yes WorkingDirectory=/root/harbor/harbor #注意你的docker-compose程序的位置。 ExecStart=/usr/local/bin/docker-compose -f /root/harbor/harbor/docker-compose.yml up -d ExecStop=/usr/local/bin/docker-compose -f /root/harbor/harbor/docker-compose.yml down #- Install部分定义了服务的安装信息。在这个例子中，指定了服务在multi-user.target下启动。 [Install] WantedBy=multi-user.target #虽然我们只指定了启动和停止的命令，但是使用systemctl restart harbor也不会报错。 systemctl daemon-reload # 加载新编写的服务项 sudo systemctl enable harbor #加入开启启动项中 sudo systemctl start harbor # 启动服务 sudo systemctl stop harbor # 停止服务 sudo systemctl restart harbor # 重启服务 sudo systemctl status harbor # 查看服务状态
使用ctr下载、上传镜像
#需要containerd，一般安装了最新版的docker-ce就会有。 #没有的话yum install -y containerd ctr image pull --user k8s:Lovedan@971220 -k 192.168.24.5/k8s/busybox:my
下载到crictl能看到的空间：
ctr image pull --user k8s:Lovedan@971220 -k 192.168.24.5/k8s/busybox:my -n k8s.io
更该镜像标签，上传镜像
[root@harbor ~]# ctr image tag registry.k8s.io/pause:3.9 192.168.24.5:80/k8s/pause:3.9 192.168.24.5:80/k8s/pause:3.9 [root@harbor ~]# [root@harbor containerd]# ctr image push --skip-verify --user k8s:Lovedan@971220 192.168.24.5:80/k8s/pause:3.9 --plain-http index-sha256:7031c1b283388d2c2e09b57badb803c05ebed362dc88d84b480cc47f72a21097: done |++++++++++++++++++++++++++++++++++++++| manifest-sha256:4fe1c6e55770c9fe3b57c6d3467d4a7d2a319f30a4c10b76ebdcf0eb550f428a: done |++++++++++++++++++++++++++++++++++++++| manifest-sha256:8d4106c88ec0bd28001e34c975d65175d994072d65341f62a8ab0754b0fafe10: done |++++++++++++++++++++++++++++++++++++++| manifest-sha256:cad97be0e860c5499b2970d7dd163c43cd08150b8551d3b7cbfa5f760ca69757: done |++++++++++++++++++++++++++++++++++++++| manifest-sha256:3ec98b8452dc8ae265a6917dfb81587ac78849e520d5dbba6de524851d20eca6: done |++++++++++++++++++++++++++++++++++++++| manifest-sha256:d9815368e0c7e55f30425cf0dc8fded3dc91437167803eef5cf12f71d3d9f075: done |++++++++++++++++++++++++++++++++++++++| manifest-sha256:313c244fbd9e5c29d41089a388c5768112a74cb02ce3328f2424b43981bd7602: done |++++++++++++++++++++++++++++++++++++++| manifest-sha256:4e2a8276ec1b1ecb4e43de739b18877964e3ed5cf89da1628f4a056b05941aec: done |++++++++++++++++++++++++++++++++++++++| config-sha256:e6f1816883972d4be47bd48879a08919b96afcd344132622e4d444987919323c: done |++++++++++++++++++++++++++++++++++++++| config-sha256:b1ab439917a297998343d6498e1e08ecfb764f4c77f965b46707b225f3abb452: done |++++++++++++++++++++++++++++++++++++++| config-sha256:f393c7d2b9c307af7aef7c283e52da404f3e5ece2ff81f40934db8b7ace447a9: done |++++++++++++++++++++++++++++++++++++++| config-sha256:83514219e9e6d1ff7e9a1fd0fb21b21d3ce08be2d3e27ced5f45f4e6c853a1bf: done |++++++++++++++++++++++++++++++++++++++| config-sha256:829e9de338bd5fdd3f16f68f83a9fb288fbc8453e881e5d5cfd0f6f2ff72b43e: done |++++++++++++++++++++++++++++++++++++++| config-sha256:fc778a09da73342ee71c0e2340979776acc6ea5a58e7d21ece7bbf849148450d: done |++++++++++++++++++++++++++++++++++++++| config-sha256:ad0e2f66ca0b6d5c762569865babaa80da7405d1a6982d93d3858742ba088e7d: done |++++++++++++++++++++++++++++++++++++++| elapsed: 1.1 s total: 18.6 K (16.9 KiB/s) [root@harbor containerd]#
K8s使用harbor镜像
首先创建docker-registry类型的secret：
[root@k8s-master0 20240401]# kubectl create secret docker-registry secret-harbor --namespace default --docker-server http://192.168.24.5:80 --docker-username k8s --docker-password Lovedan@971220 secret/secret-harbor created [root@k8s-master0 20240401]#
然后再yaml文件中指定镜像的完整路径：
[root@k8s-master0 20240401]# cat test.yaml apiVersion: v1 kind: Pod metadata: name: nginx labels: name: nginx spec: containers: - name: nginx image: 192.168.24.5/k8s/nginx:latest ports: - containerPort: 80 [root@k8s-master0 20240401]#
查看pod：
[root@k8s-master0 20240401]# kubectl get pod NAME READY STATUS RESTARTS AGE nginx 1/1 Running 0 18s [root@k8s-master0 20240401]#
pod正常：
[root@k8s-master0 20240401]# kubectl get pod -o wide NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES nginx 1/1 Running 0 54m 10.244.3.2 k8s-node2 <none> <none> [root@k8s-master0 20240401]# curl http://10.244.3.2 <!DOCTYPE html> <html> <head> <title>Welcome to nginx!</title> <style> html { color-scheme: light dark; } body { width: 35em; margin: 0 auto; font-family: Tahoma, Verdana, Arial, sans-serif; } </style> </head> <body> <h1>Welcome to nginx!</h1> <p>If you see this page, the nginx web server is successfully installed and working. Further configuration is required.</p> <p>For online documentation and support please refer to <a href="http://nginx.org/">nginx.org</a>.<br/> Commercial support is available at <a href="http://nginx.com/">nginx.com</a>.</p> <p><em>Thank you for using nginx.</em></p> </body> </html> [root@k8s-master0 20240401]#
