基于nerdctl + buildkitd+containerd构建容器镜像
一、介绍
容器技术除了docker之外，还有coreOS的rkt、google的gvisor、以及docker开源的containerd、redhat的podman、阿里的pouch等，为了保证容器生态的标准性和健康可持续发展，包括Linux基金会、Docker、微软、红帽、谷歌和IBM等公司在2015年6月共同成立一个叫open container（OCI）的组织，其目的就是制定开发的标准的容器规范，目前OCI一共发布了两个规范，分别是runtime spec和image format spec, 有了这两个规范，不同的容器公司开发的容器只兼容这两个规范，就可以保证容器的可移植性和相互可操作性。buildkit是从Docker公司的开源出来的一个镜像构建工具包，支持OCI标准的镜像构建，其中buildkitd服务端，支持runc和containerd作为构建环境，buildctl客户端，负责解析Dockerfile文件，并向服务端buildkitd发出构建请求。
详细文档可以查看: https://github.com/moby/buildkit
buildkit具备如下特征：
- 自动垃圾收集
- 可扩展的前端格式
- 并发依赖解析
- 高效的指令缓存
- 构建缓存导入/导出
- 嵌套构建作业调用
- 可分配工人
- 多种输出格式
- 可插拔架构
- 无root权限执行
二、安装containerd、runc、nerdctl、buildkit组件
安装containerd
下载containerd wget https://github.com/containerd/containerd/releases/download/v1.6.6/cri-containerd-cni-1.6.6-linux-amd64.tar.gz mkdir containerd && tar -xvzf cri-containerd-cni-1.6.6-linux-amd64.tar.gz -C containerd
cp -p /root/containerd/usr/local/bin/* /usr/local/bin/ cp -p /root/containerd/etc/systemd/system/containerd.service /usr/lib/systemd/system/containerd.service chmod +x /usr/lib/systemd/system/containerd.service
[root@easzlab-images-01 ~]# cat /lib/systemd/system/containerd.service # Copyright The containerd Authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. [Unit] Description=containerd container runtime Documentation=https://containerd.io After=network.target local-fs.target [Service] ExecStartPre=-/sbin/modprobe overlay ExecStart=/usr/local/bin/containerd Type=notify Delegate=yes KillMode=process Restart=always RestartSec=5 # Having non-zero Limit*s causes performance problems due to accounting overhead # in the kernel. We recommend using cgroups to do container-local accounting. LimitNPROC=infinity LimitCORE=infinity LimitNOFILE=infinity # Comment TasksMax if your systemd version does not supports it. # Only systemd 226 and above support this version. TasksMax=infinity OOMScoreAdjust=-999 [Install] WantedBy=multi-user.target [root@easzlab-images-01 ~]# [root@easzlab-images-01 ~]# systemctl enable --now containerd.service
安装runc
wget https://github.com/opencontainers/runc/releases/download/v1.1.3/runc.amd64 -O /usr/bin/runc chmod a+x /usr/bin/runc
安装nerdctl、buildkit
#下载nerdctl、buildkit安装包 wget https://github.com/containerd/nerdctl/releases/download/v0.22.2/nerdctl-0.22.2-linux-amd64.tar.gz wget https://github.com/moby/buildkit/releases/download/v0.10.3/buildkit-v0.10.3.linux-amd64.tar.gz tar -xvzf nerdctl-0.22.2-linux-amd64.tar.gz -C /usr/local/bin/ tar -xvzf buildkit-v0.10.3.linux-amd64.tar.gz -C /usr/local/bin/ root@easzlab-k8s-master-01:~/software# ll -h /usr/local/bin/bin/ total 111M drwxr-xr-x 2 root root 4.0K Oct 21 2015 ./ drwxr-xr-x 3 root root 4.0K Aug 9 15:06 ../ -rwxr-xr-x 1 root root 25M Oct 21 2015 buildctl* -rwxr-xr-x 1 root root 38M Oct 21 2015 buildkitd* -rwxr-xr-x 1 root root 5.2M Oct 21 2015 buildkit-qemu-aarch64* -rwxr-xr-x 1 root root 3.8M Oct 21 2015 buildkit-qemu-arm* -rwxr-xr-x 1 root root 2.9M Oct 21 2015 buildkit-qemu-i386* -rwxr-xr-x 1 root root 3.3M Oct 21 2015 buildkit-qemu-mips64* -rwxr-xr-x 1 root root 3.2M Oct 21 2015 buildkit-qemu-mips64el* -rwxr-xr-x 1 root root 3.8M Oct 21 2015 buildkit-qemu-ppc64le* -rwxr-xr-x 1 root root 3.4M Oct 21 2015 buildkit-qemu-riscv64* -rwxr-xr-x 1 root root 2.9M Oct 21 2015 buildkit-qemu-s390x* -rwxr-xr-x 1 root root 20M Oct 21 2015 buildkit-runc* root@easzlab-k8s-master-01:~/software# mv /usr/local/bin/bin/buildctl /usr/local/bin/bin/buildkitd /usr/local/bin/ root@easzlab-k8s-master-01:~# ll -h /usr/local/bin/ total 189M drwxr-xr-x 3 root root 4.0K Aug 9 17:28 ./ drwxr-xr-x 10 root root 4.0K Feb 23 16:50 ../ drwxr-xr-x 2 root root 4.0K Aug 9 15:12 bin/ -rwxr-xr-x 1 root root 25M Oct 21 2015 buildctl* -rwxr-xr-x 1 root root 38M Oct 21 2015 buildkitd* -rwxr-xr-x 1 root root 32M Jul 28 00:19 crictl* -rwxr-xr-x 1 root root 27M Aug 2 01:54 nerdctl* -rwxr-xr-x 1 502 dialout 69M Mar 15 10:07 velero* root@easzlab-k8s-master-01:~#
#在/usr/lib/systemd/system下配置buildkit启动配置文件 root@easzlab-k8s-master-01:~# cat buildkit.service [Unit] Description=BuildKit Requires=buildkit.socket After=buildkit.socket Documentation=https://github.com/moby/buildkit [Service] Type=notify ExecStart=/usr/local/bin/buildkitd --addr fd:// [Install] WantedBy=multi-user.target root@easzlab-k8s-master-01:~# root@easzlab-k8s-master-01:~# cat buildkit.socket [Unit] Description=BuildKit Documentation=https://github.com/moby/buildkit [Socket] ListenStream=%t/buildkit/buildkitd.sock SocketMode=0660 [Install] WantedBy=sockets.target root@easzlab-k8s-master-01:~#
#启动buildkit服务
systemctl enable --now buildkit.service buildkit.socket
#增加nerdctl自动补全功能
在/etc/profile末尾添加source <(nerdctl completion bash)
执行source /etc/profile让其生效
#验证是否能正常连接harbor
root@easzlab-k8s-master-01~:# nerdctl login --insecure-registry https://harbor.magedu.net WARN[0000] skipping verifying HTTPS certs for "harbor.magedu.net" WARNING: Your password will be stored unencrypted in /root/.docker/config.json. Configure a credential helper to remove this warning. See https://docs.docker.com/engine/reference/commandline/login/#credentials-store Login Succeeded root@easzlab-k8s-master-01~:#
#构建镜像并上传到harbor仓库
nerdctl build -t harbor.magedu.net/magedu/nginx-base:1.22.0 . #构建nginx1.22.0镜像
nerdctl push harbor.magedu.net/magedu/nginx-base:1.22.0 #将镜像上传到harbor
root@easzlab-k8s-master-01:~/ubunt-nginx-Dockerfile# ll -h total 1.1M drwxr-xr-x 3 root root 4.0K Aug 11 00:37 ./ drwx------ 13 root root 4.0K Aug 11 00:10 ../ -rw-r--r-- 1 root root 266 Aug 5 15:03 build-command.sh -rw-r--r-- 1 root root 886 Aug 9 17:35 Dockerfile -rw-r--r-- 1 root root 38K Aug 5 14:39 frontend.tar.gz drwxr-xr-x 3 root root 4.0K Aug 11 00:37 html/ -rw-r--r-- 1 root root 1.1M May 24 22:29 nginx-1.22.0.tar.gz -rw-r--r-- 1 root root 2.8K Oct 3 2020 nginx.conf -rw-r--r-- 1 root root 1.2K Aug 5 14:53 sources.list root@easzlab-k8s-master-01:~/ubunt-nginx-Dockerfile# nerdctl build -t harbor.magedu.net/magedu/nginx-base:1.22.0 . [+] Building 277.2s (12/12) FINISHED => [internal] load .dockerignore 0.0s => => transferring context: 2B 0.0s => [internal] load build definition from Dockerfile 0.0s => => transferring dockerfile: 925B 0.0s => [internal] load metadata for docker.io/library/ubuntu:22.04 4.7s => [1/7] FROM docker.io/library/ubuntu:22.04@sha256:34fea4f31bf187bc915536831fd0afc9d214755bf700b5cdb1336c82516d154e 18.4s => => resolve docker.io/library/ubuntu:22.04@sha256:34fea4f31bf187bc915536831fd0afc9d214755bf700b5cdb1336c82516d154e 0.0s => => sha256:d19f32bd9e4106d487f1a703fc2f09c8edadd92db4405d477978e8e466ab290d 30.43MB / 30.43MB 13.5s => => extracting sha256:d19f32bd9e4106d487f1a703fc2f09c8edadd92db4405d477978e8e466ab290d 4.8s => [internal] load build context 0.0s => => transferring context: 41.70kB 0.0s => [2/7] RUN apt update && apt install -y iproute2 ntpdate tcpdump telnet traceroute nfs-kernel-server nfs-common lrzsz tree openssl libssl-dev libpcre3 libpcre3-dev zlib1g-dev ntpdate tcpdump t 118.8s => [3/7] ADD nginx-1.22.0.tar.gz /usr/local/src/ 0.8s => [4/7] RUN cd /usr/local/src/nginx-1.22.0 && ./configure --prefix=/apps/nginx && make && make install && ln -sv /apps/nginx/sbin/nginx /usr/bin 87.6s => [5/7] RUN groupadd -g 2088 nginx && useradd -g nginx -s /usr/sbin/nologin -u 2088 nginx && chown -R nginx.nginx /apps/nginx 0.5s => [6/7] ADD nginx.conf /apps/nginx/conf/ 0.1s => [7/7] ADD frontend.tar.gz /apps/nginx/html/ 0.2s => exporting to oci image format 46.2s => => exporting layers 37.3s => => exporting manifest sha256:99926d451d59b92d018992c2b3c567bce5c3f44dfa3f7061bcbdcb973a057c9c 0.0s => => exporting config sha256:3dee21976c9b02b85a34a1cbd74fb2483ca01b44ce3bd17f689e0ed06fef51f2 0.0s => => sending tarball 8.8s unpacking harbor.magedu.net/magedu/nginx-base:1.22.0 (sha256:99926d451d59b92d018992c2b3c567bce5c3f44dfa3f7061bcbdcb973a057c9c)...done root@easzlab-k8s-master-01:~/ubunt-nginx-Dockerfile# nerdctl push harbor.magedu.net/magedu/nginx-base:1.22.0 INFO[0000] pushing as a reduced-platform image (application/vnd.docker.distribution.manifest.v2+json, sha256:99926d451d59b92d018992c2b3c567bce5c3f44dfa3f7061bcbdcb973a057c9c) manifest-sha256:99926d451d59b92d018992c2b3c567bce5c3f44dfa3f7061bcbdcb973a057c9c: done |++++++++++++++++++++++++++++++++++++++| config-sha256:3dee21976c9b02b85a34a1cbd74fb2483ca01b44ce3bd17f689e0ed06fef51f2: done |++++++++++++++++++++++++++++++++++++++| elapsed: 6.4 s total: 5.1 Ki (809.0 B/s) root@easzlab-k8s-master-01:~/ubunt-nginx-Dockerfile#
#查看harbor是否上传刚刚构建的镜像 
#其他节点下载测试
拷贝nerdctl到该节点
root@easzlab-k8s-master-01:~# scp /usr/local/bin/nerdctl root@172.16.88.157:/usr/local/bin/ root@172.16.88.157's password: nerdctl 100% 27MB 43.0MB/s 00:00 root@easzlab-k8s-master-01:~#
#上传其他下载好的镜像到harbor
#将镜像重新打tag标签
nerdctl tag quay.io/cephcsi/cephcsi:v3.6.2 harbor.magedu.net/rook-ceph/cephcsi:v3.6.2 nerdctl tag registry.k8s.io/sig-storage/csi-attacher:v3.4.0 harbor.magedu.net/rook-ceph/csi-attacher:v3.4.0 nerdctl tag registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.5.1 harbor.magedu.net/rook-ceph/csi-node-driver-registrar:v2.5.1 nerdctl tag registry.k8s.io/sig-storage/csi-provisioner:v3.1.0 harbor.magedu.net/rook-ceph/csi-provisioner:v3.1.0 nerdctl tag registry.k8s.io/sig-storage/csi-resizer:v1.4.0 harbor.magedu.net/rook-ceph/csi-resizer:v1.4.0 nerdctl tag registry.k8s.io/sig-storage/csi-snapshotter:v6.0.1 harbor.magedu.net/rook-ceph/csi-snapshotter:v6.0.1
#导入镜像
for i in `ls -lh |awk '{print $9}'`;do nerdctl load -i $i;done
#验证镜像上传
for i in `nerdctl images |grep harbor|awk '{print $1,$2}' |tr -s ' ' ':'`;do nerdctl push $i;done
#检查上否上传成功
三、配置harbor服务nginx反向代理
通过nginx服务反向代理，解决containerd通过dockerfile打包构建，拉取harbor仓库镜像取失败问题
案例场景
[root@easzlab-images-02 jdk-1.8.212]# ll -h
total 186M
drwxr-xr-x 2 root root 4.0K Oct 15 14:52 ./
drwxr-xr-x 6 root root 4.0K Aug 26 20:43 ../
-rw-r--r-- 1 root root 137 Oct 15 14:52 build-command.sh
-rw-r--r-- 1 root root 413 Aug 26 20:43 Dockerfile
-rw-r--r-- 1 root root 186M Aug 26 20:43 jdk-8u212-linux-x64.tar.gz
-rw-r--r-- 1 root root 2.1K Aug 26 20:43 profile
[root@easzlab-images-02 jdk-1.8.212]# cat Dockerfile
#JDK Base Image
FROM harbor.magedu.net/baseimages/magedu-centos-base:7.9.2009
ADD jdk-8u212-linux-x64.tar.gz /usr/local/src/
RUN ln -sv /usr/local/src/jdk1.8.0_212 /usr/local/jdk
ADD profile /etc/profile
ENV JAVA_HOME /usr/local/jdk
ENV JRE_HOME $JAVA_HOME/jre
ENV CLASSPATH $JAVA_HOME/lib/:$JRE_HOME/lib/
ENV PATH $PATH:$JAVA_HOME/bin
[root@easzlab-images-02 jdk-1.8.212]#
[root@easzlab-images-02 jdk-1.8.212]# cat build-command.sh
#!/bin/bash
nerdctl build -t harbor.magedu.net/pub-images/jdk-base:v8.212-2 .
nerdctl push harbor.magedu.net/pub-images/jdk-base:v8.212-2
[root@easzlab-images-02 jdk-1.8.212]# bash build-command.sh
[+] Building 0.3s (3/3) FINISHED
=> [internal] load build definition from Dockerfile 0.1s
=> => transferring dockerfile: 452B 0.0s
=> [internal] load .dockerignore 0.1s
=> => transferring context: 2B 0.0s
=> ERROR [internal] load metadata for harbor.magedu.net/baseimages/magedu-centos-base:7.9.2009 0.2s
------
> [internal] load metadata for harbor.magedu.net/baseimages/magedu-centos-base:7.9.2009:
------
Dockerfile:2
--------------------
1 | #JDK Base Image
2 | >>> FROM harbor.magedu.net/baseimages/magedu-centos-base:7.9.2009
3 | #FROM centos:7.9.2009
4 |
--------------------
error: failed to solve: harbor.magedu.net/baseimages/magedu-centos-base:7.9.2009: failed to do request: Head "https://harbor.magedu.net/v2/baseimages/magedu-centos-base/manifests/7.9.2009": x509: certificate signed by unknown authority
FATA[0001] unrecognized image format
FATA[0000] failed to create a tmp single-platform image "harbor.magedu.net/pub-images/jdk-base:v8.212-2-tmp-reduced-platform": image "harbor.magedu.net/pub-images/jdk-base:v8.212-2": not found
[root@easzlab-images-02 jdk-1.8.212]#
解决办法
确保harbor签发证书都同步到客户端，containerd服务也做了相关认证配置
[root@easzlab-images-02 ~]# mkdir -p /etc/containerd/certs.d/harbor.magedu.net
root@easzlab-k8s-harbor-01:/apps/harbor/certs# scp -r ca.crt magedu.net.cert magedu.net.key root@172.16.88.172:/etc/containerd/certs.d/harbor.magedu.net
[root@easzlab-images-02 ~]# ll -h /etc/containerd/certs.d/harbor.magedu.net/
total 20K
drwxr-xr-x 2 root root 4.0K Oct 15 14:59 ./
drwxr-xr-x 3 root root 4.0K Oct 15 14:57 ../
-rw-r--r-- 1 root root 2.0K Oct 15 14:59 ca.crt
-rw-r--r-- 1 root root 2.1K Oct 15 14:59 magedu.net.cert
-rw------- 1 root root 3.2K Oct 15 14:59 magedu.net.key
[root@easzlab-images-02 ~]#
[root@easzlab-images-02 ~]# vim /etc/containerd/config.toml
[root@easzlab-images-02 ~]# grep "magedu" -C 10 /etc/containerd/config.toml
145 [plugins."io.containerd.grpc.v1.cri".registry.mirrors."easzlab.io.local:5000"]
146 endpoint = ["http://easzlab.io.local:5000"]
147 [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
148 endpoint = ["https://docker.mirrors.ustc.edu.cn", "http://hub-mirror.c.163.com"]
149 [plugins."io.containerd.grpc.v1.cri".registry.mirrors."gcr.io"]
150 endpoint = ["https://gcr.mirrors.ustc.edu.cn"]
151 [plugins."io.containerd.grpc.v1.cri".registry.mirrors."k8s.gcr.io"]
152 endpoint = ["https://gcr.mirrors.ustc.edu.cn/google-containers/"]
153 [plugins."io.containerd.grpc.v1.cri".registry.mirrors."quay.io"]
154 endpoint = ["https://quay.mirrors.ustc.edu.cn"]
155 [plugins."io.containerd.grpc.v1.cri".registry.mirrors."harbor.magedu.net"]
156 endpoint = ["https://harbor.magedu.net"]
157 [plugins."io.containerd.grpc.v1.cri".registry.configs."harbor.magedu.net".tls]
158 insecure_skip_verify = false
159 ca_file = "/etc/containerd/certs.d/harbor.magedu.net/ca.crt"
160 cert_file = "/etc/containerd/certs.d/harbor.magedu.net/magedu.net.cert"
161 key_file = "/etc/containerd/certs.d/harbor.magedu.net/magedu.net.key"
162 [plugins."io.containerd.grpc.v1.cri".registry.configs."harbor.magedu.net".auth]
163 username = "admin"
164 password = "Harbor12345"
165 [plugins."io.containerd.grpc.v1.cri".x509_key_pair_streaming]
166 tls_cert_file = ""
167 tls_key_file = ""
168
169 [plugins."io.containerd.internal.v1.opt"]
170 path = "/opt/containerd"
171
172 [plugins."io.containerd.internal.v1.restart"]
[root@easzlab-images-02 ~]#
#验证登录是否成功
[root@easzlab-images-02 ~]# nerdctl login https://harbor.magedu.net WARNING: Your password will be stored unencrypted in /root/.docker/config.json. Configure a credential helper to remove this warning. See https://docs.docker.com/engine/reference/commandline/login/#credentials-store Login Succeeded [root@easzlab-images-02 ~]#
再次构建还是报错
经过分析：发现nerdctl 调用buildctl使用dockerfile文件，不能直接访问harbor 443端口拉取镜像，只能通过harbor的80拉取，然而新版k8s与harbor连接需要通过https认证访问，所以选择nginx做中间层反向代理
修改harbor.yaml，关闭https，配置nginx代理
root@easzlab-k8s-harbor-01:/apps/harbor# vi harbor.yml
root@easzlab-k8s-harbor-01:/apps/harbor# cat harbor.yml |head -n 20
# Configuration file of Harbor
# The IP address or hostname to access admin UI and registry service.
# DO NOT use localhost or 127.0.0.1, because Harbor needs to be accessed by external clients.
hostname: harbor.magedu.net
# http related config
http:
# port for http, default is 80. If https enabled, this port will redirect to https port
port: 80
# https related config
#https:
# https port for harbor, default is 443
# port: 443
# The path of cert and key files for nginx
#certificate: /apps/harbor/certs/magedu.net.crt
#private_key: /apps/harbor/certs/magedu.net.key
# # Uncomment following will enable tls communication between all harbor components
root@easzlab-k8s-harbor-01:/apps/harbor#
更新harbor配置
root@easzlab-k8s-harbor-01:/apps/harbor# ./prepare
prepare base dir is set to /apps/harbor
WARNING:root:WARNING: HTTP protocol is insecure. Harbor will deprecate http protocol in the future. Please make sure to upgrade to https
Clearing the configuration file: /config/core/app.conf
Clearing the configuration file: /config/core/env
Clearing the configuration file: /config/registry/config.yml
Clearing the configuration file: /config/registry/passwd
Clearing the configuration file: /config/registry/root.crt
Clearing the configuration file: /config/portal/nginx.conf
Clearing the configuration file: /config/log/logrotate.conf
Clearing the configuration file: /config/log/rsyslog_docker.conf
Clearing the configuration file: /config/nginx/nginx.conf
Clearing the configuration file: /config/db/env
Clearing the configuration file: /config/jobservice/config.yml
Clearing the configuration file: /config/jobservice/env
Clearing the configuration file: /config/registryctl/config.yml
Clearing the configuration file: /config/registryctl/env
Generated configuration file: /config/portal/nginx.conf
Generated configuration file: /config/log/logrotate.conf
Generated configuration file: /config/log/rsyslog_docker.conf
Generated configuration file: /config/nginx/nginx.conf
Generated configuration file: /config/core/env
Generated configuration file: /config/core/app.conf
Generated configuration file: /config/registry/config.yml
Generated configuration file: /config/registryctl/env
Generated configuration file: /config/registryctl/config.yml
Generated configuration file: /config/db/env
Generated configuration file: /config/jobservice/env
Generated configuration file: /config/jobservice/config.yml
loaded secret from file: /data/secret/keys/secretkey
Generated configuration file: /compose_location/docker-compose.yml
Clean up the input dir
root@easzlab-k8s-harbor-01:/apps/harbor#
重新启动harbor服务组件
root@easzlab-k8s-harbor-01:/apps/harbor# docker-compose up -d
WARNING: Found orphan containers (chartmuseum, trivy-adapter) for this project. If you removed or renamed this service in your compose file, you can run this command with the --remove-orphans flag to clean it up.
harbor-log is up-to-date
harbor-db is up-to-date
harbor-portal is up-to-date
registry is up-to-date
Recreating registryctl ...
Recreating registryctl ... done
Recreating harbor-core ... done
Recreating nginx ... done
Recreating harbor-jobservice ... done
root@easzlab-k8s-harbor-01:/apps/harbor#
确保此时只能通过http访问harbor
配置负载均衡器，这里选择nginx
#源码编译安装nginxf服务
[root@easzlab-k8s-harbor-nginx-01 ~]# wget http://nginx.org/download/nginx-1.20.2.tar.gz [root@easzlab-k8s-harbor-nginx-01 ~]# mkdir -p /apps/nginx [root@easzlab-k8s-harbor-nginx-01 ~]# tar -xf nginx-1.20.2.tar.gz [root@easzlab-k8s-harbor-nginx-01 ~]# cd nginx-1.20.2/ [root@easzlab-k8s-harbor-nginx-01 nginx-1.20.2]# ls CHANGES CHANGES.ru LICENSE README auto conf configure contrib html man src [root@easzlab-k8s-harbor-nginx-01 nginx-1.20.2]# apt update && apt -y install gcc make libpcre3 libpcre3-dev openssl libssl-dev zlib1g-dev [root@easzlab-k8s-harbor-nginx-01 nginx-1.20.2]#./configure --prefix=/apps/nginx \ --with-http_ssl_module \ --with-http_v2_module \ --with-http_realip_module \ --with-http_stub_status_module \ --with-http_gzip_static_module \ --with-pcre \ --with-stream \ --with-stream_ssl_module \ --with-stream_realip_module [root@easzlab-k8s-harbor-nginx-01 nginx-1.20.2]# make && make install
#同步harbor签发证书到客户端
root@easzlab-k8s-harbor-01:/apps/harbor/certs# scp magedu.net.crt magedu.net.key root@172.16.88.190:/apps/nginx/certs [root@easzlab-k8s-harbor-nginx-01 ~]# ll -h /apps/nginx/certs total 20K drwxr-xr-x 2 root root 4.0K Oct 15 16:00 ./ drwxr-xr-x 7 1001 1001 4.0K Oct 15 15:58 ../-rw-r--r-- 1 root root 2.1K Oct 15 16:00 magedu.net.crt -rw------- 1 root root 3.2K Oct 15 16:00 magedu.net.key [root@easzlab-k8s-harbor-nginx-01 ~]# [root@easzlab-k8s-harbor-nginx-01 ~]# egrep -v "^$|^#|^[[:space:]]+#" /apps/nginx/conf/nginx.conf worker_processes 1; events { worker_connections 1024; } http { include mime.types; default_type application/octet-stream; sendfile on; keepalive_timeout 65;
client_max_body_size 2000m; #设置上传harbor单个镜像最大为2G server { listen 80; listen 443 ssl;
server_name harbor.magedu.net; ssl_certificate /apps/nginx/certs/magedu.net.crt; ssl_certificate_key /apps/nginx/certs/magedu.net.key; ssl_session_cache shared:sslcache:20m; ssl_session_timeout 10m; location / { proxy_pass http://172.16.88.166; } error_page 500 502 503 504 /50x.html; location = /50x.html { root html; } } } [root@easzlab-k8s-harbor-nginx-01 conf]# /apps/nginx/sbin/nginx -t nginx: the configuration file /apps/nginx/conf/nginx.conf syntax is ok nginx: configuration file /apps/nginx/conf/nginx.conf test is successful [root@easzlab-k8s-harbor-nginx-01 conf]# /apps/nginx/sbin/nginx [root@easzlab-k8s-harbor-nginx-01 conf]# netstat -tnlp Active Internet connections (only servers) Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name tcp 0 0 127.0.0.53:53 0.0.0.0:* LISTEN 629/systemd-resolve tcp 0 0 0.0.0.0:22 0.0.0.0:* LISTEN 786/sshd: /usr/sbin tcp 0 0 0.0.0.0:443 0.0.0.0:* LISTEN 9702/nginx: master tcp 0 0 127.0.0.1:6011 0.0.0.0:* LISTEN 1231/sshd: root@pts tcp 0 0 127.0.0.1:6012 0.0.0.0:* LISTEN 8978/sshd: root@pts tcp 0 0 0.0.0.0:80 0.0.0.0:* LISTEN 9702/nginx: master tcp6 0 0 :::22 :::* LISTEN 786/sshd: /usr/sbin tcp6 0 0 ::1:6011 :::* LISTEN 1231/sshd: root@pts tcp6 0 0 ::1:6012 :::* LISTEN 8978/sshd: root@pts [root@easzlab-k8s-harbor-nginx-01 conf]#
验证harbor https反向代理
添加nerdctl、buildkitd配置⽂件
[root@easzlab-images-02 ~]# mkdir -p /etc/buildkit/
[root@easzlab-images-02 ~]# mkdir /etc/nerdctl/
[root@easzlab-images-02 ~]# vi /etc/buildkit/buildkitd.toml
[root@easzlab-images-02 ~]# cat /etc/buildkit/buildkitd.toml
[registry."harbor.magedu.net"]
http = true
insecure = true
[root@easzlab-images-02 ~]#
[root@easzlab-images-02 ~]# vi /etc/nerdctl/nerdctl.toml
[root@easzlab-images-02 ~]# cat /etc/nerdctl/nerdctl.toml
namespace = "k8s.io"
debug = false
debug_full = false
insecure_registry = true
[root@easzlab-images-02 ~]#
[root@easzlab-images-02 jdk-1.8.212]# systemctl restart containerd.service
[root@easzlab-images-02 jdk-1.8.212]# systemctl restart buildkit.service
再次验证构建
