nerdctl buildkitd containerd构建容器镜像
21. nerdctl buildkitd containerd构建容器镜像
21.1 概述
容器技术除了docker之外，还有coreOS的rkt、google的gvisor、以及docker开源的containerd、redhat的podman、阿里的pouch等，为了保证容器生态的标准性和健康可持续发展、包括Linux基金会、Docker、微软、红帽、谷歌和IBM等公司在2015年6月共同成立了一个叫open cintainer（OCI）的组织，其目的就是指定开放的标准的容器规范，目前OCI一共发布了两个规范，分别是runtime spec和image format spec，有了这两个规范，不同的容器公司开发的容器只要兼容这两个规范，就可以保证容器的可移植性和相互可操作性
- 各个官方地址
https://containerd.io
https://gvisor.dev
https://podman.io
#阿里开源
https://github.com/alibaba/pouch
#已停止维护
https://github.com/rkt/rkt
#buildkit：从Docker公司的开源出来的一个镜像构建工具包，支持OCI标准的镜像构建
https://github.com/moby/buildkit
21.2 buildkitd组成部分
buildkitd（服务端）目前支持runc和containerd作为镜像构建环境，默认是runc，可以更换为containerd
buildctl（客户端）负责解析Dockerfile文件，并向服务端buildkitd发出构建请求
21.2 buildkitd部署
下载地址：Release v0.10.6 · moby/buildkit · GitHub
- 过程
root@k8s-master1:/usr/local/src# wget https://github.com/moby/buildkit/releases/download/v0.10.6/buildkit-v0.10.6.linux-amd64.tar.gz
root@k8s-master1:/usr/local/src# tar xf buildkit-v0.10.6.linux-amd64.tar.gz -C /usr/local/bin/
root@k8s-master1:/usr/local/bin/bin# mv ./* ../
#buildkit服务链接文件
root@k8s-master1:/usr/local/bin# vim /etc/systemd/system/buildkit.socket
[Unit]
Description=BuildKit
Documentation=https://github.com/moby/buildkit
[Socket]
ListenStream=%t/buildkit/buildkitd.sock
[Install]
WantedBy=sockets.target
#buildkitd服务文件
root@k8s-master1:/usr/local/bin# vim /etc/systemd/system/buildkit.service
[Unit]
Description=BuildKit
Requires=buildkit.socket
After=buildkit.socketDocumentation=https://github.com/moby/buildkit
[Service]
ExecStart=/usr/local/bin/buildkitd --oci-worker=false --containerd-worker=true
[Install]
WantedBy=multi-user.target
systemctl enable buildkitd.service
systemctl start buildkitd.service
- 下载安装nerdctl命令
#自行安装
如果使用命令直接登录会报错
root@k8s-master1:~# nerdctl login harbor.nbrhce.com
Enter Username: quyi
Enter Password:
ERRO[0017] failed to call tryLoginWithRegHost error="failed to call rh.Client.Do: Get \"https://harbor.nbrhce.com/v2/\": x509: certificate relies on legacy Common Name field, use SANs instead" i=0
FATA[0017] failed to call rh.Client.Do: Get "https://harbor.nbrhce.com/v2/": x509: certificate relies on legacy Common Name field, use SANs instead
#加上证书信任就可以登录 不过太麻烦
root@k8s-master1:~# nerdctl login --insecure-registry harbor.nbrhce.com
Enter Username: quyi
Enter Password:
WARN[0007] skipping verifying HTTPS certs for "harbor.nbrhce.com"
WARNING: Your password will be stored unencrypted in /root/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credentials-store
Login Succeeded
- nerdctl常用命令 这是不用证书的登录方式
vim /etc/profile
source <(nerdctl completion bash)
source /etc/profile
nerdctl login --insecure-registry harbor.nbrhce.com
nerdctl pull centos:7.9.2009
nerdctl tag centos:7.9.2009 harbor.nbrhce.com/baseimages/centos:7.9.2009
nerdctl push --insecure-registry centos:7.9.2009 harbor.nbrhce.com/baseimages/centos:7.9.2009
- 分发harbor证书 来实现登录
root@k8s-master1:~# mkdir /etc/containerd/certs.d/harbor.nbrhce.com -p
#这是在harbor服务器上
root@deploy-harbor:/apps/certs# pwd
/apps/certs
#如果之前没有做过这个证书 就执行一下
root@deploy-harbor:/apps/certs# openssl x509 -inform PEM -in harbor.nbrhce.com.crt -out harbor.nbrhce.com.cert
#拷贝过去
root@deploy-harbor:/apps/certs# scp ca.crt harbor.nbrhce.com.cert harbor.nbrhce.com.key 192.168.1.70:/etc/containerd/certs.d/harbor.nbrhce.com
ca.crt 100% 2061 1.6MB/s 00:00
harbor.nbrhce.com.cert 100% 2155 1.4MB/s 00:00
harbor.nbrhce.com.key 100% 3247 4.5MB/s 00:00
#登录
root@k8s-master1:/etc/containerd/certs.d/harbor.nbrhce.com# nerdctl login harbor.nbrhce.com
Enter Username: quyi
Enter Password:
WARNING: Your password will be stored unencrypted in /root/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credentials-store
Login Succeeded
root@k8s-master1:~# nerdctl pull centos:7.9.2009
root@k8s-master1:~# nerdctl tag centos:7.9.2009 harbor.nbrhce.com/baseimages/centos:7.9.2009
root@k8s-master1:~# nerdctl push harbor.nbrhce.com/baseimages/centos:7.9.2009
INFO[0000] pushing as a reduced-platform image (application/vnd.docker.distribution.manifest.list.v2+json, sha256:48838bdd8842b5b6bf36ef486c5b670b267ed6369e2eccbf00d520824f753ee2)
index-sha256:48838bdd8842b5b6bf36ef486c5b670b267ed6369e2eccbf00d520824f753ee2: done |++++++++++++++++++++++++++++++++++++++|
manifest-sha256:dead07b4d8ed7e29e98de0f4504d87e8880d4347859d839686a31da35a3b532f: done |++++++++++++++++++++++++++++++++++++++|
config-sha256:eeb6ee3f44bd0b5103bb561b4c16bcb82328cfe5809ab675bb17ab3a16c517c9: done |++++++++++++++++++++++++++++++++++++++|
elapsed: 1.8 s total: 3.5 Ki (2.0 KiB/s)
- 上传一个Dockerfile 构建
FROM ubuntu:22.04
MAINTAINER "nginx"
#ADD sources.list /etc/apt/sources.list
RUN apt update && apt install -y iproute2 ntpdate tcpdump telnet traceroute nfs-kernel-server nfs-common lrzsz tree openssl libssl-dev libpcre3 libpcre3-dev zlib1g-dev ntpdate tcpdump telnet traceroute gcc openssh-server lrzsz tree openssl libssl-dev libpcre3 libpcre3-dev zlib1g-dev ntpdate tcpdump telnet traceroute iotop unzip zip make
ADD nginx-1.22.0.tar.gz /usr/local/src/
RUN cd /usr/local/src/nginx-1.22.0 && ./configure --prefix=/apps/nginx && make && make install && ln -sv /apps/nginx/sbin/nginx /usr/bin
RUN groupadd -g 2088 nginx && useradd -g nginx -s /usr/sbin/nologin -u 2088 nginx && chown -R nginx.nginx /apps/nginx
ADD nginx.conf /apps/nginx/conf/
ADD frontend.tar.gz /apps/nginx/html/
EXPOSE 80 443
#ENTRYPOINT ["nginx"]
CMD ["nginx","-g","daemon off;"]
效果
- 下载cni
自行安装
root@k8s-master1:~# mkdir -p /etc/nerdctl
root@k8s-master1:~# vim /etc/nerdctl/nerdctl.toml
#这里注意这个namespace 指定了哪个命令空间这个镜像就在哪里 否则你运行的时候不找不到镜像 找不到的话你可以先注释掉这个参数 运行起来在加回去 添加这个namespace就能看到k8s的镜像了
root@k8s-master1:~# cat /etc/nerdctl/nerdctl.toml
namespace = "k8s.io"
root@k8s-master1:/usr/local/src# mkdir -p /opt/cni/bin
root@k8s-master1:/usr/local/src# tar xf cni-plugins-linux-amd64-v1.1.1.tgz -C /opt/cni/bin
root@k8s-master1:/usr/local/src# root@k8s-master1:/etc/nerdctl# nerdctl run --name nginx -p 80:80 harbor.nbrhce.com/image/nginx-base:1.22.0
21.3 业务容器构建规划
#由上述所示nerdctl来官方的仓库拉取一些官方镜像作为父镜像然后把父镜像上传到harbor中这样是可以的 但是由于是自己签发的证书，在父镜像的基础之上在构建镜像的话会有报错，拉取不到元数据信息，因为目前的harbor是https的是有证书的，会有一系列证书的报错估计是证书不信任这种，但是你的证书要是花钱的买的能被信任的就不会就这个问题，或者就在运行一个nginx服务，把证书放在nginx上面然后把harbor换成http协议的这样也是可以的。
- 业务容器化优势
业务容器化优势
1.提高资源利用率，节约部署IT成本
2.提高部署效率，基于kubernetes实现微服务的快速部署与交付、容器的批量调度与秒级启动
3.实现横向扩容、灰度部署、回滚、链路追踪、服务治理等
4.可根据业务负载进行自动弹性伸缩
5.容器将环境和代码打包在镜像内，保证了测试与生产运行环境的一致性
6.紧跟云原生社区技术发展的步伐，不给公司遗留技术债，为后期技术升级务实基础
7.为个人储备前沿技术，提供个人level
21.4 实现nginx代理habor
- 操作如下
root@deploy-harbor:/apps/harbor# docker-compose stop
Stopping harbor-jobservice ... done
Stopping harbor-core ... done
Stopping registry ... done
Stopping redis ... done
Stopping harbor-db ... done
Stopping harbor-portal ... done
Stopping registryctl ... done
Stopping harbor-log ... done
#关闭https 使用http
root@deploy-harbor:/apps/harbor# vim harbor.yml
#更新配置文件
root@deploy-harbor:/apps/harbor# ./prepare
#更新之后看这里有没有nginx配置文件
root@deploy-harbor:/apps/harbor# ll common/config/portal/nginx.conf
#然后在运行harbor服务查看是否是http访问
root@deploy-harbor:/apps/harbor# docker-compose up -d
- 安装nginx 实现代理
root@k8s-master1:~# wget https://nginx.org/download/nginx-1.22.0.tar.gz
root@k8s-master1:~# tar xf nginx-1.22.0.tar.gz
root@k8s-master1:~# cd nginx-1.22.0
root@k8s-master1:~# ./configure --prefix=/apps/nginx \
--with-http_ssl_module \
--with-http_v2_module \
--with-http_realip_module \
--with-http_stub_status_module \
--with-http_gzip_static_module \
--with-pcre \
--with-stream \
--with-stream_ssl_module \
--with-stream_realip_module
root@k8s-master1:~# make && make install
root@k8s-master1:~# mkdir /apps/nginx/certs -p
#把harbor的证书拷贝过来
root@k8s-master1:~# ls /apps/nginx/certs/
harbor.nbrhce.com.crt harbor.nbrhce.com.key
#编辑配置文件 这样写是访问http或者是https都可以，最好还是http怕遇到证书不信任
root@k8s-master1:~# cat /etc/nginx/conf.d/nginx.conf
#http区域写入这个文件上传限制
client_max_body_size 2048m;
server {
listen 80;
listen 443 ssl;
server_name harbor.nbrhce.com;
ssl_certificate /apps/nginx/certs/harbor.nbrhce.com.crt;
ssl_certificate_key /apps/nginx/certs/harbor.nbrhce.com.key;
ssl_session_cache shared:sslcache:20m;
ssl_session_timeout 10m;
location / {
proxy_pass http://192.168.1.75;
}
}
#设置好域名解析 这个habor域名一定要指定在harbor哪台机器上 否则会遇到跳转404
root@k8s-master1:~# cat /etc/hosts
192.168.1.70 k8s-master1
192.168.1.71 k8s-etcd1
192.168.1.72 k8s-etcd2
192.168.1.73 k8s-etcd3
192.168.1.74 k8s-node1
192.168.1.75 k8s-deploy-harbor harbor.nbrhce.com
192.168.1.75 easzlab.io.local
#buildkitd设置信任http 没有目录的话就创建
root@k8s-master1:~# cat /etc/buildkit/buildkitd.toml
[registry."harbor.nbrhce.com"]
http = true
insecure = true
#nerdctl设置构建过程详细信息
root@k8s-master1:~# cat /etc/nerdctl/nerdctl.toml
namespace = "k8s.io"
debug = false
debug_full = false
insecure_registry = true
#构建过程 要是别的节点想使用 需要安装nerdctl cni 分发证书 做域名解析 配置buildkit nerdctl
root@k8s-master1:~/k8s-data/dockerfile/web/pub-images/jdk-1.8.212# nerdctl build -t harbor.nbrhce.com/pub-image/jdk-base:v8.212 .
#上传
root@k8s-master1:~# nerdctl push harbor.nbrhce.com/pub-image/jdk-base:v8.212
- 问题记录 没有识别到自签证书 但是我重启buildkitd服务之后就好使了 不是我证书的原因
21.5 案例nginx toncat前后端分离加业务
- 父镜像
nerdctl pull centos:7.9.2009
nerdctl tag centos:7.9.2009 harbor.nbrhce.com/baseimages/centos:7.9.2009
- 子镜像
root@k8s-master1:~/k8s-data/dockerfile/web/pub-images/jdk-1.8.212# cat Dockerfile
#JDK Base Image
FROM harbor.nbrhce.com/baseimages/centos:7.9.2009
#FROM centos:7.9.2009
MAINTAINER auth= this is tomcat image
ADD jdk-8u212-linux-x64.tar.gz /usr/local/src/
RUN ln -sv /usr/local/src/jdk1.8.0_212 /usr/local/jdk
ADD profile /etc/profile
ENV JAVA_HOME /usr/local/jdk
ENV JRE_HOME $JAVA_HOME/jre
ENV CLASSPATH $JAVA_HOME/lib/:$JRE_HOME/lib/
ENV PATH $PATH:$JAVA_HOME/bin
- 基础镜像
root@k8s-master1:~/k8s-data/dockerfile/web/pub-images/tomcat-base-8.5.43# cat Dockerfile
#Tomcat 8.5.43基础镜像
FROM harbor.nbrhce.com/pub-image/jdk-base:v8.212
MAINTAINER auth= this is tomcat image
RUN mkdir /apps /data/tomcat/webapps /data/tomcat/logs -pv
ADD apache-tomcat-8.5.43.tar.gz /apps
RUN useradd tomcat -u 2050 && ln -sv /apps/apache-tomcat-8.5.43 /apps/tomcat && chown -R tomcat.tomcat /apps /data -R
- 业务镜像
root@k8s-master1:~/k8s-data/dockerfile/web/magedu/tomcat-app1# cat Dockerfile
#tomcat web1
FROM harbor.nbrhce.com/pub-image/tomcat-base:v8.5.43
ADD catalina.sh /apps/tomcat/bin/catalina.sh
ADD server.xml /apps/tomcat/conf/server.xml
#ADD myapp/* /data/tomcat/webapps/myapp/
ADD app1.tar.gz /data/tomcat/webapps/myapp/
ADD run_tomcat.sh /apps/tomcat/bin/run_tomcat.sh
#ADD filebeat.yml /etc/filebeat/filebeat.yml
RUN useradd nginx && chown -R nginx.nginx /data/ && chown -R nginx.nginx /apps/
#ADD filebeat-7.5.1-x86_64.rpm /tmp/
#RUN cd /tmp && yum localinstall -y filebeat-7.5.1-amd64.deb
EXPOSE 8080 8443
CMD ["/apps/tomcat/bin/run_tomcat.sh"]
root@k8s-master1:~/k8s-data/dockerfile/web/magedu/tomcat-app1# cat build-command.sh
#!/bin/bash
TAG=$1
#docker build -t harbor.nbrhce.com/demo/tomcat-app1:${TAG} .
#sleep 3
#docker push harbor.nbrhce.com/demo/tomcat-app1:${TAG}
nerdctl build -t harbor.nbrhce.com/demo/tomcat-app1:${TAG} .
nerdctl push harbor.nbrhce.com/demo/tomcat-app1:${TAG}
- 测试运行业务
root@k8s-master1:~/k8s-data/dockerfile/web/magedu/tomcat-app1# nerdctl run --rm -p 8080:8080 harbor.nbrhce.com/demo/tomcat-app1:v1
Tomcat started.
- tomcat运行在k8s上
root@k8s-node1:~/k8s-data/yaml/magedu/tomcat-app1# cat tomcat-app1.yaml
kind: Deployment
#apiVersion: extensions/v1beta1
apiVersion: apps/v1
metadata:
labels:
app: magedu-tomcat-app1-deployment-label
name: magedu-tomcat-app1-deployment
namespace: magedu
spec:
replicas: 1
selector:
matchLabels:
app: magedu-tomcat-app1-selector
template:
metadata:
labels:
app: magedu-tomcat-app1-selector
spec:
containers:
- name: magedu-tomcat-app1-container
image: harbor.nbrhce.com/demo/tomcat-app1:v1
#command: ["/apps/tomcat/bin/run_tomcat.sh"]
imagePullPolicy: IfNotPresent
#imagePullPolicy: Always
ports:
- containerPort: 8080
protocol: TCP
name: http
env:
- name: "password"
value: "123456"
- name: "age"
value: "18"
#resources:
# limits:
# cpu: 1
# memory: "512Mi"
# requests:
# cpu: 500m
# memory: "512Mi"
volumeMounts:
- name: magedu-images
mountPath: /usr/local/nginx/html/webapp/images
readOnly: false
- name: magedu-static
mountPath: /usr/local/nginx/html/webapp/static
readOnly: false
volumes:
- name: magedu-images
nfs:
server: 192.168.1.75
path: /data/k8sdata/images
- name: magedu-static
nfs:
server: 192.168.1.75
path: /data/k8sdata/static
# nodeSelector:
# project: magedu
# app: tomcat
---
kind: Service
apiVersion: v1
metadata:
labels:
app: magedu-tomcat-app1-service-label
name: magedu-tomcat-app1-service
namespace: magedu
spec:
type: NodePort
ports:
- name: http
port: 80
protocol: TCP
targetPort: 8080
nodePort: 30092
selector:
app: magedu-tomcat-app1-selector
root@k8s-node1:~/k8s-data/yaml/magedu/tomcat-app1#
- nginx父镜像
root@k8s-master1:~/k8s-data/yaml/magedu/tomcat-app1# nerdctl pull centos:7.9.2009
root@k8s-master1:~/k8s-data/yaml/magedu/tomcat-app1# nerdctl tag centos:7.9.2009 harbor.nbrhce.com/baseimages/nbrhce-centos-base:7.9.2009
root@k8s-master1:~/k8s-data/yaml/magedu/tomcat-app1# nerdctl push harbor.nbrhce.com/baseimages/nbrhce-centos-base:7.9.2009
- 基础镜像
root@k8s-node1:~/k8s-data/dockerfile/web/pub-images/nginx-base# cat Dockerfile
#Nginx Base Image
FROM harbor.nbrhce.com/baseimages/nbrhce-centos-base:7.9.2009
MAINTAINER zhangshijie@magedu.net
RUN yum install -y vim wget tree lrzsz gcc gcc-c++ automake pcre pcre-devel zlib zlib-devel openssl openssl-devel iproute net-tools iotop
ADD nginx-1.22.0.tar.gz /usr/local/src/
RUN cd /usr/local/src/nginx-1.22.0 && ./configure && make && make install && ln -sv /usr/local/nginx/sbin/nginx /usr/sbin/nginx &&rm -rf /usr/local/src/nginx-1.22.0.tar.gz
root@k8s-node1:~/k8s-data/dockerfile/web/pub-images/nginx-base# cat build-command.sh
#!/bin/bash
#docker build -t harbor.nbrhce.com/pub-images/nginx-base:v1.18.0 .
#sleep 1
#docker push harbor.nbrhce.com/pub-images/nginx-base:v1.18.0
nerdctl build -t harbor.nbrhce.com/pub-image/nginx-base:v1.22.0 .
nerdctl push harbor.nbrhce.com/pub-image/nginx-base:v1.22.0
- 业务镜像
root@k8s-node1:~/k8s-data/dockerfile/web/magedu/nginx# cat Dockerfile
#Nginx 1.22.0
FROM harbor.nbrhce.com/pub-image/nginx-base:v1.22.0
ADD nginx.conf /usr/local/nginx/conf/nginx.conf
ADD app1.tar.gz /usr/local/nginx/html/webapp/
ADD index.html /usr/local/nginx/html/index.html
#静态资源挂载路径
RUN mkdir -p /usr/local/nginx/html/webapp/static /usr/local/nginx/html/webapp/images && useradd nginx
EXPOSE 80 443
CMD ["nginx"]
- nginx配置文件 匹配了一个url
user nginx nginx;
worker_processes auto;
#error_log logs/error.log;
#error_log logs/error.log notice;
#error_log logs/error.log info;
#pid logs/nginx.pid;
daemon off;
events {
worker_connections 1024;
}
http {
include mime.types;
default_type application/octet-stream;
#log_format main '$remote_addr - $remote_user [$time_local] "$request" '
# '$status $body_bytes_sent "$http_referer" '
# '"$http_user_agent" "$http_x_forwarded_for"';
#access_log logs/access.log main;
sendfile on;
#tcp_nopush on;
#keepalive_timeout 0;
keepalive_timeout 65;
#gzip on;
upstream tomcat_webserver {
server magedu-tomcat-app1-service.magedu.svc.magedu.local:80;
}
server {
listen 80;
server_name localhost;
#charset koi8-r;
#access_log logs/host.access.log main;
location / {
root html;
index index.html index.htm;
}
location /webapp {
root html;
index index.html index.htm;
}
- nginx-k8s-YAML
root@k8s-node1:~/k8s-data/yaml/magedu/nginx# cat nginx.yaml
kind: Deployment
apiVersion: apps/v1
metadata:
labels:
app: magedu-nginx-deployment-label
name: magedu-nginx-deployment
namespace: magedu
spec:
replicas: 1
selector:
matchLabels:
app: magedu-nginx-selector
template:
metadata:
labels:
app: magedu-nginx-selector
spec:
containers:
- name: magedu-nginx-container
image: harbor.nbrhce.com/demo/nginx-web1:v1
#command: ["/apps/tomcat/bin/run_tomcat.sh"]
imagePullPolicy: IfNotPresent
# imagePullPolicy: Always
ports:
- containerPort: 80
protocol: TCP
name: http
- containerPort: 443
protocol: TCP
name: https
env:
- name: "password"
value: "123456"
- name: "age"
value: "20"
# resources:
# limits:
# cpu: 500m
# memory: 512Mi
# requests:
# cpu: 500m
# memory: 256Mi
volumeMounts:
- name: magedu-images
mountPath: /usr/local/nginx/html/webapp/images
readOnly: false
- name: magedu-static
mountPath: /usr/local/nginx/html/webapp/static
readOnly: false
volumes:
- name: magedu-images
nfs:
server: 192.168.1.75
path: /data/k8sdata/images
- name: magedu-static
nfs:
server: 192.168.1.75
path: /data/k8sdata/static
#nodeSelector:
# group: magedu
---
kind: Service
apiVersion: v1
metadata:
labels:
app: magedu-nginx-service-label
name: magedu-nginx-service
namespace: magedu
spec:
type: NodePort
ports:
- name: http
port: 80
protocol: TCP
targetPort: 80
nodePort: 30090
- name: https
port: 443
protocol: TCP
targetPort: 443
nodePort: 30091
selector:
app: magedu-nginx-selector
