openshift4.7安装手册

OCP4版本安装（虚拟机安装版）：
1. 服务器准备：
  1.1 准备7台服务器，每台服务器的规划如下：
    bastion：1台，centos系统
      192.168.145.181，安装必要的辅助工具，如DNS、HTTP、LB、HARBOR等
    bootstrap：1台，无需事先安装系统，在安装OCP4的过程中，通过iso镜像安装
      192.168.145.182，安装引导节点，会临时创建一个K8S集群，负责引导OCP集群的安装，等OCP安装完成后，此节点可以删除
    master：3台，无需事先安装系统，在安装OCP4的过程中，通过iso镜像安装
      192.168.145.183
      192.168.145.184
      192.168.145.185
    worker：2台，无需事先安装系统，在安装OCP4的过程中，通过iso镜像安装
      192.168.145.186
      192.168.145.187

  1.2 修改bastion的机器主机名（示例：bastion.ocp4.liufeng.cc）

2. LB、DNS、HARBOR、HTTP的准备。本篇是所有服务均安装在bastion机器上，资源允许的话，也可以安装在不同的机器上。
  2.1 准备LB，使用haproxy实现
    2.1.1 安装haproxy
      # yum install haproxy

    2.1.2 配置负载均衡器，把如下配置追加到haproxy.cfg文件后面。
      frontend openshift-api-server              
          bind *:6443
          default_backend openshift-api-server
          mode tcp
          option tcplog
    
      backend openshift-api-server
          balance source
          mode tcp
          server bootstrap 192.168.145.182:6443 check  
          server master1 192.168.145.183:6443 check 
          server master2 192.168.145.184:6443 check
          server master3 192.168.145.185:6443 check
            
      frontend machine-config-server          
          bind *:22623
          default_backend machine-config-server
          mode tcp
          option tcplog
      
      backend machine-config-server
          balance source
          mode tcp
          server bootstrap 192.168.145.182:22623 check 
          server master1 192.168.145.183:22623 check   
          server master2 192.168.145.184:22623 check  
          server master3 192.168.145.185:22623 check

    2.1.3 启动haproxy并设置开机启动
      # systemctl start haproxy
      # systemctl enable haproxy
      # systemctl status haproxy
      如果haproxy没有启动，运行下面的命令后，再次启动haproxy
      # setsebool -P haproxy_connect_any=1

    2.1.4 开通防火墙以便可以访问到
      # firewall-cmd --add-port=6443/tcp --permanent
      # firewall-cmd --add-port=22623/tcp --permanent
      # firewall-cmd --reload
      # firewall-cmd --list-all

  2.2 准备DNS，使用dnsmasq实现
    2.2.1 安装dnsmasq
      # yum install dnsmasq 

    2.2.2 配置dns解析
      # ocp4 node
      address=/master1.ocp4.liufeng.cc/192.168.145.183
      address=/master2.ocp4.liufeng.cc/192.168.145.184
      address=/master3.ocp4.liufeng.cc/192.168.145.185
      address=/worker1.ocp4.liufeng.cc/192.168.145.186
      address=/worker2.ocp4.liufeng.cc/192.168.145.187
      
      # etcd
      address=/etcd-0.ocp4.liufeng.cc/192.168.145.183
      address=/etcd-1.ocp4.liufeng.cc/192.168.145.184
      address=/etcd-2.ocp4.liufeng.cc/192.168.145.185
      # etcd srv 
      # <name>,<target>,<port>,<priority>,<weight>
      srv-host=_etcd-server-ssl._tcp.ocp4.liufeng.cc,etcd-0.ocp4.liufeng.cc,2380,0,10
      srv-host=_etcd-server-ssl._tcp.ocp4.liufeng.cc,etcd-1.ocp4.liufeng.cc,2380,0,10
      srv-host=_etcd-server-ssl._tcp.ocp4.liufeng.cc,etcd-2.ocp4.liufeng.cc,2380,0,10
      
      # lb
      address=/.ocp4.liufeng.cc/192.168.145.186
      address=/api.ocp4.liufeng.cc/192.168.145.181
      address=/api-int.ocp4.liufeng.cc/192.168.145.181
      
      # other
      address=/bootstrap.ocp4.liufeng.cc/192.168.145.182
      address=/bastion.ocp4.liufeng.cc/192.168.145.181
      address=/harbor.ocp4.liufeng.cc/192.168.145.181

    2.2.3 启动dnsmasq并设置开机自启
      # systemctl start dnsmasq
      # systemctl enable dnsmasq

    2.2.4 防火墙及设定
      # firewall-cmd --add-port=53/tcp --permanent
      # firewall-cmd --add-port=53/udp --permanent
      # firewall-cmd --reload
      # firewall-cmd --list-all

    2.2.5 验证dns是否生效，例如：
      # dig +short -t A etcd-0.ocp4.liufeng.cc @192.168.145.181
      # dig +short -t SRV _etcd-server-ssl._tcp.ocp4.liufeng.cc @192.168.145.181 
      如果没有dig命令，请使用如下命令安装
      # yum install bind-utils

  2.3 Harbor、http服务器的准备
    2.3.1 Harbor的安装，使用https访问，http的访问留着给http服务器使用。
      见Harbor安装文档

    2.3.2 Harbor安装完成之后，就使用Harbor自带的nginx作为http服务器。
      2.3.2.1 修改docker-compose.yml文件，proxy的volumes部分，就是加一个映射（这里是把主机的/home/www目录映射成nginx容器的/var/www/html目录）：
        proxy:
          image: goharbor/nginx-photon:v2.1.3
          container_name: nginx
          restart: always
          cap_drop:
            - ALL
          cap_add:
            - CHOWN
            - SETGID
            - SETUID
            - NET_BIND_SERVICE
          volumes:
            - ./common/config/nginx:/etc/nginx:z
            - /home/harbor/data/secret/cert:/etc/cert:z
            - /home/www:/var/www/html:z
            - type: bind
              source: ./common/config/shared/trust-certificates
              target: /harbor_cust_cert
          networks:
            - harbor
          dns_search: .
          ports:
            - 80:8080
            - 443:8443
          depends_on:
            - registry
            - core
            - portal
            - log
          logging:
            driver: "syslog"
            options:
              syslog-address: "tcp://127.0.0.1:1514"
              tag: "proxy"

      2.3.2.2 修改nginx.conf
        在harbor目录中寻找到nginx的配置文件：common/config/nginx/nginx.conf
        修改如下server段，注释掉308跳转，并加一个root目录
          server {
              listen 8080;
              #server_name harbordomain.com;
              #return 308 https://$host:443$request_uri;
              root /var/www/html;
          }

      2.3.2.3 开通防火墙并验证harbor与nginx是否正常
        # firewall-cmd --add-port=443/tcp --permanent
        # firewall-cmd --add-port=80/tcp --permanent
        # firewall-cmd --reload
        # docker-compose down
        # docker-compose up -d
        # systemctl enable docker

3. 同步ocp4的镜像
  3.1 安装同步工具，也就是oc客户端
    # wget https://mirror.openshift.com/pub/openshift-v4/clients/oc/latest/linux/oc.tar.gz
    # tar xvf oc.tar.gz
    # mv kubectl oc /usr/local/bin/

  3.2 创建pull-secret.json
    3.2.1 创建私有仓库的secret信息
      # echo -n 'admin:Harbor12345' | base64 -w0    //对harbor登录信息进行base64加密，示例：YWRtaW46SGFyYm9yMTIzNDU=

    3.2.2 从官网下载pull-secret，地址：https://cloud.redhat.com/openshift/install/pull-secret,下载是一个txt文件，需要转换为json文件。
      # cat pull-secret.txt | jq . > pull-secret.json
      如果没有jq命令，请安装（需要epel源）
      # yum install jq

    3.2.3 合并pull-secret.json文件
      把上面的私有仓库的信息也添加到pull-secret.json中。下载此文件似乎要redhat的账号，那就注册一个吧！免费的。其实如果安装openshift社区版okd的话，可以不用下载，以后再表。
      合并后的json文件类似如下：
        {
          "auths": {
            "harbor.ocp4.liufeng.cc": {
              "auth": "YWRtaW46SGFyYm9yMTIzNDU="
              "email": ""
            },
            "cloud.openshift.com": {
              "auth": "b3BlbnNoaWZ0LXJlbGVhc2UtZGV2K29jbV9hY2Nlc3NfYTdmNGQ1MjZiMGVlNDkwNzk2MmViZWRiZTE1ZjEwNTI6SVVFSExFTk9SNVdQVVc4QldUT1k2VVlSMlc2V0xMQTQwNDA5UTRJRzNBRDRHS0lXR0NGTzJaN0dXOTJTMzIzMg==",
              "email": "lf_30y@163.com"
            },
            ……
          }
        }

  3.3 拉取镜像
    先在私有仓库建一个名为“openshift”的仓库（如下的openshift/ocp4.7，后面的ocp4.7就不需要手动创建了，会自动创建）。

    # export LOCAL_REGISTRY='harbor.ocp4.liufeng.cc'
    # export LOCAL_REPOSITORY='openshift/ocp4.7'
    # export PRODUCT_REPO='openshift-release-dev'
    # export RELEASE_NAME='ocp-release'
    # export OCP_RELEASE='4.7.0-fc.4'
    # export ARCHITECTURE='x86_64'
    # export LOCAL_SECRET_JSON='/root/pull-secret.json'
    # export GODEBUG='x509ignoreCN=0'
    # oc adm release mirror -a ${LOCAL_SECRET_JSON}   --from=quay.io/${PRODUCT_REPO}/${RELEASE_NAME}:${OCP_RELEASE}-${ARCHITECTURE} --to=${LOCAL_REGISTRY}/${LOCAL_REPOSITORY}   --to-release-image=${LOCAL_REGISTRY}/${LOCAL_REPOSITORY}:${OCP_RELEASE}-${ARCHITECTURE}

    ocp: https://quay.io/repository/openshift-release-dev/ocp-release?tab=tags
    okd: https://quay.io/repository/openshift/okd?tab=tags
    这两个地址是同步镜像的源仓库，对照oc adm release mirror命令，可以推出上述几个export的变量的值，同步完成之后，会显示类似下面的信息，请保存下来，后面要用到。
      Success
      Update image:  harbor.ocp4.liufeng.cc/openshift/ocp4.7:4.7.0-fc.4-x86_64
      Mirror prefix: harbor.ocp4.liufeng.cc/openshift/ocp4.7

      To use the new mirrored repository to install, add the following section to the install-config.yaml:

      imageContentSources:
      - mirrors:
        - harbor.ocp4.liufeng.cc/openshift/ocp4.7
        source: quay.io/openshift-release-dev/ocp-release
      - mirrors:
        - harbor.ocp4.liufeng.cc/openshift/ocp4.7
        source: quay.io/openshift-release-dev/ocp-v4.0-art-dev


      To use the new mirrored repository for upgrades, use the following to create an ImageContentSourcePolicy:

      apiVersion: operator.openshift.io/v1alpha1
      kind: ImageContentSourcePolicy
      metadata:
        name: example
      spec:
        repositoryDigestMirrors:
        - mirrors:
          - harbor.ocp4.liufeng.cc/openshift/ocp4.7
          source: quay.io/openshift-release-dev/ocp-release
        - mirrors:
          - harbor.ocp4.liufeng.cc/openshift/ocp4.7
          source: quay.io/openshift-release-dev/ocp-v4.0-art-dev
      
  3.4 查看同步结果：
    # curl -s -u admin:Harbor12345 -k https://harbor.ocp4.liufeng.cc/v2/openshift/ocp4.7/tags/list|jq .

4. 生成openshift-install安装文件
    # oc adm release extract -a ${LOCAL_SECRET_JSON} --command=openshift-install "${LOCAL_REGISTRY}/${LOCAL_REPOSITORY}:${OCP_RELEASE}-${ARCHITECTURE}" [--skip-verification=true --insecure=true]    
    因为要校验一致性，所以不要使用下载的openshift-install，而是用上面的命令会生成openshift-install文件，生成后的openshift-install文件拷贝到path路径下。
  
5. 准备安装文件
  5.1 因为coreos的默认用户是core，所以要准备core用户的ssh key
    # ssh-keygen -t rsa -b 4096 -N '' -f ~/.ssh/core_rsa
    # eval "$(ssh-agent -s)"
    # ssh-add ~/.ssh/core_rsa
      
  5.2 准备install-config.yaml文件
    # mkdir -pv ~/ocp4/ocp4install
    # cd ~/ocp4/ocp4install
      准备一个干净的空的文件夹（例如ocp4-install），在这个文件夹内创建install-config.yaml文件，详细内容见后面的：install-config.yaml
      5.2.1 metadata.name + baseDomain 即为集群名称 
      5.2.2 compute.replicas 设置为0
      5.2.3 pullSecret 之前拉取用的json文件再转换成txt的格式即可
      5.2.4 sshKey 为将来要ssh到集群所用的pub文件，即上面生成的core_rsa.pub文件
      5.2.5 additionalTrustBundle 为前一步骤安装harbor时本地生成的crt文件，注意缩进2格
      5.2.6 imageContentSources 为同步私服镜像后，最后生成出来的内容

  5.3 生成ign文件
    首先备份install-config.yaml文件，因为使用下面的命令会删除掉install-config.yaml。
    # openshift-install create manifests --dir=/root/ocp4/ocp4install
    # openshift-install create ignition-configs --dir=/root/ocp4/ocp4install
    最终ocp4install文件夹生成如下文件：
    .
    ├── auth
    │   ├── kubeadmin-password
    │   └── kubeconfig
    ├── bootstrap.ign
    ├── master.ign
    ├── metadata.json
    └── worker.ign
    【注意：从生成这个文件开始，24小时内必须完成ocp集群的安装！！！】

  5.4 上传文件到http服务器
    把上述生成的.ign文件上传，让其可以通过http访问到。
    # cd ~/ocp4/ocp4install
    # cp *.ign /home/www/
    # chmod +r /home/www/*.ign

6. 创建rhcos镜像
  6.1 RHCOS镜像下载地址：https://mirror.openshift.com/pub/openshift-v4/dependencies/rhcos/4.6/latest/，如果可用，请使用与 OpenShift Container Platform 版本匹配的镜像版本。没有的话，下载最高版本的镜像，其版本号应小于或等于您安装的 OpenShift Container Platform 版本。iso文件名类似：rhcos-<version>-live.<architecture>.iso

  6.2 用虚拟机启动下载的rhcos的iso，再引导启动前按Tab，加入启动参数：
    rd.neednet=1 ip=192.168.145.182::192.168.145.254:255.255.255.0:bootstrap.ocp4.liufeng.cc:eth0:none nameserver=192.168.145.181 coreos.inst.install_dev=/dev/xvda coreos.inst.ignition_url=http://192.168.145.181/bootstrap.ign
    6.2.1 rd.neednet=1，需要网络信息
    6.2.2 ip=，后面的格式是“IP地址::网关:子网掩码:完整主机名:网卡:none 
    6.2.3 nameserver=，DNS服务器地址，可以添加多个
    6.2.4 coreos.inst.install_dev=，安装到本地哪个磁盘
    6.2.5 coreos.inst.ignition_url=，ign文件的url，有三种ign文件bootstrap、master、worker，注意区分

7. 调试
  7.1 在bastion节点上，执行如下命令查看进度：
    # openshift-install --dir=/root/ocp4/ocp4install wait-for bootstrap-complete --log-level=debug
    # openshift-install --dir=/root/ocp4/ocp4install wait-for install-complete --log-level=debug

  7.2 在bastion节点上，使用oc命令：
    没有意外的话，执行下面的命令，可执行oc
    # export KUBECONFIG=/root/ocp4/ocp4install/auth/kubeconfig
    # oc get nodes
    # oc get ns
    # oc get pods --all-namespaces

8. oc的补全命令：
  # yum install bash-completion
  # oc completion bash > ~/.kube/completion.bash.inc
  在~/.bash_profile里添加：source '/root/.kube/completion.bash.inc'

9. 创建用户（使用htpasswd）
  9.1 创建htpass-secret
    # htpasswd -c -B -b users.htpasswd admin liufeng.cc0021    //第一个用户
    # htpasswd -b -B users.htpasswd liufeng 8888.8888       //添加后续用户 
    # oc create secret generic htpass-secret --from-file=htpasswd=</path/to/users.htpasswd> -n openshift-config

  9.2 创建HTPasswd CR，新建一个文件（假设为htpasswd-cr.yaml），保存如下yaml：
    apiVersion: config.openshift.io/v1
    kind: OAuth
    metadata:
      name: cluster
    spec:
      identityProviders:
      - name: my_htpasswd_provider 
        mappingMethod: claim 
        type: HTPasswd
        htpasswd:
          fileData:
            name: htpass-secret

  9.3 应用HTPasswd CR及授权
    # oc apply -f htpasswd-cr.yaml
    # oc adm policy add-cluster-role-to-user cluster-admin admin

  9.4 登录集群
    # oc login -u <username>
    # oc whoami

install-config.yaml内容如下（请注意格式与缩进）：

apiVersion: v1
baseDomain: liufeng.cc
compute:
- hyperthreading: Enabled  
  name: worker
  replicas: 0 
controlPlane:
  hyperthreading: Enabled  
  name: master 
  replicas: 3 
metadata:
  name: ocp4
networking:
  clusterNetwork:
  - cidr: 10.128.0.0/14 
    hostPrefix: 23 
  networkType: OpenShiftSDN
  serviceNetwork: 
  - 172.30.0.0/16
platform:
  none: {} 
fips: false
pullSecret: '{"auths":{"harbor.ocp4.liufeng.cc":{"auth":"YWRtaW46SGFyYm9yMTIzNDU=","email":""},"quay.io":{"auth":"b3BlbnNoaWZ0LXJlbGVhc2UtZGV2K29jbV9hY2Nlc3NfYTdmNGQ1MjZiMGVlNDkwNzk2MmViZWRiZTE1ZjEwNTI6SVVFSExFTk9SNVdQVVc4QldUT1k2VVlSMlc2V0xMQTQwNDA5UTRJRzNBRDRHS0lXR0NGTzJaN0dXOTJTMzIzMg==","email":"lf_30y@163.com"},"registry.connect.redhat.com":{"auth":"NTMyMTk5MTJ8dWhjLTFmUXpOTjRWWHhaQ29OT0ZpR2QyOGVvYW9YSDpleUpoYkdjaU9pSlNVelV4TWlKOS5leUp6ZFdJaU9pSXlNVEkyTm1Kak4ySTJZamswTTJFd1lXRTVNVEl4TXpaa1kyUmhaVEl6TXlKOS53WHptUDhZT0ZPelh2TzQ5RXNCMlA3Q0VDc3ktRFctWG5CcWtTMXc5MkJSWU0wSzU1RDhHS3FwLW1VOFRMUk5Fa01Ja3dRWE5fRkdSY1Q3eEtqd2Z2SDVLTVNjVkpCdUpYSmtoS1gtdjEwRmZ6ZXF6U2tOZ2FsUFM4VnpXWi1WRG9iaXViQWtjSEYyeXotLUVGUVpGeEktbXZkYklZTkt5dXZSVkZQUW56cUJ5Q0s2d0pRMXI2MWRFRUxyRnU3a25lVFlZSU9wWXR0V21QTG5rU21FUjNrei14S2FGcG4zZFFhMG9rUUVDZlFHRVBVbFlvNkZqSTk3R3Z1Uk9NTmFqckxwUE1GSV9QZ21oeGd2OHFqVHg3M0hsOE9rWFlOSkRCSUMwN1dNcWN4ZHBfM3ZhMmRVdm5JUUo4X2o0OGt0dFpScTNVQWZMMkVRUXpQMDZIanJWVlgyWTZkWEhYVWRHWFZPem9JckVLcHBTQTBDRExCM1dkOUU1YVpHRmF5a21YOTZOSmdWWnB5enVDcW54WWZIWWM3WDliLXl2UkdCSGtrNzdxdTBJNGowa2JqUWZvelJNS0VVcUo0VFgtVUtNX3ZqLUE5b09Tek04bWJEa3JOWEFMM0NzOHFGc2ZBRTVPUDFYdlIyRDRDck81eWdEZzVVMks2dHYyajktRlE3Zmx1eTg4b0ZWUEZybG5IbTNNX1I2ZXBocXFZZVZSME1hMEN6bHNUZzJLUjBRSHZvN3ZvaWRsaG1NcHdFWkJWalJPR3p0a1FPcUdqdnZMc0ZxZWpsOGRsb3I1MFFtUFltd2E0aTFrdU5wY0l5ZzhZeGU3ODFhYUtMbjFldzZXVC1veWtTbjhNaF9RYWM2SHExdkpDTWcyM1BDVzU2M1VNV3FhU2ZRbnhhWFdxTQ==","email":"lf_30y@163.com"},"registry.redhat.io":{"auth":"NTMyMTk5MTJ8dWhjLTFmUXpOTjRWWHhaQ29OT0ZpR2QyOGVvYW9YSDpleUpoYkdjaU9pSlNVelV4TWlKOS5leUp6ZFdJaU9pSXlNVEkyTm1Kak4ySTJZamswTTJFd1lXRTVNVEl4TXpaa1kyUmhaVEl6TXlKOS53WHptUDhZT0ZPelh2TzQ5RXNCMlA3Q0VDc3ktRFctWG5CcWtTMXc5MkJSWU0wSzU1RDhHS3FwLW1VOFRMUk5Fa01Ja3dRWE5fRkdSY1Q3eEtqd2Z2SDVLTVNjVkpCdUpYSmtoS1gtdjEwRmZ6ZXF6U2tOZ2FsUFM4VnpXWi1WRG9iaXViQWtjSEYyeXotLUVGUVpGeEktbXZkYklZTkt5dXZSVkZQUW56cUJ5Q0s2d0pRMXI2MWRFRUxyRnU3a25lVFlZSU9wWXR0V21QTG5rU21FUjNrei14S2FGcG4zZFFhMG9rUUVDZlFHRVBVbFlvNkZqSTk3R3Z1Uk9NTmFqckxwUE1GSV9QZ21oeGd2OHFqVHg3M0hsOE9rWFlOSkRCSUMwN1dNcWN4ZHBfM3ZhMmRVdm5JUUo4X2o0OGt0dFpScTNVQWZMMkVRUXpQMDZIanJWVlgyWTZkWEhYVWRHWFZPem9JckVLcHBTQTBDRExCM1dkOUU1YVpHRmF5a21YOTZOSmdWWnB5enVDcW54WWZIWWM3WDliLXl2UkdCSGtrNzdxdTBJNGowa2JqUWZvelJNS0VVcUo0VFgtVUtNX3ZqLUE5b09Tek04bWJEa3JOWEFMM0NzOHFGc2ZBRTVPUDFYdlIyRDRDck81eWdEZzVVMks2dHYyajktRlE3Zmx1eTg4b0ZWUEZybG5IbTNNX1I2ZXBocXFZZVZSME1hMEN6bHNUZzJLUjBRSHZvN3ZvaWRsaG1NcHdFWkJWalJPR3p0a1FPcUdqdnZMc0ZxZWpsOGRsb3I1MFFtUFltd2E0aTFrdU5wY0l5ZzhZeGU3ODFhYUtMbjFldzZXVC1veWtTbjhNaF9RYWM2SHExdkpDTWcyM1BDVzU2M1VNV3FhU2ZRbnhhWFdxTQ==","email":"lf_30y@163.com"}}}'
sshKey: 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC3Aav1hMwvu6tQUtAy9VVrot8HBgQ7fQ8uZlw5GvPnts31BLrEha4UvbXbZAF1eACcwYhKRIEDz9VAt63WiGX3FqgMO3l5E5Ekk5V8GQBbSf0TezNLYTiPEvZV7JDNlsF2vAhGyHrYmnElJriVxZn3KzwqeIz9l2aOXHi1o/t0FD4P6FbAjT3xB1x+EX5aaAeRT5uiIiPzW6MljZOoyyApriO6FsrblIiksx4MZazp63ubB20IKYh54Cg7TIzQ5XkR8DCXPcZ1Jm2+XnRyQaN9ydJf9hwTivCGLesnHDBTkZ/wolh3hAMlHg1wlvAwYhRC+Ukc+Ewqit/Z5MkRCDHqYFUJVJwrLZQElBpBmjMKLBKXj1lo1CplF77ubqutl+hs46UIu+Hc+elqdDxn03TOsBouuJVwuvCU1IaPUeLtkz0LxaAykpVog/5NopPNVWVB5htkyqTeONfEFeeUL6/Af8Pn7ARd6giLmuTggG8lCxWYnN+Y48pmc3adG8NavsjKfEuhFzbgMu7t3DF6Aqz3emxGaUfJ5pE8vNVjisP4ufOwTYvjckO1adui1XLAx6i0CDrQO17nyK3lrRQLeySZlQmg2elI2OaPqBLVfBQBuWGI6ECPAmBdjJ/9BcsPlboXOkan1UmJpeFBcAkiClO0kozz3NUvmkoXXxzjULqa3w== root@bastion.ocp4.baison.cc'
additionalTrustBundle: |
  -----BEGIN CERTIFICATE-----
  MIIFtzCCA5+gAwIBAgIJAJo2D89dAHnlMA0GCSqGSIb3DQEBCwUAMHExCzAJBgNV
  BAYTAkNOMREwDwYDVQQIDAhTaGFuZ2hhaTERMA8GA1UEBwwIU2hhbmdoYWkxDzAN
  BgNVBAoMBkJhaXNvbjELMAkGA1UECwwCaXQxHjAcBgNVBAMMFWhhcmJvci5vY3A0
  LmJhaXNvbi5jYzAgFw0yMTAxMzAxMTM2NDhaGA8yMTIxMDEwNjExMzY0OFowcTEL
  MAkGA1UEBhMCQ04xETAPBgNVBAgMCFNoYW5naGFpMREwDwYDVQQHDAhTaGFuZ2hh
  aTEPMA0GA1UECgwGQmFpc29uMQswCQYDVQQLDAJpdDEeMBwGA1UEAwwVaGFyYm9y
  Lm9jcDQuYmFpc29uLmNjMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA
  0V0po4o0ZskyvkzC4uhME+Pv5AZbnONRMkLLVRTMjEKNOnmyS84T5aN3EYGujfGl
  FLTyraSpRQNLA3PkJr7pEWaRljpTjOvrNxo3u819VDcFBOn9GEVprvEd5HaennBQ
  ip6BZrhHPIjv8uHs4TXSWfxPZMuX9gpg0bj5Icm+V3lHImTALvOlDXloDTokPlq9
  kX16ZB14AGA287w3p4B7S2a5+b109DTHLAyDGi8JMiLJYJb0Xf4fdv8K5qv5WTPl
  qyjjkVnLi8ka3TLVDXKxYTDtCkqMVp3MmPpyntBLkoiB2F7GVSwVruppf4F+TGJw
  gaSz3RGl4Mnpy3qgUjtZ4dgXsYL+Bpg3+LzJe22lyIejYEDCw/QKhzqfoxOIPiD8
  TAyjqH9nEZ2pYF0gBIrZDe1cCfpN/+cdBTwac96Ph45rkWkF+BVSqRV0g0ppTSMi
  BQ+x5c1WooUfVqtunqJ2rixapm+ASmbMG1aCP/5/18gE/pZQlX0cxOrZa9R8lZgZ
  rq3HLRScGqC1rhe/NDpV2zAx4bFNwKKLqPu6bLPkn7jpWcuQhHdZcCfvHojz1IPn
  C5zJPmu7D3HRubLcQ3AK97etyG+yS7Orrmwr2Ci84eqcZYy66fEoJA7a69kolFW1
  z6+0rQIrIrEjqsxsY5xTt8wbTN81LbyKal8ly819TtMCAwEAAaNQME4wHQYDVR0O
  BBYEFB0hnyLxArythRW0K/7/LaSr00HCMB8GA1UdIwQYMBaAFB0hnyLxArythRW0
  K/7/LaSr00HCMAwGA1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggIBAA8+l3YW
  MIpl3oeGsFSZ8n+HGNCcOZz6eiL4qx9m2pvCR2VJ4FYdFchEfX9hadXRi6o6p9Jo
  XNkaLsymlaLtU4eCgTUpiXz8v29zqGm+M+0OHr/EEqS3DoN6sfSEalH+KiKXA6sB
  C6BP0afnftR3TIjxmMEjfjOcsyaaOn/oJ6qstViP1M8vajKtYlKWvhg7cD4pOoCy
  1LyIeibBoHNFPI4qVhd43pPnTukeV61X5DZsEIuODKXvS7RFrpx4x5um58fogsP/
  tBfwrOhNAXSpO3p6OwdE9Zk/CQm4Irj7NIIB1sc5X0LreWLEQHIlGEfFfpMPKPEU
  fKOnfjx1k5pUDn8fpwqFGpMvn5qB+jHpMe4xeJy6L8ge2JygpEZd69EgdF3KwK8M
  szOkxXZNlUg9F3B4BhYMqHft6lI6yz6Vn+h5yOdDTiP33jytQDuLdwgiWD8MJXiu
  ta3pXc0/fpuPa5UKa75D9vyXZIPG340x/LprezwpYYR2inEu3a6OF65Nyi0FRsac
  L7lbhEtZdOX+ZkuhvL02+Cy3JyipibShyK9Z+aoMHR+1sv/0qjN8hhIo3kcMXtcG
  l722rM9IIawR9o5f/IJO9AgvHd1QwJdRY2ftvyf7cLLlbEnOK1K/YFdYzAciHzoD
  dpez8+4JZAi+6si63NSuVPh2ZcZRiIln+PEX
  -----END CERTIFICATE-----
imageContentSources:
- mirrors:
  - harbor.ocp4.liufeng.cc/openshift/ocp4.7
  source: quay.io/openshift-release-dev/ocp-release
- mirrors:
  - harbor.ocp4.liufeng.cc/openshift/ocp4.7
  source: quay.io/openshift-release-dev/ocp-v4.0-art-dev