企业级高可用Harbor 2.3

环境说明

主机名	IP	系统	用途
harbor01.ecloud.com	192.168.32.137	CentOS 7.6	安装harbor仓库、HTTPS
harbor02.ecloud.com	192.168.32.145	CentOS 7.6	安装harbor仓库、HTTPS
db.ecloud.com	192.168.32.147	CentOS 7.6	安装nfs、redis、postgresql

两台harbor仓库使用NFS文件系统来存储数据，使用外部的redis和postgresql来实现高可用。

安装NFS

停止防火墙

systemctl stop firewalld && systemctl disable firewalld

下载nfs-server包

yum install -y nfs-utils

配置nfs

mkdir -p /nfs/harbor
echo '/nfs/harbor 192.168.32.0/24(rw,no_root_squash)' >> /etc/exports

启动nfs

systemctl start nfs.service && systemctl enable nfs.service

安装redis

设置环境变量

echo 'net.core.somaxconn=511' >> /etc/sysctl.d/redis.conf
echo 'vm.overcommit_memory=1' >> /etc/sysctl.d/redis.conf
sysctl -p /etc/sysctl.d/redis.conf

ulimit -n 65535
echo '*    hard    nofile    65536' >> /etc/security/limits.d/redis.conf
echo '*    soft    nofile    65536' >> /etc/security/limits.d/redis.conf

下载依赖包

yum install -y wget gcc bzip2 readline-devel zlib-devel

下载redis包

wget https://download.redis.io/releases/redis-6.2.4.tar.gz

解压包

mkdir -p /app/redis
tar xf redis-6.2.4.tar.gz -C /app/redis

创建启动用户

adduser redis && echo redis | passwd redis --stdin
chown -R redis.redis /app/redis
su - redis

编译redis

cd /app/redis/redis-6.2.4
make -j8

修改配置文件

sed -ri 's/port 6379/port 7777/g' redis.conf
sed -ri 's/# requirepass.*/requirepass redis/g' redis.conf
sed -ri 's@(^logfile).*@\1 "/app/redis/redis-6.2.4/redis.log"@g' /app/redis/redis-6.2.4/redis.conf
sed -ri 's/^(bind).*/\1 * -::*/g' /app/redis/redis-6.2.4/redis.conf

启动redis

/app/redis/redis-6.2.4/src/redis-server /app/redis/redis-6.2.4/redis.conf &

测试redis

/app/redis/redis-6.2.4/src/redis-cli -h 192.168.32.147 -p 7777
192.168.32.147:7777> auth redis  # redis是验证的密码，即上面设置requirepass的值

输入 auth redis 返回是 ok 就正常。

安装postgresql

下载依赖包

yum install -y wget gcc bzip2 readline-devel zlib-devel

下载postgresql包

wget https://mirrors.tuna.tsinghua.edu.cn/postgresql/source/v12.7/postgresql-12.7.tar.bz2

解压postgresql包

tar xf postgresql-12.7.tar.bz2
cd postgresql-12.7

编译postgresql包

mkdir -p /app/pgsql
./configure --prefix=/app/pgsql
make -j8 world -j8
make -j8 install-world

添加启动用户

adduser postgres && echo postgres | passwd postgres --stdin

修改目录权限

mkdir /app/pgsql/{data,logs}
chown -R postgres:postgres /app/pgsql
su - postgres

初始化数据库

/app/pgsql/bin/initdb -D /app/pgsql/data -U admin -W -A md5

修改配置文件

sed -ri 's/^#(port).*/\1 = 5400/g' /app/pgsql/data/postgresql.conf
sed -ri 's/^#(log_destination.*)/\1/g' /app/pgsql/data/postgresql.conf
sed -ri 's/^#(logging_collector).*(#.*)/\1 = no \2/g' /app/pgsql/data/postgresql.conf
sed -ri "s@^#(log_directory).*(#.*)@\1 = '/app/postgresql/logs/' \2@g" /app/pgsql/data/postgresql.conf
sed -ri 's/^#(log_filename.*)/\1/g' /app/pgsql/data/postgresql.conf
sed -ri 's/^#(log_file_mode.*)/\1/g' /app/pgsql/data/postgresql.conf
sed -ri "s@^#(listen_addresses).*(#.*)@\1 = '*' \2@g" /app/pgsql/data/postgresql.conf

修改登录权限

vi /app/pgsql/data/pg_hba.conf
# IPv4 local connections:
host    all             all             127.0.0.1/32            md5
host    all             all             192.168.32.0/24         md5		# 新增的一行

在IPv4连接下面添加一个行，整个网段可以通过账号密码登录。

启动postgresql

/app/pgsql/bin/pg_ctl -D /app/pgsql/data start

获取habor数据

安装harbor仓库

获取数据,其中一台harbor主机执行即可

wget https://github.com/goharbor/harbor/releases/download/v2.3.0/harbor-offline-installer-v2.3.0.tgz
tar xf harbor-offline-installer-v2.3.0.tgz
cd harbor
cp harbor.yml.tmpl harbor.yml
vi harbor.yml
# 1.修改名称
hostname: harbor.ecloud.com
# 2.注释https相关配置

./prepare
./install.sh

导出postgresql数据

docker exec -it harbor-db sh
pg_dump registry > /tmp/registry.sql
docker cp harbor-db:/tmp/registry.sql ~

复制到db主机上

scp ~/registry.sql postgres@192.168.32.147:~

删除harbor仓库

docker-compose down

导入数据

# 创建库
/app/pgsql/bin/createdb -h localhost -p 5400 -U admin registry
/app/pgsql/bin/createdb -h localhost -p 5400 -U admin notaryserver
/app/pgsql/bin/createdb -h localhost -p 5400 -U admin notarysigner

# 导入数据
/app/pgsql/bin/psql -h localhost -p 5400 -U admin -d registry -f ~/registry.sql

# 验证
/app/pgsql/bin/psql -h localhost -p 5400 -U admin registry

安装harbor仓库

安装nfs客户端

mkdir /data
yum -y install nfs-utils
echo "192.168.32.147:/nfs/harbor /data nfs defaults,_netdev 0 0">> /etc/fstab
mount -a

生成harbor服务之间的证书(单节点执行)

docker pull goharbor/prepare:v2.3.0
docker run -v /:/hostfs goharbor/prepare:v2.3.0 gencert -p /data/tls/harbor/

一个客户端执行即可，NFS文件系统共享数据

生成nginx证书(单节点执行)

# 创建目录
mkdir -p /data/tls/nginx && cd /data/tls/nginx

# 生成 CA 证书私钥
openssl genrsa -out ca.key 4096

# 生成 CA 证书
openssl req -x509 -new -nodes -sha512 -days 3650 \
-subj "/C=CN/ST=Guangdong/L=Guangzhou/O=Personal/OU=Personal/CN=ecloud.com" \
-key ca.key \
-out ca.crt

# 生成私钥
openssl genrsa -out ecloud.com.key 4096

# 生成证书签名请求 (CSR)
openssl req -sha512 -new \
-subj "/C=CN/ST=Guangdong/L=Guangzhou/O=Personal/OU=Personal/CN=ecloud.com" \
-key ecloud.com.key \
-out ecloud.com.csr

# 生成 x509 v3 扩展文件
cat > v3.ext <<-EOF
authorityKeyIdentifier=keyid,issuer
basicConstraints=CA:FALSE
keyUsage = digitalSignature, nonRepudiation, keyEncipherment, dataEncipherment
extendedKeyUsage = serverAuth
subjectAltName = @alt_names

[alt_names]
DNS.1=harbor.ecloud.com	# 写成harbor.yml中的hostname的值
DNS.2=*.ecloud.com
EOF

# 使用该v3.ext文件为您的 Harbor 主机生成证书。
openssl x509 -req -sha512 -days 3650 \
    -extfile v3.ext \
    -CA ca.crt -CAkey ca.key -CAcreateserial \
    -in ecloud.com.csr \
    -out ecloud.com.crt

# 转换yourdomain.com.crt为yourdomain.com.cert，供 Docker 使用。
openssl x509 -inform PEM -in ecloud.com.crt -out ecloud.com.cert

一个客户端执行即可，NFS文件系统共享数据

docker配置证书

rm -rf /etc/docker/certs.d/harbor.ecloud.com
mkdir -p /etc/docker/certs.d/harbor.ecloud.com/
cp /data/tls/nginx/{ecloud.com.cert,ecloud.com.key,ca.crt} /etc/docker/certs.d/harbor.ecloud.com/

# 重启docker服务
systemctl restart docker

将服务器证书和密钥复制到 Harbor 主机上的 certficates 文件夹中，目录名称创建为harbor.yml中的hostname的值

下载harbor包

wget https://github.com/goharbor/harbor/releases/download/v2.3.0/harbor-offline-installer-v2.3.0.tgz

解压harbor仓库

tar xf harbor-offline-installer-v2.3.0.tgz

修改harbor配置

cd harbor
cp harbor.yml.tmpl harbor.yml
vim harbor.yml

修改的内容如下：

# 这个就是docker login 的名称
hostname: harbor.ecloud.com
# 配置nginx的证书
https
  # https port for harbor, default is 443
  port: 443
  # The path of cert and key files for nginx
  certificate: /data/tls/nginx/ecloud.com.crt
  private_key: /data/tls/nginx/ecloud.com.key
# 配置harbor服务之间的证书
internal_tls:
  # set enabled to true means internal tls is enabled
  enabled: true
  # put your cert and key files on dir
  dir: /data/tls/harbor
# 启用外部代理
external_url: https://harbor.ecloud.com
# 连接外部数据库
external_database:
  harbor:
    host: 192.168.32.147
    port: 5400
    db_name: registry
    username: admin
    password: admin
    ssl_mode: disable
    max_idle_conns: 2
    max_open_conns: 0
  notary_signer:
    host: 192.168.32.147
    port: 5400
    db_name: notarysigner
    username: admin
    password: admin
    ssl_mode: disable
  notary_server:
    host: 192.168.32.147
    port: 5400
    db_name: notaryserver
    username: admin
    password: admin
    ssl_mode: disable
# 连接外部redis
external_redis:
  # support redis, redis+sentinel
  # host for redis: <host_redis>:<port_redis>
  # host for redis+sentinel:
  #  <host_sentinel1>:<port_sentinel1>,<host_sentinel2>:<port_sentinel2>,<host_sentinel3>:<port_sentinel3>
  host: 192.168.32.147:7777
  password: redis
  # sentinel_master_set must be set to support redis+sentinel
  #sentinel_master_set:
  # db_index 0 is for core, it's unchangeable
  registry_db_index: 1
  jobservice_db_index: 2
  chartmuseum_db_index: 3
  trivy_db_index: 5
  idle_timeout_seconds: 30

安装harbor

./prepare
./install.sh

安装harbor扩展功能(char、漏洞扫描、镜像签名)

下载离线的trivy包

wget https://github.com/aquasecurity/trivy-db/releases/download/v1-2021072600/trivy-light-offline.db.tgz

下载离线的trivy包的话，一定要可以tag改成今天的 v1-2021072600 ，否则提示 ERROR 404: Not Found.

解压trivy

mkdir -p /data/trivy-adapter/trivy/db/
tar xf trivy-offline.db.tgz -C /data/trivy-adapter/trivy/db/
chown 10000:10000 -R /data/trivy-adapter/trivy/db

安装扩展功能

./prepare --with-notary --with-chartmuseum --with-trivy
docker-compose -f docker-compose.yml up -d

如果只安装某个服务的话，那就添加对应的选项。

配置notary

# 配置环境变量
cat > /etc/profile.d/notary << EOF
export DOCKER_CONTENT_TRUST=1
export DOCKER_CONTENT_TRUST_SERVER=https://harbor.ecloud.com:4443
export NOTARY_SNAPSHOT_PASSPHRASE=Harbor12345
export NOTARY_ROOT_PASSPHRASE=Harbor12345
export NOTARY_TARGETS_PASSPHRASE=Harbor12345
EOF
source /etc/profile.d/notary

# docker证书
mkdir -p ~/.docker/tls/harbor.ecloud.com:4443
cp /etc/docker/certs.d/harbor.ecloud.com/ca.crt  ~/.docker/tls/harbor.ecloud.com:4443

# 安装notary命令
wget https://github.com/theupdateframework/notary/releases/download/v0.6.1/notary-Linux-amd64
chmod +x notary-Linux-amd64 && mv notary-Linux-amd64 /usr/local/bin/notary

# 初始化集合
notary -s https://harbor.ecloud.com:4443 -d ~/.docker/trust --tlscacert /etc/docker/certs.d/harbor.ecloud.com/ca.crt init harbor.ecloud.com/library/nginx

# 查看拥有的key信息
notary -s https://harbor.ecloud.com:4443 -d ~/.docker/trust --tlscacert /etc/docker/certs.d/harbor.ecloud.com/ca.crt key list

# tag签名
echo 123456 > file
notary -s https://harbor.ecloud.com:4443 -d ~/.docker/trust --tlscacert /etc/docker/certs.d/harbor.ecloud.com/ca.crt add harbor.ecloud.com/library/nginx latest file

# 查看该信任集合的状态
notary -s https://harbor.ecloud.com:4443 -d ~/.docker/trust --tlscacert /etc/docker/certs.d/harbor.ecloud.com/ca.crt status harbor.ecloud.com/library/nginx

# 该信任集合的改变推送至notary服务端
notary -s https://harbor.ecloud.com:4443 -d ~/.docker/trust --tlscacert /etc/docker/certs.d/harbor.ecloud.com/ca.crt publish harbor.ecloud.com/library/nginx

测试

下载nginx镜像

export DOCKER_CONTENT_TRUST=0
docker pull nginx

修改tag

docker tag nginx:latest harbor.ecloud.com/library/nginx:latest 

登录harbor

echo 'Harbor12345' | docker login harbor.ecloud.com -u admin --password-stdin

推镜像

docker push harbor.ecloud.com/library/nginx:latest
The push refers to repository [harbor.ecloud.com/library/nginx]
e3135447ca3e: Pushed 
b85734705991: Pushed 
988d9a3509bb: Pushed 
59b01b87c9e7: Pushed 
7c0b223167b9: Pushed 
814bff734324: Pushed 
latest: digest: sha256:3f13b4376446cf92b0cb9a5c46ba75d57c41f627c4edb8b635fa47386ea29e20 size: 1570
Signing and pushing trust metadata
Enter passphrase for repository key with ID cfa46c2: 
Enter passphrase for repository key with ID d6efb6a: 
Successfully signed harbor.ecloud.com/library/nginx:latest

验证

keepalived配置

关闭防火墙及selinux

systemctl stop firewalld 
setenforce 0

创建ARP冲突检测脚本

vi /etc/init.d/realserver

#!/bin/bash
# description:Script to start LVS DR real server.
. /etc/rc.d/init.d/functions
VIP=192.168.32.100

#修改相应的VIP
case "$1" in
    start)
        #启动 LVS-DR 模式，real server on this machine. 关闭ARP冲突检测。
        echo "Start LVS of Real Server!"
        /sbin/ifconfig lo down
        /sbin/ifconfig lo up
        echo 1 > /proc/sys/net/ipv4/conf/lo/arp_ignore
        echo 2 > /proc/sys/net/ipv4/conf/lo/arp_announce
        echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore
        echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce
        /sbin/ifconfig lo:0 $VIP broadcast $VIP netmask 255.255.255.255 up
        /sbin/route add -host $VIP dev lo:0
        sudo sysctl -p
    ;;

    stop)
        #停止LVS-DR real server loopback device(s).
        echo "Close LVS Director Server!"
        /sbin/ifconfig lo:0 down
        echo 0 > /proc/sys/net/ipv4/conf/lo/arp_ignore
        echo 0 > /proc/sys/net/ipv4/conf/lo/arp_announce
        echo 0 > /proc/sys/net/ipv4/conf/all/arp_ignore
        echo 0 > /proc/sys/net/ipv4/conf/all/arp_announce
        sudo sysctl -p
    ;;
    
    status)
        # Status of LVS-DR real server.
        islothere=`/sbin/ifconfig lo:0 | grep $VIP`
        isrothere=`netstat -rn | grep "lo:0" | grep $VIP`
        if [ ! "$islothere" -o ! "isrothere" ];then
        # Either the route or the lo:0 device
        # not found.
            echo "LVS-DR real server Stopped!"
        else
            echo "LVS-DR real server Running..."
        fi
    ;;
    
    *)
        # Invalid entry.
        echo "$0: Usage: $0 {start|status|stop}"
        exit 1
    ;;

esac

关闭ARP冲突检测

chmod +x /etc/init.d/realserver 
chmod +x /etc/rc.d/init.d/functions
service realserver start

下载安装包

yum install -y keepalived ipvsadm

修改keepalive master配置

mv /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf.bak
vi /etc/keepalived/keepalived.conf

global_defs {
   smtp_server 127.0.0.1
}

vrrp_instance VI_1 {
    state MASTER          #指定Keepalived的角色，MASTER为主，BACKUP为备 记得大写
    interface ens33       #网卡id 不同的电脑网卡id会有区别 可以使用:ip a查看
    virtual_router_id 51  #虚拟路由编号，主备要一致
    priority 100          #定义优先级，数字越大，优先级越高，主DR必须大于备用DR
    advert_int 1          #检查间隔，默认为1s
    authentication {      #这里配置的密码最多为8位，主备要一致，否则无法正常通讯
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.32.100    #定义虚拟IP(VIP)为192.168.1.200，可多设，每行一个
    }
}

# 定义对外提供服务的LVS的VIP以及port
virtual_server 192.168.32.100 443 {
    delay_loop 6          # 设置健康检查时间，单位是秒
    lb_algo wlc           # 设置负载调度的算法为wlc
    lb_kind DR            # 设置LVS实现负载的机制，有NAT、TUN、DR三个模式
    nat_mask 255.255.255.0
    persistence_timeout 0
    protocol TCP
    real_server 192.168.32.137 443 {  # 指定real server1的IP地址
        weight 3   # 配置节点权值，数字越大权重越高
        TCP_CHECK {
        connect_timeout 10
        nb_get_retry 3
        delay_before_retry 3
        connect_port 443
        }
    }
    real_server 192.168.32.145 443 {  # 指定real server2的IP地址
        weight 3  # 配置节点权值，数字越大权重越高
        TCP_CHECK {
        connect_timeout 10
        nb_get_retry 3
        delay_before_retry 3
        connect_port 443
        }
     }
}

修改keepalived backup配置

mv /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf.bak
vi /etc/keepalived/keepalived.conf

global_defs {
   smtp_server 127.0.0.1
}

vrrp_instance VI_1 {
    state BACKUP #指定Keepalived的角色，MASTER为主，BACKUP为备 记得大写
    interface ens33  #网卡id 不同的电脑网卡id会有区别 可以使用:ip a查看
    virtual_router_id 51  #虚拟路由编号，主备要一致
    priority 50  #定义优先级，数字越大，优先级越高，主DR必须大于备用DR
    advert_int 1  #检查间隔，默认为1s
    authentication {   #这里配置的密码最多为8位，主备要一致，否则无法正常通讯
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.32.100  #定义虚拟IP(VIP)为192.168.1.200，可多设，每行一个
    }
}

# 定义对外提供服务的LVS的VIP以及port
virtual_server 192.168.32.100 443 {
    delay_loop 6 # 设置健康检查时间，单位是秒
    lb_algo wlc  # 设置负载调度的算法为wlc
    lb_kind DR   # 设置LVS实现负载的机制，有NAT、TUN、DR三个模式
    nat_mask 255.255.255.0
    persistence_timeout 0
    protocol TCP
    real_server 192.168.32.137 443 {  # 指定real server1的IP地址
        weight 3   # 配置节点权值，数字越大权重越高
        TCP_CHECK {
        connect_timeout 10
        nb_get_retry 3
        delay_before_retry 3
        connect_port 443
        }
    }

    real_server 192.168.32.145 443 {  # 指定real server2的IP地址
        weight 3  # 配置节点权值，数字越大权重越高
        TCP_CHECK {
        connect_timeout 10
        nb_get_retry 3
        delay_before_retry 3
        connect_port 443
        }
     }
}

启动服务

systemctl start keepalived

docker客户端的配置

获取harbor仓库的证书

scp -r 192.168.32.137:/etc/docker/certs.d /etc/docker/
systemctl restart docker

添加完证书后，需要重启 docker 服务。

IP域名映射

echo 192.168.32.100 harbor.ecloud.com >> /etc/hosts

192.168.32.100 是VIP地址。

所有配置已设置，可以体验私有的仓库