Kubernetes+Harbor部署

Kubernetes+Harbor部署

1：环境

Hostname	CPU	MEM	IP
k8s-master	2	4	10.0.0.10
k8s-node1	2	2	10.0.0.11
k8s-node2	2	2	10.0.0.12
k8s-registry	2	2	10.0.0.13

2：基础配置

1、所以节点关闭防火墙
systemctl stop firewalld
systemctl disable firewalld

2、所以节点关闭selinux
sed -i "s/^SELINUX=.*/SELINUX=disabled/g" /etc/selinux/config
setenforce 0

3、所以节点关闭swap
swapoff -a  # 临时关闭
vi /etc/fstab 注释到swap那一行 # 永久关闭
sed -i 's/.*swap.*/#&/g' /etc/fstab

4、所有节点添加主机名与IP对应关系
cat >> /etc/hosts << EOF
10.0.0.10 k8s-master
10.0.0.11 k8s-node1
10.0.0.12 k8s-node2
10.0.0.13 k8s-registry
10.0.0.13 registry.kubernetes.com
EOF
# 配置Hostname
hostnamectl set-hostname k8s-master
hostnamectl set-hostname k8s-node1
hostnamectl set-hostname k8s-node2
hostnamectl set-hostname k8s-registry

时间同步
yum install chrony -y && systemctl enable chrony --now && chronyc sources -v

5、将桥接的IPv4流量传递到iptables的链（k8s节点执行）、
cat > /etc/sysctl.d/k8s.conf << EOF
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
EOF
sysctl --system

6、所有节点安装docker
yum -y install docker-ce
# 启动docker
systemctl start docker
systemctl enable docker
# 配置加速
# 配置加速，并设置驱动
cat > /etc/docker/daemon.json <<EOF
{
  "registry-mirrors": ["https://6ze43vnb.mirror.aliyuncs.com"],
  "insecure-registries": ["registry.kubernetes.com"],
  "exec-opts": ["native.cgroupdriver=systemd"],
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "100m"
  },
  "storage-driver": "overlay2"
}
EOF

# 加载daemon并重启docker
systemctl daemon-reload
systemctl restart docker

7、部署Harbor镜像仓库（k8s-registry操作）
安装docker-compose并下载harbor最新包
wget https://github.com/goharbor/harbor/releases/download/v2.4.1/harbor-offline-installer-v2.4.1.tgz
wget https://github.com/docker/compose/releases/download/v2.2.3/docker-compose-linux-x86_64 && mv docker-compose-linux-x86_64 docker-compose && chmod +x docker-compose && mv docker-compose /usr/local/bin/

解压harbor-offline-installer-v2.4.1.tgz
tar xf harbor-offline-installer-v2.4.1.tgz
[root@k8s-registry ~]# ls
anaconda-ks.cfg  harbor

进入harbor内
[root@k8s-registry harbor]# ls
common.sh  harbor.v2.4.1.tar.gz  harbor.yml.tmpl  install.sh  LICENSE  prepare
[root@k8s-registry harbor]# cp harbor.yml.tmpl harbor.yml

harbor.yml如下
[root@k8s-registry harbor]# cat harbor.yml | grep -v "#" | grep -v "^$"
hostname: registry.kubernetes.com
http:
  port: 80
harbor_admin_password: Harbor12345
database:
  password: root123
  max_idle_conns: 100
  max_open_conns: 900
data_volume: /data
trivy:
  ignore_unfixed: false
  skip_update: false
  insecure: false
jobservice:
  max_job_workers: 10
notification:
  webhook_job_max_retry: 10
chart:
  absolute_url: disabled
log:
  level: info
  local:
    rotate_count: 50
    rotate_size: 200M
    location: /var/log/harbor
_version: 2.4.0
proxy:
  http_proxy:
  https_proxy:
  no_proxy:
  components:
    - core
    - jobservice
    - trivy
    
# 若要做存储请修改挂在本机的/data目录即可

执行./install安装harbor
[root@k8s-registry harbor]# ./install.sh 

[Step 0]: checking if docker is installed ...

Note: docker version: 20.10.12

[Step 1]: checking docker-compose is installed ...

Note: docker-compose version: 2.2.3

[Step 2]: loading Harbor images ...
# 此时等待安装完成即可

✔ ----Harbor has been installed and started successfully.----
出现这个之后即可去访问了
# 主要提供webUI管理仓库
http://registry.kubernetes.com（此为内网，并且没有开启SSL，若要开SSL可具体参考配置文件）
账号：admin
密码：Harbor12345
# 到此 harbor就安装完成了

3：安装Kubeadm以及初始化Kubernetes集群

1：K8S节点安装kubeadm，kubelet和kubectl
yum install -y kubelet-1.23.1 kubeadm-1.23.1 kubectl-1.23.1
systemctl enable kubelet --now

2：初始化master节点
- 只在master节点执行
- 由于默认拉取镜像地址k8s.gcr.io国内无法访问，这里指定阿里云镜像仓库地址
- 执行成功以后最后结果会输出
kubeadm init \
  --apiserver-advertise-address=10.0.0.10 \
  --image-repository registry.aliyuncs.com/google_containers \
  --kubernetes-version v1.23.1 \
  --pod-network-cidr=100.1.0.0/16 \
  --service-cidr=172.1.0.0/16
  
3：kubeadm join 10.0.0.10:6443 --token hddt9h.0npkq0zjk2q9irex \
	--discovery-token-ca-cert-hash sha256:4148718860ea26e64908bb9007e548a98370141fb1f934dfadf651463b9da4d2 
	
4：设置kubectl默认访问的api
mkdir -p $HOME/.kube
sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
sudo chown $(id -u):$(id -g) $HOME/.kube/config
echo "source <(kubectl completion bash)" >> ~/.bashrc
source ~/.bashrc

5：部署calico网络插件
curl https://docs.projectcalico.org/manifests/calico.yaml -O
kubectl apply -f calico.yaml

6：查询集群pod
[root@k8s-master ~]# kubectl get pod -A
NAMESPACE     NAME                                       READY   STATUS    RESTARTS   AGE
kube-system   calico-kube-controllers-85b5b5888d-vvzsf   1/1     Running   0          26m
kube-system   calico-node-2hsmk                          1/1     Running   0          24m
kube-system   calico-node-nr68m                          1/1     Running   0          26m
kube-system   calico-node-nwvbz                          1/1     Running   0          26m
kube-system   coredns-6d8c4cb4d-ctg8m                    1/1     Running   0          45m
kube-system   coredns-6d8c4cb4d-h6tvp                    1/1     Running   0          45m
kube-system   etcd-k8s-master                            1/1     Running   0          45m
kube-system   kube-apiserver-k8s-master                  1/1     Running   0          45m
kube-system   kube-controller-manager-k8s-master         1/1     Running   0          45m
kube-system   kube-proxy-9gr5t                           1/1     Running   0          28m
kube-system   kube-proxy-cbrrk                           1/1     Running   0          45m
kube-system   kube-proxy-f6s6s                           1/1     Running   0          25m
kube-system   kube-scheduler-k8s-master                  1/1     Running   0          45m

4：测试kubernetes与harbor的连通性

1：拉取镜像并导入镜像仓库并测试从K8S拉取镜像是否正常
从harbor本身拉取镜像并推送到仓库内
[root@k8s-registry harbor]# docker pull nginx:alpine
# 拉取代码
[root@k8s-registry harbor]# docker tag nginx:alpine registry.kubernetes.com/library/nginx:alpine
# 打上tag
[root@k8s-registry harbor]# docker login registry.kubernetes.com
Username: admin
Password: 
WARNING! Your password will be stored unencrypted in /root/.docker/config.json.
Configure a credential helper to remove this warning. See
https://docs.docker.com/engine/reference/commandline/login/#credentials-store

Login Succeeded
# 登录仓库
[root@k8s-registry harbor]# docker push registry.kubernetes.com/library/nginx:alpine 
The push refers to repository [registry.kubernetes.com/library/nginx]
419df8b60032: Pushed 
0e835d02c1b5: Pushed 
5ee3266a70bd: Pushed 
3f87f0a06073: Pushed 
1c9c1e42aafa: Pushed 
8d3ac3489996: Pushed 
alpine: digest: sha256:544ba2bfe312bf2b13278495347bb9381ec342e630bcc8929af124f1291784bb size: 1568
# 上传代码到仓库

接下来测试k8s拉取代码并部署服务

nginx.yaml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: nginx
spec:
  replicas: 1
  selector:
    matchLabels:
      app: nginx
  template:
    metadata:
      labels:
        app: nginx
    spec:
      containers:
      - name: nginx
        image: registry.kubernetes.com/library/nginx:alpine
        imagePullPolicy: IfNotPresent
        ports:
        - containerPort: 80
---
apiVersion: v1
kind: Service
metadata:
  name: nginx-svc
spec:
  type: NodePort
  ports:
  - port: 80
    protocol: TCP
    targetPort: 80
  selector:
    name: nginx
    
[root@k8s-master ~]# kubectl get pod 
NAME                     READY   STATUS    RESTARTS   AGE
nginx-7f59b97747-pdz74   1/1     Running   0          15s


[root@k8s-master ~]# kubectl describe pod nginx-7f59b97747-pdz74 
---
Events:
  Type    Reason     Age   From               Message
  ----    ------     ----  ----               -------
  Normal  Scheduled  22s   default-scheduler  Successfully assigned default/nginx-7f59b97747-pdz74 to k8s-node1
  Normal  Pulling    21s   kubelet            Pulling image "registry.kubernetes.com/library/nginx:alpine"
  Normal  Pulled     20s   kubelet            Successfully pulled image "registry.kubernetes.com/library/nginx:alpine" in 944.744861ms
  Normal  Created    20s   kubelet            Created container nginx
  Normal  Started    20s   kubelet            Started container nginx

可以看到镜像地址是我们私有的