openshift 3.11离线安装

本次基于redhat官方的rpm包安装，并导入提前下载好的docker镜像

一、环境描述

master节点：10.157.27.117
infra节点：10.157.27.118
node 节点：10.157.27.119

操作系统：RHEL7.6

1.添加Hosts解析(openshift-cluster.suixin.com作为集群入口)

~]# cat /etc/hosts
10.157.27.117 master1.suixin.com 
10.157.27.119 node1.suixin.com
10.157.27.118 infra1.suixin.com
10.157.27.117 openshift-cluster.suixin.com

2、配置SELINUX

~]# cat /etc/selinux/config    
# This file controls the state of SELinux on the system.
# SELINUX= can take one of these three values:
#     enforcing - SELinux security policy is enforced.
#     permissive - SELinux prints warnings instead of enforcing.
#     disabled - No SELinux policy is loaded.
SELINUX=permissive
# SELINUXTYPE= can take one of three two values:
#     targeted - Targeted processes are protected,
#     minimum - Modification of targeted policy. Only selected processes are protected.
#     mls - Multi Level Security protection.
SELINUXTYPE=targeted
~]# setenforce 0

3、配置免密钥登录,修改主机名

~]# ssh-keygen
~]# for i in  master1.suixin.com  node1.suixin.com infra1.suixin.com; do echo $i; ssh-copy-id $i; done;
~]# for i in master1.suixin.com  node1.suixin.com infra1.suixin.com;do echo $i;ssh $i hostnamectl "set-hostname $i"; done;

4、搭建Openshift yum源(选择infra节点)

~]# tar -zxvf ocp311rpms.tar.gz -C /data/repos/
~]# ll /data/repos/
总用量 0
drwxr-xr-x. 4 root root 55 Oct 19 23:50 rhel-7-server-ansible-2.6-rpms
drwxr-xr-x. 4 root root 55 Oct 19 23:50 rhel-7-server-extras-rpms
drwxr-xr-x. 4 root root 55 Oct 20 11:04 rhel-7-server-ose-3.11-rpms
drwxr-xr-x. 4 root root 55 Oct 19 23:48 rhel-7-server-rpms
~]# cd /data/repos/
~]# createrepo ./
~]# yum -y install httpd
~]# cat /etc/httpd/conf.d/yum.conf
Alias /repos "/opt/repos"
<Directory "/opt/repos">
  Options +Indexes +FollowSymLinks
  Require all granted
</Directory>
<Location /repos>
SetHandler None
 </Location>

#修改httpd的服务为81端口，防止与router的80端口冲突
~]# vi /etc/httpd/conf/httpd.conf
Listen 81
~]# systemctl enable httpd;systemctl restart httpd
~]# cat /etc/yum.repos.d/yum.repo
[rhel-7-server-ansible-2.6-rpms]
name=rhel-7-server-ansible-2.6-rpms
baseurl=http://master1.suixin.com:81/repos/rhel-7-server-ansible-2.6-rpms
enable=1
gpgcheck=0

[rhel-7-server-extras-rpms]
name=rhel-7-server-extras-rpms
baseurl=http://master1.suixin.com:81/repos/rhel-7-server-extras-rpms
enable=1
gpgcheck=0

[rhel-7-server-ose-3.11-rpms]
name=rhel-7-server-ose-3.11-rpms
baseurl=http://master1.suixin.com:81/repos/rhel-7-server-ose-3.11-rpms
enable=1
gpgcheck=0

[rhel-7-server-rpms]
name=rhel-7-server-rpms
baseurl=http://master1.suixin.com:81/repos/rhel-7-server-rpms
enable=1
gpgcheck=0

~]# yum clean all;yum repolist

5、安装iptables

~]# systemctl stop firewalld
~]# yum install iptables-services -y
~]# cp /etc/sysconfig/iptables /etc/sysconfig/iptables.bak.$(date "+%Y%m%d%H%M%S");
~]# sed -i '/.*--dport 22 -j ACCEPT.*/a\-A INPUT -p tcp -m state --state NEW -m tcp --dport 53 -j ACCEPT' /etc/sysconfig/iptables;
~]# sed -i '/.*--dport 22 -j ACCEPT.*/a\-A INPUT -p udp -m state --state NEW -m udp --dport 53 -j ACCEPT' /etc/sysconfig/iptables;
~]# sed -i '/.*--dport 22 -j ACCEPT.*/a\-A INPUT -p tcp -m state --state NEW -m tcp --dport 5000 -j ACCEPT' /etc/sysconfig/iptables;
~]# sed -i '/.*--dport 22 -j ACCEPT.*/a\-A INPUT -p tcp -m state --state NEW -m tcp --dport 81 -j ACCEPT' /etc/sysconfig/iptables;
~]# systemctl restart iptables;systemctl enable iptables

6、安装openshift-ansible

~]# yum install openshift-ansible -y
~]# cat /etc/ansible/hosts 
# Create an OSEv3 group that contains the masters, nodes, and etcd groups
[OSEv3:children]
masters
nodes
etcd
    
# Set variables common for all OSEv3 hosts
[OSEv3:vars]
ansible_ssh_user=root
openshift_deployment_type=openshift-enterprise
openshift_image_tag=v3.11.98
# If ansible_ssh_user is not root, ansible_become must be set to true
#ansible_become=true
    
    
# default selectors for router and registry services
# openshift_router_selector='node-role.kubernetes.io/infra=true'
# openshift_registry_selector='node-role.kubernetes.io/infra=true'
    
# uncomment the following to enable htpasswd authentication; defaults to DenyAllPasswordIdentityProvider
openshift_master_identity_providers=[{'name': 'htpasswd_auth', 'login': 'true', 'challenge': 'true', 'kind': 'HTPasswdPasswordIdentityProvider'}]
openshift_master_default_subdomain=suixin.com
openshift_disable_check=memory_availability,disk_availability,docker_image_availability
    
os_sdn_network_plugin_name=redhat/openshift-ovs-multitenant
    
openshift_master_cluster_method=native
openshift_master_cluster_hostname=openshift-cluster.suixin.com
openshift_master_cluster_public_hostname=openshift-cluster.suixin.com
# false
ansible_service_broker_install=false
openshift_enable_service_catalog=false
template_service_broker_install=false
    
# registry passwd
oreg_url=infra1.suixin.com:5000/openshift3/ose-${component}:${version}
openshift_examples_modify_imagestreams=true
    
# docker config
openshift_docker_additional_registries=infra1.suixin.com:5000
openshift_docker_insecure_registries=infra1.suixin.com:5000
#openshift_docker_blocked_registries
openshift_docker_options="--log-driver json-file --log-opt max-size=1M --log-opt max-file=3"
    
# prometheus
openshift_cluster_monitoring_operator_install=true
openshift_cluster_monitoring_operator_node_selector={'node-role.kubernetes.io/infra': 'true'}
# logging
openshift_enable_unsupported_configurations=true
openshift_logging_install_logging=true
openshift_logging_image_prefix=infra1.suixin.com:5000/openshift3/
openshift_logging_image_version=v3.11.98
openshift_logging_es_nodeselector={'node-role.kubernetes.io/infra': 'true'}
openshift_logging_kibana_nodeselector={'node-role.kubernetes.io/infra': 'true'}
# router
openshift_route_selector='node-role.kubernetes.io/infra=true'
#openshift_set_node_ip=true
openshift_ip_check=false

[masters]
master1.suixin.com
    
# host group for etcd
[etcd]
master1.suixin.com
    
# host group for nodes, includes region info
[nodes]
master1.suixin.com openshift_node_group_name='node-config-master'
node1.suixin.com openshift_node_group_name='node-config-compute'
infra1.suixin.com openshift_node_group_name='node-config-infra'

7、分发文件到其他节点

~]# ansible all -m copy -a "src=/etc/hosts dest=/etc/hosts "
~]# ansible all -m copy -a "src=/etc/yum.repos.d/ocp.repo dest=/etc/yum.repos.d/ocp.repo"
~]# ansible all -m copy -a "src=/etc/selinux/config dest=/etc/selinux/config"
~]# ansible all -m shell -a "setenforce 0"
~]# ansible all -m copy -a "src=/etc/sysconfig/iptables dest=/etc/sysconfig/iptables"
~]# ansible all -m shell -a "systemctl restart iptables"

8、所有节点基础包安装

~]# yum install wget git net-tools bind-utils yum-utils iptables-services bridge-utils bash-completion kexec-tools sos psacct vim lrzsz python-setuptools unzip tree docker-1.13.1 NetworkManager -y
~]# ansible all -m shell -a 'systemctl start NetworkManager' #不启动，安装时会报错

9、修改docker配置文件
(1)配置docker镜像仓库地址

~]# vi /etc/containers/registries.conf
# This is a system-wide configuration file used to
# keep track of registries for various container backends.
# It adheres to TOML format and does not support recursive
# lists of registries.

# The default location for this configuration file is /etc/containers/registries.conf.

# The only valid categories are: 'registries.search', 'registries.insecure', 
# and 'registries.block'.

[registries.search]
#registries = ['registry.access.redhat.com', 'docker.io', 'registry.fedoraproject.org', 'quay.io', 'registry.centos.org']
registries = ['infra1.suixin.com']

# If you need to access insecure registries, add the registry's fully-qualified name.
# An insecure registry is one that does not have a valid SSL certificate or only does HTTP.
[registries.insecure]
registries = ['infra1.suixin.com']


# If you need to block pull access from a registry, uncomment the section below
# and add the registries fully-qualified name.
#
# Docker only
[registries.block]
registries = ['infra1.suixin.com']

(2)配置docker的镜像仓库使用http(默认使用https)

~]# cat /etc/sysconfig/docker
# /etc/sysconfig/docker

# Modify these options if you want to change the way the docker daemon runs
OPTIONS=' --selinux-enabled       --log-driver json-file --log-opt max-size=1M --log-opt max-file=3 --signature-verification=False'
if [ -z "${DOCKER_CERT_PATH}" ]; then
    DOCKER_CERT_PATH=/etc/docker
fi

# Do not add registries in this file anymore. Use /etc/containers/registries.conf
# instead. For more information reference the registries.conf(5) man page.

# Location used for temporary files, such as those created by
# docker load and build operations. Default is /var/lib/docker/tmp
# Can be overriden by setting the following environment variable.
# DOCKER_TMPDIR=/var/tmp

# Controls the /etc/cron.daily/docker-logrotate cron job status.
# To disable, uncomment the line below.
# LOGROTATE=false

# docker-latest daemon can be used by starting the docker-latest unitfile.
# To use docker-latest client, uncomment below lines
#DOCKERBINARY=/usr/bin/docker-latest
#DOCKERDBINARY=/usr/bin/dockerd-latest
#DOCKER_CONTAINERD_BINARY=/usr/bin/docker-containerd-latest
#DOCKER_CONTAINERD_SHIM_BINARY=/usr/bin/docker-containerd-shim-latest
ADD_REGISTRY='--add-registry infra1.suixin.com:5000 --add-registry registry.redhat.io'
INSECURE_REGISTRY='--insecure-registry infra1.suixin.com:5000'

(3) 修改docker镜像存放位置

~]# vi /usr/lib/systemd/system/docker.service
ExecStart=/usr/bin/dockerd-current \
	  --graph=/data/docker \
      ......

(4)启动Docker

~]# ansible all -m copy -a "src=/etc/containers/registries.conf dest=/etc/containers/registries.conf "
~]# ansible all -m copy -a "src=/etc/sysconfig/docker dest=/etc/sysconfig/docker "
~]# ansible all -a 'systemctl start docker';ansible all -a 'systemctl enable docker'

二、镜像仓库搭建(infra节点)

1、安装docker-distribution

~]# yum install -y docker-distribution;systemctl start docker-distribution;systemctl enable docker-distribution

2、上传镜像包并导入，此过程会占用大量磁盘空间和内存资源

~]# docker load -i ocp3.11-98-images.tar
~]# docker images|grep registry.redhat.io|awk '{print $1"/"$2}'|awk -F "/" '{print "docker tag "$1"/"$2"/"$3":"$4" infra1.suixin.com:5000/"$2"/"$3":"$4}'|sh
~]# docker images|grep infra1.suixin.com5000|awk '{print "docker push " $1":"$2}'|sh

三、安装openshift

1、部署前检查

~]# ansible-playbook /usr/share/ansible/openshift-ansible/playbooks/prerequisites.yml

2、开始部署

~]# ansible-playbook /usr/share/ansible/openshift-ansible/playbooks/deploy_cluster.yml

3、卸载

~]# ansible-playbook  /usr/share/ansible/openshift-ansible/playbooks/adhoc/uninstall.yml

4、创建用户

~]# htpasswd -cb /etc/origin/master/htpasswd admin 123456
~]# oc adm policy add-cluster-role-to-user cluster-admin admin