Oracle 12cR2 RAC集群安装指南
一、环境介绍
|
服务器 |
vSphere 6虚拟机 |
|
操作系统 |
OEL 7.0 with Kernal 3.8.13-35.3.1.el7uek |
|
硬件配置 |
CPU 32 Cores 内存 96 G 硬盘 150 G |
二、IP规划
|
主机名 |
Public IP |
Private IP |
VIP |
Scan IP |
|
oraclerac8281 |
172.16.82.81 |
192.168.82.81 |
172.16.82.84 |
172.16.82.87 |
|
oraclerac8282 |
172.16.82.82 |
192.168.82.82 |
172.16.82.85 |
|
|
oraclerac8283 |
172.16.82.83 |
192.168.82.83 |
172.16.82.86 |
三、存储规划
|
硬盘 |
ASM Disk类型 |
大小 |
|
/dev/sdb |
OCR/Voting Disk/GIMR |
50G |
|
/dev/sdc |
OCR/Voting Disk/GIMR |
50G |
|
/dev/sdd |
OCR/Voting Disk/GIMR |
50G |
|
/dev/sde |
DATA |
2T |
|
/dev/sdf |
RECOVERY |
1T |
四、系统配置（每个节点）
1、禁用Selinux/Firewall
sed -i '/^SELINUX=.*/ s//SELINUX=disabled/' /etc/selinux/config
echo "Disable Firewall"
systemctl disable firewalld
2、配置本地Yum库
cat > /etc/yum.repos.d/local.repo <<!
[rhel-media]
name=RHEL-\$releasever - Media
baseurl=file:///media
enabled=1
gpgcheck=0
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release
!
3、配置Host解析
sed -i 3,\$d /etc/hosts
cat >> /etc/hosts <<!
#Public IP
172.16.82.81 oraclerac8281
172.16.82.82 oraclerac8282
172.16.82.83 oraclerac8283
#Private IP
192.168.82.81 oraclerac8281-priv
192.168.82.82 oraclerac8282-priv
192.168.82.83 oraclerac8283-priv
#VIP
172.16.82.84 oraclerac8281-vip
172.16.82.85 oraclerac8282-vip
172.16.82.86 oraclerac8283-vip
#Scan-IP
172.16.82.87 oraclerac-act-scan
!
4、依赖包安装
yum -y install binutils compat-libcap1 compat-libstdc++-33 \
compat-libstdc++-33*.i686 gcc gcc-c++ glibc glibc*.i686 \
glibc-devel glibc-devel*.i686 ksh libaio libaio-*.i686 \
libaio-devel libaio-devel*.i686 libgcc libgcc*.i686 \
libstdc++ libstdc++-*.i686 libstdc++-devel libstdc++devel*.i686 \
libXi libXi*.i686 libXtst libXtst*.i686 make sysstat \
unixODBC unixODBC-devel unixODBC*.i686 zlib-devel zlib-devel.i686 \
nfs-utils sysstat chrony smartmontools net-tools zip unzip
5、系统参数配置
MEMTOTAL=$(free -b | sed -n '2p' | awk '{print $2}')
let SHMMAX=$MEMTOTAL*4/5
SHMMNI=4096
let SHMALL=$MEMTOTAL/4096
echo SHMMAX:$SHMMAX SHMALL:$SHMALL
cat > /etc/sysctl.d/98-oracle.conf <<!
fs.aio-max-nr = 1048576
fs.file-max = 6815744
kernel.shmall = $SHMALL
kernel.shmmax = $SHMMAX
kernel.shmmni = 4096
kernel.sem = 250 32000 100 128
net.ipv4.ip_local_port_range = 9000 65500
net.core.rmem_default = 262144
net.core.rmem_max = 4194304
net.core.wmem_default = 262144
net.core.wmem_max = 1048576
kernel.panic_on_oops = 1
net.ipv4.conf.all.rp_filter = 2
net.ipv4.conf.default.rp_filter = 2
!
/sbin/sysctl -p /etc/sysctl.d/98-oracle.conf
6、添加用户和组
groupadd -g 54321 oinstall
groupadd -g 54322 dba
groupadd -g 54323 oper
groupadd -g 54324 backupdba
groupadd -g 54325 dgdba
groupadd -g 54326 kmdba
groupadd -g 54327 asmdba
groupadd -g 54328 asmoper
groupadd -g 54329 asmadmin
groupadd -g 54330 racdba
useradd -u 54321 -g oinstall -G dba,asmdba,backupdba,dgdba,kmdba,racdba,oper oracle
useradd -u 54331 -g oinstall -G dba,oper,backupdba,dgdba,kmdba,asmdba,asmoper,asmadmin,racdba grid
echo "grid" | passwd --stdin grid
echo "oracle:oracle" | chpasswd
7、创建目录并设置权限
mkdir -p /u01/app/grid
mkdir -p /u01/app/12.2.0/grid
mkdir -p /u01/app/oracle
mkdir -p /u01/app/oracle/product/12.2.0/db_1
mkdir -p /u01/app/oraInventory
chown -R grid:oinstall /u01/app
chown -R oracle:oinstall /u01/app/oracle
chmod -R 775 /u01/app
8、修改用户安全限制
cat > /etc/security/limits.d/oracle-database-server-12cR2-preinstall.conf <<!
oracle soft nofile 2048
oracle hard nofile 65536
oracle soft nproc 16384
oracle hard nproc 16384
oracle soft stack 10243
oracle hard stack 32768
oracle hard memlock 134217728
oracle soft memlock 134217728
grid soft nofile 2048
grid hard nofile 65536
grid soft nproc 16384
grid hard nproc 16384
grid soft stack 10243
grid hard stack 32768
grid hard memlock 134217728
grid soft memlock 134217728!
9、添加PAM模块
cat >> /etc/pam.d/login <<!
session required pam_limits.so
!
10、禁用透明大页
cat >> /etc/rc.local <<!
if test -f /sys/kernel/mm/redhat_transparent_hugepage/enabled; then
echo never > /sys/kernel/mm/redhat_transparent_hugepage/enabled
fi
!
11、配置Grid环境变量
sed -i 13,\$d ~grid/.bash_profile
cat >> ~grid/.bash_profile <<!
export ORACLE_BASE=/u01/app/grid
export ORACLE_HOME=/u01/app/12.2.0/grid
export ORACLE_SID=+ASM$(STR=$(hostname); echo ${STR: -1})
export PATH=\$ORACLE_HOME/bin:\$ORACLE_HOME/OPatch:\$PATH
export LD_LIBRARY_PATH=\$ORACLE_HOME/lib:\$LD_LIBRARY_PATH
!
cat ~grid/.bash_profile
12、配置Oracle环境变量
sed -i 13,\$d ~oracle/.bash_profile
cat >> ~oracle/.bash_profile <<!
export ORACLE_BASE=/u01/app/oracle
export ORACLE_HOME=\$ORACLE_BASE/product/12.2.0/db_1
export ORACLE_SID=orcl1$(STR=$(hostname); echo ${STR: -1})
export NLS_LANG=AMERICAN_AMERICA.AL32UTF8
export PATH=\$ORACLE_HOME/bin:\$ORACLE_HOME/OPatch:\$PATH
export LD_LIBRARY_PATH=\$ORACLE_HOME/lib:\$LD_LIBRARY_PATH
!
cat ~oracle/.bash_profile
13、配置Oracle/Grid互信（只需在一个节点执行）
Oracle提供的database安装包里有脚本可以配置互信，比较方便。
su - oracle -c "unzip -qo /tmp/linuxx64_12201_database.zip -d /u01/app"
/u01/app/database/sshsetup/sshUserSetup.sh -user oracle -hosts "oraclerac8281 oraclerac8282 oraclerac8283" -advanced -noPromptPassphrase
/u01/app/database/sshsetup/sshUserSetup.sh -user grid -hosts "oraclerac8281 oraclerac8282 oraclerac8283" -advanced -noPromptPassphrase
配置完成后使用下面命令测试用户连通性
su - oracle -c "ssh oraclerac8281 date; ssh oraclerac8282 date; ssh oraclerac8283 date; exit"
su - grid -c "ssh oraclerac8281 date; ssh oraclerac8282 date; ssh oraclerac8283 date; exit"
分发cvuqdisk安装包到所有节点/tmp下
su - oracle -c "
scp /u01/app/database/rpm/cvuqdisk-1.0.10-1.rpm oraclerac8281:/tmp
scp /u01/app/database/rpm/cvuqdisk-1.0.10-1.rpm oraclerac8282:/tmp
scp /u01/app/database/rpm/cvuqdisk-1.0.10-1.rpm oraclerac8283:/tmp
"
14、配置NOZEROCONF
echo "NOZEROCONF=yes" >> /etc/sysconfig/network
grep ^NOZEROCONF /etc/sysconfig/network
15、修改nsswitch.conf
sed -i "/^hosts:/chosts: files dns nis" /etc/nsswitch.conf
grep ^hosts /etc/nsswitch.conf
16、配置tmpfs挂载大小
默认tmpfs挂载的大小为内存的一半，SGA设置如果超过内存一半，需要重新配置。
mem=$(free -m | grep ^Mem | awk '{print$2}'); let sga=mem/100*64
sed -i /^tmpfs/d /etc/fstab
echo "tmpfs /dev/shm tmpfs size=${sga}M,mode=1777,nosuid,nodev,strictatime 0 0" >> /etc/fstab
mount -o loop /dev/shm
df -h /dev/shm
17、安装CVU Disk
export CVUQDISK_GRP=asmadmin
rpm -ivh /tmp/cvuqdisk-1.0.10-1.rpm
18、桌面环境安装（只需要装一个节点）
如果RAC服务器和本地节点全端口互通，只需要安装X Window即可，否则需要安装桌面环境
X Window安装"（可选）
yum -y groupinstall "X Window system"
Gnome桌面环境安装（可选）
yum -y groupinstall "Server with GUI"
五、UDEV配置
1、获取Disk UUID
虚拟机需要关机，设置disk.EnableUUID=TRUE。
虚拟机属性->选项->高级->常规->配置参数
然后执行下面命令获取挂载硬盘的UUID
for i in b c d e f #这里需要列出需要的盘符
do
echo -n "/dev/sd$i: "
/lib/udev/scsi_id --whitelisted --replace-whitespace --device=/dev/sd$i
done
2、配置UDEV Rule文件
使用上一步获取到的UUID一一对应替换下面RESULT值，并自定义挂载名称（标红部分）。
cat > /etc/udev/rules.d/99-oracle-asmdevices.rules <<!
KERNEL=="sd*", ENV{DEVTYPE}=="disk", SUBSYSTEM=="block", PROGRAM=="/usr/lib/udev/scsi_id -g -u -d \$devnode", RESULT=="36000c29d6d2dd44e3dc77ec91b3c4b27", RUN+="/bin/sh -c 'mknod /dev/asmdisk01 b \$major \$minor; chown grid:asmadmin /dev/asmdisk01; chmod 0660 /dev/asmdisk01'"
KERNEL=="sd*", ENV{DEVTYPE}=="disk", SUBSYSTEM=="block", PROGRAM=="/usr/lib/udev/scsi_id -g -u -d \$devnode", RESULT=="36000c29f112c5cc79f547b0571cf17e6", RUN+="/bin/sh -c 'mknod /dev/asmdisk02 b \$major \$minor; chown grid:asmadmin /dev/asmdisk02; chmod 0660 /dev/asmdisk02'"
KERNEL=="sd*", ENV{DEVTYPE}=="disk", SUBSYSTEM=="block", PROGRAM=="/usr/lib/udev/scsi_id -g -u -d \$devnode", RESULT=="36000c29e113772582ecb184c62e6f470", RUN+="/bin/sh -c 'mknod /dev/asmdisk03 b \$major \$minor; chown grid:asmadmin /dev/asmdisk03; chmod 0660 /dev/asmdisk03'"
KERNEL=="sd*", ENV{DEVTYPE}=="disk", SUBSYSTEM=="block", PROGRAM=="/usr/lib/udev/scsi_id -g -u -d \$devnode", RESULT=="36000c290e51112c33800622a77a3cc0e", RUN+="/bin/sh -c 'mknod /dev/datdisk01 b \$major \$minor; chown grid:asmadmin /dev/datdisk01; chmod 0660 /dev/datdisk01'"
KERNEL=="sd*", ENV{DEVTYPE}=="disk", SUBSYSTEM=="block", PROGRAM=="/usr/lib/udev/scsi_id -g -u -d \$devnode", RESULT=="36000c29783cf6791a19b69403dd9a064", RUN+="/bin/sh -c 'mknod /dev/arcdisk01 b \$major \$minor; chown grid:asmadmin /dev/arcdisk01; chmod 0660 /dev/arcdisk01'"
!
cat >> /etc/udev/rules.d/99-oracle-asmdevices.rules <<!
ACTION=="add|change", ENV{ID_SERIAL}=="36000c29d6d2dd44e3dc77ec91b3c4b27", NAME="asmdisk01", OWNER="grid", GROUP="asmadmin", MODE="0660"
ACTION=="add|change", ENV{ID_SERIAL}=="36000c29f112c5cc79f547b0571cf17e6", NAME="asmdisk02", OWNER="grid", GROUP="asmadmin", MODE="0660"
ACTION=="add|change", ENV{ID_SERIAL}=="36000c29e113772582ecb184c62e6f470", NAME="asmdisk03", OWNER="grid", GROUP="asmadmin", MODE="0660"
ACTION=="add|change", ENV{ID_SERIAL}=="36000c290e51112c33800622a77a3cc0e", NAME="datdisk01", OWNER="grid", GROUP="asmadmin", MODE="0660"
ACTION=="add|change", ENV{ID_SERIAL}=="36000c29783cf6791a19b69403dd9a064", NAME="arcdisk01", OWNER="grid", GROUP="asmadmin", MODE="0660"
!
3、重新加载规则
udevadm control --reload-rules
/sbin/udevadm trigger --type=devices --action=change
sleep 3
ls -Ll /dev/sd*
ls -Ll /dev/*disk0*
确认Owner/Group正确。
# ls -Ll /dev/sd*
brw-rw---- 1 root disk 8, 0 Apr 24 2020 /dev/sda
brw-rw---- 1 root disk 8, 1 Apr 24 2020 /dev/sda1
brw-rw---- 1 root disk 8, 2 Apr 24 2020 /dev/sda2
brw-rw---- 1 grid asmadmin 8, 16 Apr 24 2020 /dev/sdb
brw-rw---- 1 grid asmadmin 8, 32 Apr 24 2020 /dev/sdc
brw-rw---- 1 grid asmadmin 8, 48 Apr 24 2020 /dev/sdd
brw-rw---- 1 grid asmadmin 8, 64 Apr 24 2020 /dev/sde
brw-rw---- 1 grid asmadmin 8, 80 Apr 24 2020 /dev/sdf
# ls /dev/*disk0* -l
brw-rw---- 1 grid asmadmin 8, 80 Apr 23 22:35 /dev/arcdisk01
brw-rw---- 1 grid asmadmin 8, 16 Apr 23 22:35 /dev/asmdisk01
brw-rw---- 1 grid asmadmin 8, 32 Apr 23 22:35 /dev/asmdisk02
brw-rw---- 1 grid asmadmin 8, 48 Apr 23 22:35 /dev/asmdisk03
brw-rw---- 1 grid asmadmin 8, 64 Apr 23 22:35 /dev/datdisk01
插曲：一定要确保原始盘的属主配置正确，否则在GI执行root.sh第18步的时候会出现CLSRSC-182: Failed to add the resource for ASM diskgroup的错误。
六、安装Grid Infrastructure（节点1执行）
1、解压
因为从12c R2开始，Grid组件以解压安装的方式，所以需要将压缩包直接解压到$ORACLE_HOME目录，否则解压的目录就是安装目录了。
unzip -q /tmp/linuxx64_12201_grid_home.zip -d $ORACLE_HOME
2、检测安装环境
$ORACLE_HOME/runcluvfy.sh stage -pre crsinst -n oraclerac8281,oraclerac8282,oraclerac8283 -fixup -verbose
3、开始安装
$ORACLE_HOME/gridSetup.sh
4、执行脚本（root）
/u01/app/oraInventory/orainstRoot.sh
/u01/app/12.2.0/grid/root.sh
等节点1执行成功之后，在其它节点并行执行。
5、检查集群状态
crsctl status res -t
6、创建其它磁盘组
asmca
七、安装Database（节点1执行）
1、安装
/u01/database/runInstaller
2、执行脚本（root）
/u01/app/oracle/product/12.2.0/db_1/root.sh
3、检查数据库集群状态
srvctl status database -d orcl
附录
1、ORACLE 12.2认证的OS软件版本
|
序号 |
组件 |
|
1 |
Oracle Linux 7 with the Unbreakable Enterprise kernel: |
|
2 |
Oracle Linux 7 with the Red Hat Compatible kernel: |
|
3 |
Oracle Linux 6.4 with the Unbreakable Enterprise kernel
2: |
|
4 |
Oracle Linux 6 with the Red Hat Compatible kernel: |
|
5 |
Red Hat Enterprise Linux 7: 3.10.0-123.el7.x86_64 or later |
|
6 |
Red Hat Enterprise Linux 6: 2.6.32-358.el6.x86_64 or later |
|
7 |
SUSE Linux Enterprise Server 12: 3.12.28-4-default or later |
|
8 |
SUSE Linux Enterprise Server 11 SP2: 3.0.13-0.27 or later |
2、集群管理命令
启动Cluster集群(root)
/u01/app/12.2.0/grid/bin/crsctl start cluster [-all | -n <server name>]
关闭Cluster集群(root)
/u01/app/12.2.0/grid/bin/crsctl start cluster [-all | -n <server name>]
-all 启动集群内所有节点
-n <server name> 启动指定节点
查看集群状态(grid)
crsctl status res -t
查看数据库集群状态(oracle)
srvctl status database -d orcl
查看Listener状态
srvctl status listener
关闭实例
srvctl stop instance -d orcl -n <server name>
启动实例
srvctl start instance -d orcl -n <server name>
关闭数据库
srvctl stop database -d orcl
启动数据库
srvctl start database -d orcl
