2.两个节点安装系统补丁（64位）
binutils-2.17.50.0.6
compat-libstdc++-33-3.2.3
compat-libstdc++-33-3.2.3（32 位）
elfutils-libelf-0.125
elfutils-libelf-devel-0.125
gcc-4.1.2
gcc-c++-4.1.2
glibc-2.5-24
glibc-2.5-24（32 位）
glibc-common-2.5
glibc-devel-2.5
glibc-devel-2.5（32 位）
glibc-headers-2.5
libaio-0.3.106
libaio-0.3.106（32 位）
libaio-devel-0.3.106
libaio-devel-0.3.106（32 位）
libgcc-4.1.2
libgcc-4.1.2（32 位）
libstdc++-4.1.2
libstdc++-4.1.2（32 位）
libstdc++-devel 4.1.2
make-3.81
sysstat-7.0.2
unixODBC-2.2.11
unixODBC-2.2.11（32 位）
unixODBC-devel-2.2.11
unixODBC-devel-2.2.11（32 位）

[root@oracle-db1 ~]# yum install -y binutils* compat-libstdc* elfutils-libelf* gcc* glibc* libaio* libgcc* libstdc++* make* sysstat* unixODBC*

[root@oracle-db2 ~]#yum install -y binutils* compat-libstdc* elfutils-libelf* gcc* glibc* libaio* libgcc* libstdc++* make* sysstat* unixODBC*

3两个节点.网络配置

cat /etc/hosts
127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
# Public IP
10.10.0.17        oracle-db1
10.10.0.18        oracle-db2

# Private IP
10.0.0.1       db1-priv
10.0.0.2       db2-priv

# Virtual IP
10.10.0.8        db1-vip
10.10.0.9        db2-vip

# Scan IP
10.10.0.10       rac-scan

4.两个节点服务器停用NTP时间同步

[root@oracle-db1 ~]#sbin/service ntpd stop

[root@oracle-db1 ~]#chkconfig ntpd off

[root@oracle-db1 ~]#mv /etc/ntp.conf /etc/ntp.conf.org

[root@oracle-db1 ~]#rm /var/run/ntpd.pid

5.禁止两个节点所有防火墙

[root@oracle-db1 ~]#service iptables stop

[root@oracle-db1 ~]#service ip6tables stop

[root@oracle-db1 ~]#chkconfig iptables off

[root@oracle-db1 ~]#chkconfig ip6tables off

6.关闭所有节点服务器的selinux

[root@oracle-db1 ~]# vi /etc/sysconfig/selinux

#SELINUX=enforcing
SELINUX=disabled

7.给两个节点创建用户，组及目录，并赋予相应的权限

[root@oracle-db1 ~]#groupadd -g 1000 oinstall
[root@oracle-db1 ~]#groupadd -g 1200 asmadmin
[root@oracle-db1 ~]#groupadd -g 1201 asmdba
[root@oracle-db1 ~]#groupadd -g 1202 asmoper
[root@oracle-db1 ~]#groupadd -g 1300 dba
[root@oracle-db1 ~]#groupadd -g 1301 oper
[root@oracle-db1 ~]#useradd -m -u 1100 -g oinstall -G asmadmin,asmdba,asmoper,dba -d /home/grid grid
[root@oracle-db1 ~]#useradd -m -u 1101 -g oinstall -G dba,oper,asmdba -d /home/oracle oracle
[root@oracle-db1 ~]#mkdir -p /home/u01/app/grid
[root@oracle-db1 ~]#mkdir -p /home/u01/app/11.2.0/grid
[root@oracle-db1 ~]#chown -R grid:oinstall /home/u01
[root@oracle-db1 ~]#mkdir -p /home/u01/app/oracle
[root@oracle-db1 ~]#chown oracle:oinstall /home/u01/app/oracle
[root@oracle-db1 ~]#chmod -R 775 /home/u01

[root@oracle-db1 ~]#passwd oracle

[root@oracle-db1 ~]#passwd grid

[root@oracle-db1 ~]# id oracle -a
uid=1101(oracle) gid=1000(oinstall) groups=1000(oinstall),1201(asmdba),1300(dba),1301(oper)
[root@oracle-db1 ~]# id grid -a
uid=1100(grid) gid=1000(oinstall) groups=1000(oinstall),1200(asmadmin),1201(asmdba),1202(asmoper),1300(dba)

8.修改两个节点的系统参数

在“/etc/sysctl.conf”文件中添加如下内容

fs.aio-max-nr = 1048576

fs.file-max = 6815744

kernel.shmmni = 4096

kernel.sem = 250 32000 100 128

net.ipv4.ip_local_port_range = 9000 65500

net.core.rmem_default = 262144

net.core.rmem_max = 4194304

net.core.wmem_default = 262144

net.core.wmem_max = 1048586

编辑完成后，执行下面命令生效：

#/sbin/sysctl -p

9.在/etc/security/limits.conf添加如下内容

grid soft nproc 2047

grid hard nproc 16384

grid soft nofile 1024

grid hard nofile 65536

oracle soft nproc 2047

oracle hard nproc 16384

oracle soft nofile 1024

oracle hard nofile 65536

10.在 /etc/pam.d/login添加

session required pam_limits.so

11.给两个节点的grid,oracle用户配置环境变量

[root@oracle-db1 ~]# su - oracle
[oracle@oracle-db1 ~]$ vi .bash_profile

ORACLE_HOSTNAME=oracle-db1; export ORACLE_HOSTNAME

ORACLE_UNQNAME=racdb; export ORACLE_UNQNAME

ORACLE_SID=racdb1; export ORACLE_SID

JAVA_HOME=/usr/local/java: export JAVA_HOME

ORACLE_BASE=/home/u01/app/oracle; export ORACLE_BASE

ORACLE_HOME=$ORACLE_BASE/product/11.2.0/db_1; export ORACLE_HOME

ORACLE_PATH=/home/u01/app/common/oracle/sql; export ORACLE_PATH

ORACLE_TERM=xterm; export ORACLE_TERM

TNS_ADMIN=$ORACLE_HOME/network/admin; export TNS_ADMIN

ORA_NLS11=$ORACLE_HOME/nls/data; export ORA_NLS11

PATH=.:${JAVA_HOME}/bin:${PATH}:$HOME/bin:$ORACLE_HOME/bin

PATH=${PATH}:/usr/bin:/bin:/usr/bin/X11:/usr/local/bin

PATH=${PATH}:/home/u01/app/common/oracle/bin

export PATH

LD_LIBRARY_PATH=$ORACLE_HOME/lib

LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:$ORACLE_HOME/oracm/lib

LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/lib:/usr/lib:/usr/local/lib

export LD_LIBRARY_PATH

CLASSPATH=$ORACLE_HOME/JRE

CLASSPATH=${CLASSPATH}:$ORACLE_HOME/jlib

CLASSPATH=${CLASSPATH}:$ORACLE_HOME/rdbms/jlib

CLASSPATH=${CLASSPATH}:$ORACLE_HOME/network/jlib

export CLASSPATH

export TEMP=/tmp

export TMPDIR=/tmp

umask 022

[oracle@oracle-db1 ~]$ su - grid
[grid@oracle-db1 ~]$ vi .bash_profile

ORACLE_SID=+ASM1; export ORACLE_SID

JAVA_HOME=/usr/local/java; export JAVA_HOME

ORACLE_BASE=/home/u01/app/grid; export ORACLE_BASE

ORACLE_HOME=/home/u01/app/11.2.0/grid; export ORACLE_HOME

ORACLE_PATH=/home/u01/app/oracle/common/oracle/sql; export ORACLE_PATH

ORACLE_TERM=xterm; export ORACLE_TERM

TNS_ADMIN=$ORACLE_HOME/network/admin; export TNS_ADMIN

ORA_NLS11=$ORACLE_HOME/nls/data; export ORA_NLS11

PATH=.:${JAVA_HOME}/bin:${PATH}:$HOME/bin:$ORACLE_HOME/bin

PATH=${PATH}:/usr/bin:/bin:/usr/bin/X11:/usr/local/bin

PATH=${PATH}:/home/u01/app/common/oracle/bin

export PATH

LD_LIBRARY_PATH=$ORACLE_HOME/lib

LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:$ORACLE_HOME/oracm/lib

LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/lib:/usr/lib:/usr/local/lib

export LD_LIBRARY_PATH

CLASSPATH=$ORACLE_HOME/JRE

CLASSPATH=${CLASSPATH}:$ORACLE_HOME/jlib

CLASSPATH=${CLASSPATH}:$ORACLE_HOME/rdbms/jlib

CLASSPATH=${CLASSPATH}:$ORACLE_HOME/network/jlib

export CLASSPATH

export TEMP=/tmp

export TMPDIR=/tmp

umask 022

在oracle-db2上需要将oracle用户的ORACLE_SID改为racdb2,grid用户的ORACLE_SID改为+ASM2

12配置共享磁盘udev

查看多链路劲的存储信息

[root@oracle-db1 ~]# multipath -ll
VOTE3 (360060e80122d0b0050402d0b00000241) dm-7 HITACHI,OPEN-V
size=1.0G features='1 queue_if_no_path' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
|- 1:0:3:6 sdi 8:128 active ready running
`- 2:0:2:6 sds 65:32 active ready running
VOTE2 (360060e80122d0b0050402d0b00000240) dm-8 HITACHI,OPEN-V
size=2.0G features='1 queue_if_no_path' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
|- 1:0:3:5 sdh 8:112 active ready running
`- 2:0:2:5 sdr 65:16 active ready running
VOTE1 (360060e80122d0b0050402d0b0000023f) dm-6 HITACHI,OPEN-V
size=1.0G features='1 queue_if_no_path' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
|- 1:0:3:4 sdg 8:96 active ready running
`- 2:0:2:4 sdq 65:0 active ready running
DATA4 (360060e80122d0b0050402d0b0000023e) dm-4 HITACHI,OPEN-V
size=200G features='1 queue_if_no_path' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
|- 1:0:3:3 sdf 8:80 active ready running
`- 2:0:2:3 sdp 8:240 active ready running
DATA3 (360060e80122d0b0050402d0b0000023d) dm-5 HITACHI,OPEN-V
size=200G features='1 queue_if_no_path' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
|- 1:0:3:2 sde 8:64 active ready running
`- 2:0:2:2 sdo 8:224 active ready running
DATA2 (360060e80122d0b0050402d0b0000023c) dm-3 HITACHI,OPEN-V
size=310G features='1 queue_if_no_path' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
|- 1:0:3:1 sdd 8:48 active ready running
`- 2:0:2:1 sdn 8:208 active ready running
DATA1 (360060e80122d0b0050402d0b0000023b) dm-2 HITACHI,OPEN-V
size=310G features='1 queue_if_no_path' hwhandler='0' wp=rw
`-+- policy='round-robin 0' prio=1 status=active
|- 1:0:3:0 sdc 8:32 active ready running
`- 2:0:2:0 sdm 8:192 active ready running

1) 在两台服务器上编辑/etc/udev/rules.d/50-udev.rules，添加如下内容：

SUBSYSTEM=="block", KERNEL=="dm-2", GROUP="asmadmin", OWNER="grid", MODE="0660"
SUBSYSTEM=="block", KERNEL=="dm-3", GROUP="asmadmin", OWNER="grid", MODE="0660"
SUBSYSTEM=="block", KERNEL=="dm-4, GROUP="asmadmin", OWNER="grid", MODE="0660"
SUBSYSTEM=="block", KERNEL=="dm-5", GROUP="asmadmin", OWNER="grid", MODE="0660"
SUBSYSTEM=="block", KERNEL=="dm-6", GROUP="asmadmin", OWNER="grid", MODE="0660"
SUBSYSTEM=="block", KERNEL=="dm-7", GROUP="asmadmin", OWNER="grid", MODE="0660"
SUBSYSTEM=="block", KERNEL=="dm-8", GROUP="asmadmin", OWNER="grid", MODE="0660"

也可以写成

SUBSYSTEM=="block", KERNEL=="dm-[2-8]", GROUP="asmadmin", OWNER="grid", MODE="0660"

2) 在两台服务器上执行下面命令，重启udev

[root@oracle-db1 ~]#/sbin/start_udev

13.安装grid软件

两个节点安装grid下的包

解压进到该目录

[root@oracle-db1 ~]# cd /home/software/grid
[root@oracle-db1 grid]# cd rpm/
[root@oracle-db1 rpm]# rpm -ivh cvuqdisk-1.0.9-1.rpm
Preparing... ########################################### [100%]
Using default group oinstall to install package
1:cvuqdisk ########################################### [100%]

[grid@oracle-db1 grid]$ xhost +
access control disabled, clients can connect from any host
[grid@oracle-db1 grid]$ ./runInstaller

在root用户执行上述弹出的脚本，要按先后顺序

[root@oracle-db1 ~]# /home/u01/app/oraInventory/orainstRoot.sh
Changing permissions of /home/u01/app/oraInventory.
Adding read,write permissions for group.
Removing read,write,execute permissions for world.

Changing groupname of /home/u01/app/oraInventory to oinstall.
The execution of the script is complete.

[root@oracle-db1 ~]# /home/u01/app/11.2.0/grid/root.sh
Performing root user operation for Oracle 11g

The following environment variables are set as:
    ORACLE_OWNER= grid
    ORACLE_HOME= /home/u01/app/11.2.0/grid

Enter the full pathname of the local bin directory: [/usr/local/bin]:
The contents of "dbhome" have not changed. No need to overwrite.
The contents of "oraenv" have not changed. No need to overwrite.
The contents of "coraenv" have not changed. No need to overwrite.

Entries will be added to the /etc/oratab file as needed by
Database Configuration Assistant when a database is created
Finished running generic part of root script.
Now product-specific root actions will be performed.
Using configuration parameter file: /home/u01/app/11.2.0/grid/crs/install/crsconfig_params
User ignored Prerequisites during installation
OLR initialization - successful
root wallet
root wallet cert
root cert export
peer wallet
profile reader wallet
pa wallet
peer wallet keys
pa wallet keys
peer cert request
pa cert request
peer cert
pa cert
peer root cert TP
profile reader root cert TP
pa root cert TP
peer pa cert TP
pa peer cert TP
profile reader pa cert TP
profile reader peer cert TP
peer user cert
pa user cert
Adding Clusterware entries to upstart
CRS-2672: Attempting to start 'ora.mdnsd' on 'oracle-db1'
CRS-2676: Start of 'ora.mdnsd' on 'oracle-db1' succeeded
CRS-2672: Attempting to start 'ora.gpnpd' on 'oracle-db1'
CRS-2676: Start of 'ora.gpnpd' on 'oracle-db1' succeeded
CRS-2672: Attempting to start 'ora.cssdmonitor' on 'oracle-db1'
CRS-2672: Attempting to start 'ora.gipcd' on 'oracle-db1'
CRS-2676: Start of 'ora.cssdmonitor' on 'oracle-db1' succeeded
CRS-2676: Start of 'ora.gipcd' on 'oracle-db1' succeeded
CRS-2672: Attempting to start 'ora.cssd' on 'oracle-db1'
CRS-2672: Attempting to start 'ora.diskmon' on 'oracle-db1'
CRS-2676: Start of 'ora.diskmon' on 'oracle-db1' succeeded
CRS-2676: Start of 'ora.cssd' on 'oracle-db1' succeeded

ASM created and started successfully.

Disk Group CRS created successfully.

clscfg: -install mode specified
Successfully accumulated necessary OCR keys.
Creating OCR keys for user 'root', privgrp 'root'..
Operation successful.
CRS-4256: Updating the profile
Successful addition of voting disk 956d6cc946fc4f75bfdabd0bb28afd97.
Successful addition of voting disk 9d7f8b1ba3b34f4bbf89c41eabb7969a.
Successful addition of voting disk 7e81e9e8a00a4fa1bffd65fa3d7069c0.
Successfully replaced voting disk group with +CRS.
CRS-4256: Updating the profile
CRS-4266: Voting file(s) successfully replaced
## STATE    File Universal Id                File Name Disk group
-- -----    -----------------                --------- ---------
1. ONLINE   956d6cc946fc4f75bfdabd0bb28afd97 (/dev/mapper/VOTE1) [CRS]
2. ONLINE   9d7f8b1ba3b34f4bbf89c41eabb7969a (/dev/mapper/VOTE2) [CRS]
3. ONLINE   7e81e9e8a00a4fa1bffd65fa3d7069c0 (/dev/mapper/VOTE3) [CRS]
Located 3 voting disk(s).
CRS-2672: Attempting to start 'ora.asm' on 'oracle-db1'
CRS-2676: Start of 'ora.asm' on 'oracle-db1' succeeded
CRS-2672: Attempting to start 'ora.CRS.dg' on 'oracle-db1'
CRS-2676: Start of 'ora.CRS.dg' on 'oracle-db1' succeeded
Configure Oracle Grid Infrastructure for a Cluster ... succeeded

[root@oracle-db2 ~]# /home/u01/app/oraInventory/orainstRoot.sh
Changing permissions of /home/u01/app/oraInventory.
Adding read,write permissions for group.
Removing read,write,execute permissions for world.

Changing groupname of /home/u01/app/oraInventory to oinstall.
The execution of the script is complete.

[root@oracle-db2 ~]# /home/u01/app/11.2.0/grid/root.sh
Performing root user operation for Oracle 11g

The following environment variables are set as:
    ORACLE_OWNER= grid
    ORACLE_HOME= /home/u01/app/11.2.0/grid

Enter the full pathname of the local bin directory: [/usr/local/bin]:
   Copying dbhome to /usr/local/bin ...
   Copying oraenv to /usr/local/bin ...
   Copying coraenv to /usr/local/bin ...

Creating /etc/oratab file...
Entries will be added to the /etc/oratab file as needed by
Database Configuration Assistant when a database is created
Finished running generic part of root script.
Now product-specific root actions will be performed.
Using configuration parameter file: /home/u01/app/11.2.0/grid/crs/install/crsconfig_params
Creating trace directory
User ignored Prerequisites during installation
OLR initialization - successful
Adding Clusterware entries to upstart
CRS-4402: The CSS daemon was started in exclusive mode but found an active CSS daemon on node oracle-db1, number 1, and is terminating
An active cluster was found during exclusive startup, restarting to join the cluster
Configure Oracle Grid Infrastructure for a Cluster ... succeeded

14.grid用户创建ASM磁盘

[grid@oracle-db1 ~]$ asmca

15.安装Oracle11g软件

Error in invoking target 'install' of makefile '/u01/app/oracle/product/dbhome_1/ctx/lib/ins_ctx.mk'.

如果安装时弹出该错误，什么系统包没有装完整，可能是32的没有装好，64位的软件要包含安装32位的系统包

yum install -y glibc-devel*

到94%时会弹出脚本框执行相应的脚本，按顺序执行

[root@oracle-db1 ~]# /home/u01/app/oracle/product/11.2.0/db_1/root.sh
Performing root user operation for Oracle 11g

The following environment variables are set as:
ORACLE_OWNER= oracle
ORACLE_HOME= /home/u01/app/oracle/product/11.2.0/db_1

Enter the full pathname of the local bin directory: [/usr/local/bin]:
The contents of "dbhome" have not changed. No need to overwrite.
The contents of "oraenv" have not changed. No need to overwrite.
The contents of "coraenv" have not changed. No need to overwrite.

Entries will be added to the /etc/oratab file as needed by
Database Configuration Assistant when a database is created
Finished running generic part of root script.
Now product-specific root actions will be performed.
Finished product-specific root actions.

[root@oracle-db2 ~]# /home/u01/app/oracle/product/11.2.0/db_1/root.sh
Performing root user operation for Oracle 11g

The following environment variables are set as:
ORACLE_OWNER= oracle
ORACLE_HOME= /home/u01/app/oracle/product/11.2.0/db_1

Enter the full pathname of the local bin directory: [/usr/local/bin]:
The contents of "dbhome" have not changed. No need to overwrite.
The contents of "oraenv" have not changed. No need to overwrite.
The contents of "coraenv" have not changed. No need to overwrite.

Entries will be added to the /etc/oratab file as needed by
Database Configuration Assistant when a database is created
Finished running generic part of root script.
Now product-specific root actions will be performed.
Finished product-specific root actions.