1、明确任务

准备一台RHEL5的PC机充当SAN设备，san（同时充当DNS和授时系统）

准备两台RHEL5的PC机充当RAC节点，r1和r2

将RAC安装在ASM磁盘上，包括OCR、VOTING、库

2、修改r1和r2的内核参数与shell限制

在r1和r2上分别进行

vim /etc/sysctl.conf
------------------------------------------
kernel.shmmax = 4294967296
kernel.shmmni = 4096
kernel.shmall = 2097152
kernel.sem = 250 32000 100 128
fs.file-max = 6815744
fs.aio-max-nr = 1048576
net.ipv4.ip_local_port_range = 9000 65500
net.core.rmem_default = 262144
net.core.rmem_max = 4194304
net.core.wmem_default = 262144
net.core.wmem_max = 1048586

sysctl -p

vim /etc/security/limits.conf
------------------------------------------
#Oracle configure shell parameters
oracle soft nofile 65536
oracle hard nofile 65536
oracle soft nproc 16384
oracle hard nproc 16384

grid soft nofile 65536
grid hard nofile 65536
grid soft nproc 16384
grid hard nproc 16384

3、配置san、r1、r2的网络

在san、r1、r2上分别配置

vim/etc/hosts

127.0.0.1san localhost.localdomain localhost
::1localhost6.localdomain6 localhost6

192.168.33.100san

192.168.33.101 r1
192.168.33.102 r2

192.168.33.201 r1-vip#不要配置在网卡上
192.168.33.202 r2-vip#不要配置在网卡上

10.0.0.1 r1-priv
10.0.0.2 r2-priv

4、在san上配置DNS

yum install bind bind-chroot caching-nameserver

cd /var/named/chroot/etc

cp -p named.caching-nameserver.conf named.conf
vim /var/named/chroot/etc/named.conf

options {
listen-on port 53 { any; };
listen-on-v6 port 53 { ::1; };
directory "/var/named";
dump-file "/var/named/data/cache_dump.db";
statistics-file "/var/named/data/named_stats.txt";
memstatistics-file "/var/named/data/named_mem_stats.txt";

allow-query { any; };
allow-query-cache { any; };
};
logging {
channel default_debug {
file "data/named.run";
severity dynamic;
};
};
view localhost_resolver {
match-clients { any; };
match-destinations { any; };
recursion yes;
include "/etc/named.zones";
};

cp -p named.rfc1912.zones named.zones

vim /var/named/chroot/etc/named.zones

// named.rfc1912.zones:
//
// Provided by Red Hat caching-nameserver package
//
// ISC BIND named zone configuration for zones recommended by
// RFC 1912 section 4.1 : localhost TLDs and address zones
//
// See /usr/share/doc/bind*/sample/ for example named configuration files.
//
zone "." IN {
type hint;
file "named.ca";
};

zone "localdomain" IN {
type master;
file "localdomain.zone";
allow-update { none; };
};

zone "localhost" IN {
type master;
file "localhost.zone";
allow-update { none; };
};

zone "0.0.127.in-addr.arpa" IN {
type master;
file "named.local";
allow-update { none; };
};

zone "0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.ip6.arpa" IN {
type master;
file "named.ip6.local";
allow-update { none; };
};

zone "255.in-addr.arpa" IN {
type master;
file "named.broadcast";
allow-update { none; };
};

zone "0.in-addr.arpa" IN {
type master;
file "named.zero";
allow-update { none; };
};

zone "ora11grac.com" IN {
type master;
file "ora11grac.com.zone";
allow-update { none; };
};

zone "33.168.192.in-addr.arpa" IN {
type master;
file "33.168.192.local";
allow-update { none; };
};

cd /var/named/chroot/var/named
cp -p named.zero ora11grac.com.zone
cp -p named.local 33.168.192.local

vim /var/named/chroot/var/named/ora11grac.com.zone

$TTL 86400
@ IN SOA dns.ora11grac.com. root.ora11grac.com. (
42 ; serial (d. adams)
3H ; refresh
15M ; retry
1W ; expiry
1D ) ; minimum
@ IN NS dns.ora11grac.com.
r1 IN A 192.168.33.101
r2 IN A 192.168.33.102
r1-vip IN A 192.168.33.201
r2-vip IN A 192.168.33.202
scan IN A 192.168.33.195
scan IN A 192.168.33.196
scan IN A 192.168.33.197

vim /var/named/chroot/var/named/33.168.192.local

$TTL 86400
@ IN SOA dns.ora11grac.com. root.ora11grac.com. (
1997022700 ; Serial
28800 ; Refresh
14400 ; Retry
3600000 ; Expire
86400 ) ; Minimum
@ IN NS dns.ora11grac.com.
101 IN PTR r1.ora11grac.com.
102 IN PTR r2.ora11grac.com.
201 IN PTR r1-vip.ora11grac.com.
202 IN PTR r2-vip.ora11grac.com.
195 IN PTR scan.ora11grac.com.
196 IN PTR scan.ora11grac.com.
197 IN PTR scan.ora11grac.com.

service named restart

chkconfig named on

完成后在客户端测试

vim /etc/resolve.conf

search ora11grac.com

nameserver 192.168.33.100

然后ping scan.ora11grac.com，或者nslookup scan.ora11grac.com

5、添加用户和组在r1、r2

在r1、r2分别

groupadd -g 501 oinstall
groupadd -g 502 dba
groupadd -g 503 oper
groupadd -g 504 asmadmin
groupadd -g 505 asmdba
groupadd -g 506 asmoper

useradd -u 501 -g oinstall -G dba,oper,asmdba oracle
useradd -u 502 -g oinstall -G dba,asmadmin,asmdba,asmoper grid

在r1、r2上修改配置文件

vim /home/grid/.bashrc

export ORACLE_BASE=/u01/app/oracle
export ORACLE_HOME=/u01/app/11.2.0/grid
export PATH=$ORACLE_HOME/bin:$PATH
export ORACLE_OWNER=oracle
export ORACLE_SID=+ASM1 #r2节点为 export ORACLE_SID=+ASM2
export ORACLE_TERM=vt100
export THREADS_FLAG=native
export LD_LIBRARY_PATH=$ORACLE_HOME/lib:$LD_LIBRARY_PATH
export PATH=$ORACLE_HOME/bin:$PATH
export LANG=en_US
alias sqlplus='rlwrap sqlplus'
alias lsnrctl='rlwrap lsnrctl'
alias asmcmd='rlwrap asmcmd'

vim /home/oracle/.bashrc

export ORACLE_BASE=/u01/app/oracle
export ORACLE_HOME=$ORACLE_BASE/product/11.2.0/db_1
export PATH=$ORACLE_HOME/bin:$PATH
export ORACLE_OWNER=oracle
export ORACLE_SID=orcl1 #rac2节点为 export ORACLE_SID=orcl2
export ORACLE_TERM=vt100
export THREADS_FLAG=native
export LD_LIBRARY_PATH=$ORACLE_HOME/lib:$LD_LIBRARY_PATH
export PATH=$ORACLE_HOME/bin:$PATH
export EDITOR=vi
export SQLPATH=/home/oracle
export LANG=en_US
alias sqlplus='rlwrap sqlplus'
alias lsnrctl='rlwrap lsnrctl'
alias rman='rlwrap rman'
alias dgmgrl='rlwrap dgmgrl'

6、创建目录在r1、r2

在r1、r2分别

mkdir -p /u01/app/11.2.0/grid
chown -R grid:oinstall /u01
mkdir /u01/app/oracle
chown -R oracle:oinstall /u01/app/oracle
chmod -R 775 /u01/

7、配置grid、oracle用户下的双机互信（需要单步）

在r1上

su - grid

ssh-keygen -t rsa

ssh-keygen -t dsa

cd .ssh

cat *.pub > authorized_keys

在r2上

su - grid

ssh-keygen -t rsa

ssh-keygen -t dsa

cd .ssh

cat *.pub > authorized_keys

在r1上

scp authorized_keys grid@r2:/home/grid/.ssh/k1

在r2上

cat /home/grid/.ssh/k1 >> authorized_keys

scp authorized_keys grid@r1:/home/grid/.ssh/

在r1上

su - oracle

ssh-keygen -t rsa

ssh-keygen -t dsa

cd .ssh

cat *.pub > authorized_keys

在r2上

su - oracle

ssh-keygen -t rsa

ssh-keygen -t dsa

cd .ssh

cat *.pub > authorized_keys

在r1上

scp authorized_keys oracle@r2:/home/oracle/.ssh/k1

在r2上

cat /home/oracle/.ssh/k1 >> authorized_keys

scp authorized_keys oracle@r1:/home/oracle/.ssh/

分别在r1、r2上测试双机信任

ssh r1 date

ssh r2 date

ssh r1-priv date

ssh r2-priv date

8、配置NTP服务器在san上（不在san上做在其它机器上做也可以）

在san上

vim /etc/sysconfig/ntpd
# Drop root to id 'ntp:ntp' by default.
OPTIONS="-x -u ntp:ntp -p /var/run/ntpd.pid"

# Set to 'yes' to sync hw clock after successful ntpdate
SYNC_HWCLOCK=no

# Additional options for ntpdate

vim /etc/ntp.conf
restrict default nomodify
restrict 127.0.0.1
server 127.127.1.0
fudge 127.127.1.0 stratum 8
driftfile /var/lib/ntp/drift
broadcastdelay 0.008

启动ntpd服务

service ntpd restart

chkconfig ntpd on

在客户端r1、r2

ntpdate san

然后使用crontab -e

*/5 * * * * /usr/sbin/ntpdate san

建议拷贝配置文件到r1、r2，然后在r1、r2也启动ntpd，因为grid安装的时候会进行校验

9、配置集中存储san

在san上

yum install scsi-target-utils -y
vim /etc/tgt/targets.conf

backing-store /dev/sdb1
backing-store /dev/sdc1
backing-store /dev/sdd1
backing-store /dev/sde1
backing-store /dev/sdf1
backing-store /dev/sdf2
backing-store /dev/sdf3

service tgtd start
tgtadm --lld iscsi -m target --op show
chkconfig tgtd on

在r1和r2上

yum install iscsi-initiator-utils -y
service iscsid start
iscsiadm -m discovery -t st -p san
service iscsi start
chkconfig iscsid on
chkconfig iscsi on

10、配置udev绑定裸设备在r1、r2上

在r1、r2上

ll /dev/sd*

brw-r----- 1 root disk 8,0 Jan4 19:31 /dev/sda
brw-r----- 1 root disk 8,1 Jan4 11:31 /dev/sda1
brw-r----- 1 root disk 8,2 Jan4 19:31 /dev/sda2
brw-r----- 1 root disk 8, 16 Jan4 13:31 /dev/sdb
brw-r----- 1 root disk 8, 17 Jan4 16:30 /dev/sdb1
brw-r----- 1 root disk 8, 32 Jan4 13:31 /dev/sdc
brw-r----- 1 root disk 8, 33 Jan4 16:30 /dev/sdc1
brw-r----- 1 root disk 8, 48 Jan4 13:32 /dev/sdd
brw-r----- 1 root disk 8, 49 Jan4 16:30 /dev/sdd1
brw-r----- 1 root disk 8, 64 Jan4 13:32 /dev/sde
brw-r----- 1 root disk 8, 65 Jan4 16:30 /dev/sde1
brw-r----- 1 root disk 8, 80 Jan4 13:32 /dev/sdf
brw-r----- 1 root disk 8, 81 Jan4 16:29 /dev/sdf1
brw-r----- 1 root disk 8, 96 Jan4 13:33 /dev/sdg
brw-r----- 1 root disk 8, 97 Jan4 16:29 /dev/sdg1
brw-r----- 1 root disk 8, 98 Jan4 16:29 /dev/sdg2

vim /etc/udev/rules.d/60-raw.rules

ACTION=="add", KERNEL=="sdf1", RUN+="/bin/raw /dev/raw/raw1 %N"
ACTION=="add", KERNEL=="sdg1", RUN+="/bin/raw /dev/raw/raw2 %N"
ACTION=="add", KERNEL=="sdg2", RUN+="/bin/raw /dev/raw/raw3 %N"
KERNEL=="raw1", OWNER="grid", GROUP="asmadmin", MODE="660"
KERNEL=="raw2", OWNER="grid", GROUP="asmadmin", MODE="660"
KERNEL=="raw3", OWNER="grid", GROUP="asmadmin", MODE="660"

11、安装依赖包

yum install binutils compat-libstdc++ elfutils-libelf elfutils-libelf-devel gcc gcc-c++ glibc glibc-common glibc-devel glibc-headers ksh libaio libaio-devel libgcc libstdc++ libstdc++-devel make sysstat unixODBC unixODBC-devel -y

12、安装ASM的RPM包（务必注意版本和平台）

rpm -ivh oracleasm-2.6.18-164.el5-2.0.5-1.el5.x86_64.rpm oracleasm-support-2.1.7-1.el5.x86_64.rpm oracleasmlib-2.0.4-1.el5.x86_64.rpm

13、配置ASM

在r1上

/etc/init.d/oracleasm configure

用户指定：grid

组指定：asmadmin

其余都是yes

建立oracleasm磁盘

/etc/init.d/oracleasm createdisk V1 /dev/sdb1

/etc/init.d/oracleasm createdisk V2 /dev/sdc1

/etc/init.d/oracleasm createdisk V3 /dev/sdd1

/etc/init.d/oracleasm createdisk V4 /dev/sde1

在r2上进行扫描

/etc/init.d/oracleasm scandisks

14、用grid用户安装grid校验

./runcluvfy.sh stage -pre crsinst -n r1,r2 -fixup -verbose

15、安装grid

在grid用户下

./runInstaller

在Instalaation Type选择

Install and Configure Grid Infrustructure for a Cluster

在Product Languages选择

Advanced Installation

在Grid Plug and Play Information填写

Cluster Name: r-cluster

SCAN Name: scan.ora11grac.com

SCAN Port: 1521

Configure GNS不勾选

在Cluster Node Information填写

r1.ora11grac.comr1-vip.ora11grac.com

r2.ora11grac.comr2-vip.ora11grac.com

这个vip不要使用ifconfig去添加

在Specify Network Interface Usage对应填写

eth0192.168.33.0Public

eth210.0.0.0Private

在Storage Option Information选择

Automatic Storage Management(ASM)

在Create ASM Disk Group

选择刚刚认出的raw1, raw2, raw3组成Disk Group Name为grid

在Specify ASM Password

填写用户口令

在Failure Isolation Support选择

Do not use Intelligent Platform Management(IPMI)

在Privileged Operating System Groups填写：

ASM Database Administrator (OSDBA) Group: asmdba

ASM Instance Administration Operator (OSOPER) Group: asmoper

ASM Instance Administrator (OSASM) Group：asmadmin

在Specify Installation Location中填写：

Oracle Base: /u01/app/oracle

Software Location: /u01/app/oracle/product/11.2.0/grid

在Create Inventory中填写：

Inventory Directory: /u01/app/oraInventory

oraInventory Group Name: oinstall

安装完成后

在r1运行

/u01/app/oraInventory/orainstRoot.sh

在r2运行

/u01/app/oraInventory/orainstRoot.sh

在r1运行

/u01/app/11.2.0/grid/root.sh

在r2运行

/u01/app/11.2.0/grid/root.sh

16、安装数据库

先要确保所有的重要服务都是ONLINE的

crs_stat -t

一般oc4j会不在线，那就

srvctl enable oc4j

srvctl start oc4j

olsnodes -n

出现r1，r2

这样就OK了

（gsd服务不在线不影响库软件的安装和库的建立）

用grid用户，开启asmca，创建一个ASM磁盘组+ASM

在r1切换到

su - oracle

./runInstaller

在Select Installation Option

选择Install database software only，只安装数据库引擎软件

在Node Selection

勾选r1，r2两个节点

在Select Database Edition选择

Enterprise Edition

在Specify Installation Location填写

Oracle Base: /u01/app/Oracle

Software Location: /u01/app/oracle/product/11.2.0/dbhome_1

执行完毕后运行，在r1、r2运行

/u01/app/oracle/product/11.2.0/db_1/root.sh

完成。