本次安装选择VMmare Workstation 10.0.1 ，创建四台虚拟机。如下：

Openfiler （openfileresa-2.99.1-x86_64）

RAC节点  （OracleLinux-R6-U4-Server-x86_64）

1. 三台机器都配置/etc/hosts，内容如下

#node1
192.168.199.10    11grac1.localdomain.com               11grac1
192.168.199.20    11grac1-vip.localdomain.com           11grac1-vip
192.168.10.10     11grac1-priv.localdomain.com          11grac1-priv
#node2
192.168.199.11    11grac2.localdomain.com               11grac2
192.168.199.21    11grac2-vip.localdomain.com           11grac2-vip
192.168.10.11     11grac2-priv.localdomain.com          11grac2-priv
#node3
192.168.199.12    11grac3.localdomain.com               11grac3
192.168.199.22    11grac3-vip.localdomain.com           11grac3-vip
192.168.10.12     11grac3-priv.localdomain.com          11grac3-priv
#scanip
192.168.199.30    scan-cluster.localdomain.com          scan-cluster

2.三台机器都关闭防火墙

chkconfig iptables off
service iptables stop

3.关闭SeLinux

vi /etc/sysconfig/selinux
SELINUX=enforcing 改为 SELINUX=disabled

4.配置DNS服务器（在11grac3服务器上）

4.1 安装rpm包

[root@localhost Packages]# rpm -ivh bind-9.8.2-0.17.rc1.0.2.el6.x86_64.rpm 
warning: bind-9.8.2-0.17.rc1.0.2.el6.x86_64.rpm: Header V3 RSA/SHA256 Signature, key ID ec551f03: NOKEY
Preparing...                ########################################### [100%]
   1:bind                   ########################################### [100%]

4.2 编辑/etc/named.conf,替换成如下内容

options {
     directory      "/var/named";
};

zone "." IN {
     type hint;
     file "named.ca";
};

zone "localhost" IN {
        type master;
        file "named.loopback";
};

zone "0.0.127.in-addr.apra" IN {
        type master;
        file "named.localhost";
};

zone "localdomain.com" IN {
       type master;
       file "localdomain.com.zone";
};

4.3 在/var/named/目录下，新建localdomain.com.zone文件，内容如下

$TTL 1D
$ORIGIN localdomain.com.
@     IN SOA     ns.localdomain.com.    admin (
                         0     ; serial
                         1D     ; refresh
                         1H     ; retry
                         1W     ; expire
                         3H )     ; minimum
                     IN         NS     ns
ns                   IN         A       172.168.199.12
11grac3              IN         A       192.168.199.12
11grac1              IN         A       192.168.199.10
11grac2              IN         A       192.168.199.11
scan-cluster         IN         A       192.168.199.30

4.4 启动DNS服务器

[root@11grac3 etc]# chkconfig named on
[root@localhost named]# service named restart
Stopping named:                                            [  OK  ]
Starting named:                                            [  OK  ]

如果发生了错误，尝试执行如下命令试试

[root@localhost named]# rndc-confgen -r /dev/urandom -a
wrote key file "/etc/rndc.key"

4.5 修改各服务器dns指向

[root@localhost named]# vi /etc/resolv.conf

添加如下内容

search localdomain.com
nameserver 192.168.199.12

注意：这样设置完DNS后，网络重启后就失效啦。使用如下方法

 vi /etc/sysconfig/network-scripts/ifcfg-eth0 
 #添加如下两行
 DNS1=192.168.199.12
 DOMAIN=localdomain.com

重启网络，/etc/resolv.conf中将自动完成DNS的设置。

4.6 验证dns

[root@localhost named]# nslookup 11grac1
Server:		192.168.199.12
Address:	192.168.199.12#53

Name:	11grac1.localdomain.com
Address: 192.168.199.10

[root@localhost named]# nslookup 11grac2
Server:		192.168.199.12
Address:	192.168.199.12#53

Name:	11grac2.localdomain.com
Address: 192.168.199.11

[root@localhost named]# nslookup 11grac3
Server:		192.168.199.12
Address:	192.168.199.12#53

Name:	11grac3.localdomain.com
Address: 192.168.199.12

[root@localhost named]# nslookup scan-cluster
Server:		192.168.199.12
Address:	192.168.199.12#53

Name:	scan-cluster.localdomain.com
Address: 192.168.199.30

5. 建立用户、建立目录、修改参数文件等 （脚本见附件）

[root@localhost scripts]# ./1preusers.sh 
[root@localhost scripts]# ./2predir.sh 
[root@localhost scripts]# ./3prelimits.sh 
[root@localhost scripts]# ./4prelogin.sh 
[root@localhost scripts]# ./5preprofile.sh 
[root@localhost scripts]# ./6presysctl.sh

6. 建立grid、oracle用户的无密码登录

6.1 在三台机器上都执行如下命令

[root@11grac1 scripts]# su - oracle
11grac1-> ssh-keygen -t rsa #后面都直接回车
Generating public/private rsa key pair.
Enter file in which to save the key (/home/oracle/.ssh/id_rsa): 
Created directory '/home/oracle/.ssh'.
Enter passphrase (empty for no passphrase): 
Enter same passphrase again: 
Your identification has been saved in /home/oracle/.ssh/id_rsa.
Your public key has been saved in /home/oracle/.ssh/id_rsa.pub.
The key fingerprint is:
31:43:2d:7e:30:bb:c9:d9:d9:8a:01:a2:51:bc:8b:2d oracle@11grac1.localdomain.com
The key's randomart p_w_picpath is:
+--[ RSA 2048]----+
|   .    ..       |
|    o  .+ .      |
|   . . .+=       |
|  . o . o+.      |
|   = o oS* o     |
|  E o   * o .    |
|   .     o .     |
|        . .      |
|                 |
+-----------------+

6.2 在三台机器上执行如下命令

11grac1-> ssh-copy-id -i .ssh/id_rsa.pub 11grac1
The authenticity of host '11grac1 (192.168.199.10)' can't be established.
RSA key fingerprint is cc:d0:c3:de:37:33:a7:cb:91:40:c5:5f:18:07:06:9f.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added '11grac1,192.168.199.10' (RSA) to the list of known hosts.
oracle@11grac1's password: 
Now try logging into the machine, with "ssh '11grac1'", and check in:

  .ssh/authorized_keys

to make sure we haven't added extra keys that you weren't expecting.

11grac2-> ssh-copy-id -i .ssh/id_rsa.pub 11grac1    
The authenticity of host '11grac1 (192.168.199.10)' can't be established.
RSA key fingerprint is cc:d0:c3:de:37:33:a7:cb:91:40:c5:5f:18:07:06:9f.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added '11grac1,192.168.199.10' (RSA) to the list of known hosts.
oracle@11grac1's password: 
Now try logging into the machine, with "ssh '11grac1'", and check in:

  .ssh/authorized_keys

to make sure we haven't added extra keys that you weren't expecting.

11grac3-> ssh-copy-id -i .ssh/id_rsa.pub 11grac1
The authenticity of host '11grac1 (192.168.199.10)' can't be established.
RSA key fingerprint is cc:d0:c3:de:37:33:a7:cb:91:40:c5:5f:18:07:06:9f.
Are you sure you want to continue connecting (yes/no)? yes
Warning: Permanently added '11grac1,192.168.199.10' (RSA) to the list of known hosts.
oracle@11grac1's password: 
Now try logging into the machine, with "ssh '11grac1'", and check in:

  .ssh/authorized_keys

to make sure we haven't added extra keys that you weren't expecting.

6.3 以上命令会在11grac1的/home/oracle/.ssh目录下生成authorized_keys，将该文件copy到其他两机器上

11grac1-> scp authorized_keys 11grac2:/home/oracle/.ssh/
11grac1-> scp authorized_keys 11grac3:/home/oracle/.ssh/

6.4 重复6.1~6.3步骤，生成grid用户的无密码验证。

7. 验证各节点必须直接是否能无密码登录

   ssh 11grac1 date
   ssh 11grac2 date
   ssh 11grac3 date
   ssh 11grac3-priv date
   ssh 11grac2-priv date
   ssh 11grac1-priv date
   ssh 11grac1.localdomain.com date
   ssh 11grac2.localdomain.com date
   ssh 11grac3.localdomain.com date
   ssh 11grac3-priv.localdomain.com date
   ssh 11grac2-priv.localdomain.com date
   ssh 11grac1-priv.localdomain.com date

8. 挂载iscsi硬盘（每台机器上都执行）

8.1 查找iscsi硬盘

[root@11grac1 scripts]# iscsiadm -m discovery -t sendtargets -p 192.168.199.219
192.168.199.219:3260,1 iqn.2006-01.com.openfiler:fast_recover_area01
192.168.199.219:3260,1 iqn.2006-01.com.openfiler:rac_oradata01
192.168.199.219:3260,1 iqn.2006-01.com.openfiler:rac_ocrdisk03
192.168.199.219:3260,1 iqn.2006-01.com.openfiler:rac_ocrdisk02
192.168.199.219:3260,1 iqn.2006-01.com.openfiler:rac_ocrdisk01

8.2 设置开机自动挂载iscsi盘

iscsiadm -m node -T iqn.2006-01.com.openfiler:rac_oradata01 -p 192.168.199.219 --op update -n node.startup -v automatic
iscsiadm -m node -T iqn.2006-01.com.openfiler:fast_recover_area01 -p 192.168.199.219 --op update -n node.startup -v automatic
iscsiadm -m node -T iqn.2006-01.com.openfiler:rac_ocrdisk01 -p 192.168.199.219 --op update -n node.startup -v automatic
iscsiadm -m node -T iqn.2006-01.com.openfiler:rac_ocrdisk02 -p 192.168.199.219 --op update -n node.startup -v automatic
iscsiadm -m node -T iqn.2006-01.com.openfiler:rac_ocrdisk03 -p 192.168.199.219 --op update -n node.startup -v automatic

9.创建asm磁盘

9.1 安装oracleasm-support、oracleasmlib （三台机器都执行）

注：因oracle linux已经将oracleasm内置到内核中了，所以无需安装oracleasm的rpm包。如果使用非oracle Linux操作系统。请按装kmod-oracleasm-2.0.6.rh2-2.el6.x86_64.rpm 包。

[root@11grac2 /]# rpm -ivh oracleasm-support-2.1.8-1.el6.x86_64.rpm 
warning: oracleasm-support-2.1.8-1.el6.x86_64.rpm: Header V3 RSA/SHA256 Signature, key ID ec551f03: NOKEY
Preparing...                ########################################### [100%]
   1:oracleasm-support      ########################################### [100%]
[root@11grac2 /]# rpm -ivh oracleasmlib-2.0.4-1.el6.x86_64.rpm 
warning: oracleasmlib-2.0.4-1.el6.x86_64.rpm: Header V3 RSA/SHA256 Signature, key ID ec551f03: NOKEY
Preparing...                ########################################### [100%]
   1:oracleasmlib           ########################################### [100%]

[root@11grac1 Packages]# oracleasm configure -i
Configuring the Oracle ASM library driver.

This will configure the on-boot properties of the Oracle ASM library
driver.  The following questions will determine whether the driver is
loaded on boot and what permissions it will have.  The current values
will be shown in brackets ('[]').  Hitting  without typing an
answer will keep that current value.  Ctrl-C will abort.

Default user to own the driver interface []: grid
Default group to own the driver interface []: asmdba
Start Oracle ASM library driver on boot (y/n) [n]: y
Scan for Oracle ASM disks on boot (y/n) [y]: y
Writing Oracle ASM library driver configuration: done

9.2 将磁盘分区（只需单节点执行）

9.3 创建asm磁盘

[root@11grac1 ~]# ll /dev/disk/by-path/
ip-192.168.199.219:3260-iscsi-iqn.2006-01.com.openfiler:fast_recover_area01-lun-0 -> ../../sde
ip-192.168.199.219:3260-iscsi-iqn.2006-01.com.openfiler:fast_recover_area01-lun-0-part1 -> ../../sde1
ip-192.168.199.219:3260-iscsi-iqn.2006-01.com.openfiler:rac_ocrdisk01-lun-0 -> ../../sdb
ip-192.168.199.219:3260-iscsi-iqn.2006-01.com.openfiler:rac_ocrdisk01-lun-0-part1 -> ../../sdb1
ip-192.168.199.219:3260-iscsi-iqn.2006-01.com.openfiler:rac_ocrdisk02-lun-0 -> ../../sdd
ip-192.168.199.219:3260-iscsi-iqn.2006-01.com.openfiler:rac_ocrdisk02-lun-0-part1 -> ../../sdd1
ip-192.168.199.219:3260-iscsi-iqn.2006-01.com.openfiler:rac_ocrdisk03-lun-0 -> ../../sdc
ip-192.168.199.219:3260-iscsi-iqn.2006-01.com.openfiler:rac_ocrdisk03-lun-0-part1 -> ../../sdc1
ip-192.168.199.219:3260-iscsi-iqn.2006-01.com.openfiler:rac_oradata01-lun-0 -> ../../sdf
ip-192.168.199.219:3260-iscsi-iqn.2006-01.com.openfiler:rac_oradata01-lun-0-part1 -> ../../sdf1

注意以上iscsi磁盘与/dev/sd*的对应关系。

[root@11grac1 ~]# oracleasm createdisk oradata /dev/sdf1 
Writing disk header: done
Instantiating disk: done
[root@11grac1 ~]# oracleasm createdisk fra /dev/sde1 
Writing disk header: done
Instantiating disk: done
[root@11grac1 ~]# oracleasm createdisk ocr01 /dev/sdb1 
Writing disk header: done
Instantiating disk: done
[root@11grac1 ~]# oracleasm createdisk ocr02 /dev/sdd1 
Writing disk header: done
Instantiating disk: done
[root@11grac1 ~]# oracleasm createdisk ocr03 /dev/sdc1

查看磁盘

[root@11grac1 ~]# oracleasm listdisks
FRA
OCR01
OCR02
OCR03
ORADATA
[root@11grac1 ~]#

在另外两台机器上执行磁盘扫描操作

[root@11grac3 ~]# oracleasm scandisks
Reloading disk partitions: done
Cleaning any stale ASM disks...
Scanning system for ASM disks...
Instantiating disk "OCR01"
Instantiating disk "OCR03"
Instantiating disk "OCR02"
Instantiating disk "FRA"
Instantiating disk "ORADATA"

10. 禁用ntp时间同步服务

  因为11g Clusterware中已经提供了时间同步服务，所以需要禁用ntp服务。

[root@11grac3 ~]# cd /etc/
[root@11grac3 etc]# mv ntp.conf ntp.conf.bak