Oracle 12cR2 RAC集群安裝指南

 

一、環境介紹

服務器	vSphere 6虛擬機
操作系統	OEL 7.0 with Kernal 3.8.13-35.3.1.el7uek
硬件配置	CPU 32 Cores 內存 96 G 硬盤 150 G

 

二、IP規划

主機名	Public IP	Private IP	VIP	Scan IP
oraclerac8281	172.16.82.81	192.168.82.81	172.16.82.84	172.16.82.87
oraclerac8282	172.16.82.82	192.168.82.82	172.16.82.85
oraclerac8283	172.16.82.83	192.168.82.83	172.16.82.86

 

三、存儲規划

硬盤	ASM Disk類型	大小
/dev/sdb	OCR/Voting Disk/GIMR	50G
/dev/sdc	OCR/Voting Disk/GIMR	50G
/dev/sdd	OCR/Voting Disk/GIMR	50G
/dev/sde	DATA	2T
/dev/sdf	RECOVERY	1T

 

四、系統配置（每個節點）

1、禁用Selinux/Firewall

sed -i '/^SELINUX=.*/ s//SELINUX=disabled/' /etc/selinux/config

 

echo "Disable Firewall"

systemctl disable firewalld

 

2、配置本地Yum庫

cat > /etc/yum.repos.d/local.repo <<!

[rhel-media]

name=RHEL-\$releasever - Media

baseurl=file:///media

enabled=1

gpgcheck=0

gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-redhat-release

!

 

3、配置Host解析

sed -i 3,\$d /etc/hosts

cat >> /etc/hosts <<!

#Public IP

172.16.82.81     oraclerac8281

172.16.82.82     oraclerac8282

172.16.82.83     oraclerac8283

 

#Private IP

192.168.82.81    oraclerac8281-priv

192.168.82.82    oraclerac8282-priv

192.168.82.83    oraclerac8283-priv

 

#VIP

172.16.82.84     oraclerac8281-vip

172.16.82.85     oraclerac8282-vip

172.16.82.86     oraclerac8283-vip

 

#Scan-IP

172.16.82.87     oraclerac-act-scan

!

 

4、依賴包安裝

yum -y install binutils compat-libcap1 compat-libstdc++-33 \

 compat-libstdc++-33*.i686 gcc gcc-c++ glibc glibc*.i686 \

 glibc-devel glibc-devel*.i686 ksh libaio libaio-*.i686 \

 libaio-devel libaio-devel*.i686 libgcc libgcc*.i686 \

 libstdc++ libstdc++-*.i686 libstdc++-devel libstdc++devel*.i686 \

 libXi libXi*.i686 libXtst libXtst*.i686 make sysstat \

 unixODBC unixODBC-devel unixODBC*.i686 zlib-devel zlib-devel.i686 \

 nfs-utils sysstat chrony smartmontools net-tools zip unzip

 

5、系統參數配置

MEMTOTAL=$(free -b | sed -n '2p' | awk '{print $2}')

let SHMMAX=$MEMTOTAL*4/5

SHMMNI=4096

let SHMALL=$MEMTOTAL/4096

echo SHMMAX:$SHMMAX SHMALL:$SHMALL

 

cat > /etc/sysctl.d/98-oracle.conf <<!

fs.aio-max-nr = 1048576

fs.file-max = 6815744

kernel.shmall = $SHMALL

kernel.shmmax = $SHMMAX

kernel.shmmni = 4096

kernel.sem = 250 32000 100 128

net.ipv4.ip_local_port_range = 9000 65500

net.core.rmem_default = 262144

net.core.rmem_max = 4194304

net.core.wmem_default = 262144

net.core.wmem_max = 1048576

kernel.panic_on_oops = 1

net.ipv4.conf.all.rp_filter = 2

net.ipv4.conf.default.rp_filter = 2

!

/sbin/sysctl -p /etc/sysctl.d/98-oracle.conf

 

6、添加用戶和組

groupadd -g 54321 oinstall

groupadd -g 54322 dba

groupadd -g 54323 oper

groupadd -g 54324 backupdba

groupadd -g 54325 dgdba

groupadd -g 54326 kmdba

groupadd -g 54327 asmdba

groupadd -g 54328 asmoper

groupadd -g 54329 asmadmin

groupadd -g 54330 racdba

 

useradd -u 54321 -g oinstall -G dba,asmdba,backupdba,dgdba,kmdba,racdba,oper oracle

useradd -u 54331 -g oinstall -G dba,oper,backupdba,dgdba,kmdba,asmdba,asmoper,asmadmin,racdba grid

 

echo "grid" | passwd --stdin grid

echo "oracle:oracle" | chpasswd

 

7、創建目錄並設置權限

mkdir -p /u01/app/grid

mkdir -p /u01/app/12.2.0/grid

mkdir -p /u01/app/oracle

mkdir -p /u01/app/oracle/product/12.2.0/db_1

mkdir -p /u01/app/oraInventory

chown -R grid:oinstall /u01/app

chown -R oracle:oinstall /u01/app/oracle

chmod -R 775 /u01/app

 

8、修改用戶安全限制

cat > /etc/security/limits.d/oracle-database-server-12cR2-preinstall.conf <<!

oracle     soft     nofile   2048

oracle     hard     nofile   65536

oracle     soft     nproc    16384

oracle     hard     nproc    16384

oracle     soft     stack    10243

oracle     hard     stack    32768

oracle     hard     memlock  134217728

oracle     soft     memlock  134217728

grid       soft     nofile   2048

grid       hard     nofile   65536

grid       soft     nproc    16384

grid       hard     nproc    16384

grid       soft     stack    10243

grid       hard     stack    32768

grid       hard     memlock  134217728

grid       soft     memlock  134217728!

 

9、添加PAM模塊

cat >> /etc/pam.d/login <<!

session   required    pam_limits.so

!

 

10、禁用透明大頁

cat >> /etc/rc.local <<!

if test -f /sys/kernel/mm/redhat_transparent_hugepage/enabled; then

  echo never > /sys/kernel/mm/redhat_transparent_hugepage/enabled

fi

!

 

11、配置Grid環境變量

sed -i 13,\$d ~grid/.bash_profile

cat >> ~grid/.bash_profile <<!

export ORACLE_BASE=/u01/app/grid

export ORACLE_HOME=/u01/app/12.2.0/grid

export ORACLE_SID=+ASM$(STR=$(hostname); echo ${STR: -1})

export PATH=\$ORACLE_HOME/bin:\$ORACLE_HOME/OPatch:\$PATH

export LD_LIBRARY_PATH=\$ORACLE_HOME/lib:\$LD_LIBRARY_PATH

!

cat ~grid/.bash_profile

 

12、配置Oracle環境變量

sed -i 13,\$d ~oracle/.bash_profile

cat >> ~oracle/.bash_profile <<!

export ORACLE_BASE=/u01/app/oracle

export ORACLE_HOME=\$ORACLE_BASE/product/12.2.0/db_1

export ORACLE_SID=orcl1$(STR=$(hostname); echo ${STR: -1})

export NLS_LANG=AMERICAN_AMERICA.AL32UTF8

export PATH=\$ORACLE_HOME/bin:\$ORACLE_HOME/OPatch:\$PATH

export LD_LIBRARY_PATH=\$ORACLE_HOME/lib:\$LD_LIBRARY_PATH

!

cat ~oracle/.bash_profile

 

13、配置Oracle/Grid互信（只需在一個節點執行）

Oracle提供的database安裝包里有腳本可以配置互信，比較方便。

su - oracle -c "unzip -qo /tmp/linuxx64_12201_database.zip -d /u01/app"

 

/u01/app/database/sshsetup/sshUserSetup.sh -user oracle -hosts "oraclerac8281 oraclerac8282 oraclerac8283" -advanced -noPromptPassphrase

 

/u01/app/database/sshsetup/sshUserSetup.sh -user grid -hosts "oraclerac8281 oraclerac8282 oraclerac8283" -advanced -noPromptPassphrase

 

配置完成后使用下面命令測試用戶連通性

su - oracle -c "ssh oraclerac8281 date; ssh oraclerac8282 date; ssh oraclerac8283 date; exit"

 

su - grid -c "ssh oraclerac8281 date; ssh oraclerac8282 date; ssh oraclerac8283 date; exit"

 

分發cvuqdisk安裝包到所有節點/tmp下

su - oracle -c "

scp /u01/app/database/rpm/cvuqdisk-1.0.10-1.rpm oraclerac8281:/tmp

scp /u01/app/database/rpm/cvuqdisk-1.0.10-1.rpm oraclerac8282:/tmp

scp /u01/app/database/rpm/cvuqdisk-1.0.10-1.rpm oraclerac8283:/tmp

"

 

14、配置NOZEROCONF

echo "NOZEROCONF=yes" >> /etc/sysconfig/network

grep ^NOZEROCONF /etc/sysconfig/network

 

15、修改nsswitch.conf

sed -i "/^hosts:/chosts:      files dns nis" /etc/nsswitch.conf

grep ^hosts /etc/nsswitch.conf

 

16、配置tmpfs掛載大小

默認tmpfs掛載的大小為內存的一半，SGA設置如果超過內存一半，需要重新配置。

mem=$(free -m | grep ^Mem | awk '{print$2}'); let sga=mem/100*64

sed -i /^tmpfs/d /etc/fstab

echo "tmpfs /dev/shm tmpfs size=${sga}M,mode=1777,nosuid,nodev,strictatime 0 0" >> /etc/fstab

mount -o loop /dev/shm

df -h /dev/shm

 

17、安裝CVU Disk

export CVUQDISK_GRP=asmadmin

rpm -ivh /tmp/cvuqdisk-1.0.10-1.rpm

 

18、桌面環境安裝（只需要裝一個節點）

如果RAC服務器和本地節點全端口互通，只需要安裝X Window即可，否則需要安裝桌面環境

X Window安裝"（可選）

yum -y groupinstall "X Window system"

 

Gnome桌面環境安裝（可選）

yum -y groupinstall "Server with GUI"

 

五、UDEV配置

1、獲取Disk UUID

虛擬機需要關機，設置disk.EnableUUID=TRUE。

虛擬機屬性->選項->高級->常規->配置參數

 

 

然后執行下面命令獲取掛載硬盤的UUID

for i in b c d e f #這里需要列出需要的盤符

do

echo -n "/dev/sd$i: "

/lib/udev/scsi_id --whitelisted --replace-whitespace --device=/dev/sd$i

done

 

2、配置UDEV Rule文件

使用上一步獲取到的UUID一一對應替換下面RESULT值，並自定義掛載名稱（標紅部分）。

cat > /etc/udev/rules.d/99-oracle-asmdevices.rules <<!

KERNEL=="sd*", ENV{DEVTYPE}=="disk", SUBSYSTEM=="block", PROGRAM=="/usr/lib/udev/scsi_id -g -u -d \$devnode", RESULT=="36000c29d6d2dd44e3dc77ec91b3c4b27", RUN+="/bin/sh -c 'mknod /dev/asmdisk01 b \$major \$minor; chown grid:asmadmin /dev/asmdisk01; chmod 0660 /dev/asmdisk01'"

KERNEL=="sd*", ENV{DEVTYPE}=="disk", SUBSYSTEM=="block", PROGRAM=="/usr/lib/udev/scsi_id -g -u -d \$devnode", RESULT=="36000c29f112c5cc79f547b0571cf17e6", RUN+="/bin/sh -c 'mknod /dev/asmdisk02 b \$major \$minor; chown grid:asmadmin /dev/asmdisk02; chmod 0660 /dev/asmdisk02'"

KERNEL=="sd*", ENV{DEVTYPE}=="disk", SUBSYSTEM=="block", PROGRAM=="/usr/lib/udev/scsi_id -g -u -d \$devnode", RESULT=="36000c29e113772582ecb184c62e6f470", RUN+="/bin/sh -c 'mknod /dev/asmdisk03 b \$major \$minor; chown grid:asmadmin /dev/asmdisk03; chmod 0660 /dev/asmdisk03'"

KERNEL=="sd*", ENV{DEVTYPE}=="disk", SUBSYSTEM=="block", PROGRAM=="/usr/lib/udev/scsi_id -g -u -d \$devnode", RESULT=="36000c290e51112c33800622a77a3cc0e", RUN+="/bin/sh -c 'mknod /dev/datdisk01 b \$major \$minor; chown grid:asmadmin /dev/datdisk01; chmod 0660 /dev/datdisk01'"

KERNEL=="sd*", ENV{DEVTYPE}=="disk", SUBSYSTEM=="block", PROGRAM=="/usr/lib/udev/scsi_id -g -u -d \$devnode", RESULT=="36000c29783cf6791a19b69403dd9a064", RUN+="/bin/sh -c 'mknod /dev/arcdisk01 b \$major \$minor; chown grid:asmadmin /dev/arcdisk01; chmod 0660 /dev/arcdisk01'"

!

 

cat >> /etc/udev/rules.d/99-oracle-asmdevices.rules <<!

ACTION=="add|change", ENV{ID_SERIAL}=="36000c29d6d2dd44e3dc77ec91b3c4b27", NAME="asmdisk01",  OWNER="grid", GROUP="asmadmin", MODE="0660"

ACTION=="add|change", ENV{ID_SERIAL}=="36000c29f112c5cc79f547b0571cf17e6", NAME="asmdisk02",  OWNER="grid", GROUP="asmadmin", MODE="0660"

ACTION=="add|change", ENV{ID_SERIAL}=="36000c29e113772582ecb184c62e6f470", NAME="asmdisk03",  OWNER="grid", GROUP="asmadmin", MODE="0660"

ACTION=="add|change", ENV{ID_SERIAL}=="36000c290e51112c33800622a77a3cc0e", NAME="datdisk01",  OWNER="grid", GROUP="asmadmin", MODE="0660"

ACTION=="add|change", ENV{ID_SERIAL}=="36000c29783cf6791a19b69403dd9a064", NAME="arcdisk01",  OWNER="grid", GROUP="asmadmin", MODE="0660"

!

 

3、重新加載規則

udevadm control --reload-rules

/sbin/udevadm trigger --type=devices --action=change

sleep 3

ls -Ll /dev/sd*

ls -Ll /dev/*disk0*

 

確認Owner/Group正確。

# ls -Ll /dev/sd*

brw-rw---- 1 root disk     8,  0 Apr 24  2020 /dev/sda

brw-rw---- 1 root disk     8,  1 Apr 24  2020 /dev/sda1

brw-rw---- 1 root disk     8,  2 Apr 24  2020 /dev/sda2

brw-rw---- 1 grid asmadmin 8, 16 Apr 24  2020 /dev/sdb

brw-rw---- 1 grid asmadmin 8, 32 Apr 24  2020 /dev/sdc

brw-rw---- 1 grid asmadmin 8, 48 Apr 24  2020 /dev/sdd

brw-rw---- 1 grid asmadmin 8, 64 Apr 24  2020 /dev/sde

brw-rw---- 1 grid asmadmin 8, 80 Apr 24  2020 /dev/sdf

# ls /dev/*disk0* -l

brw-rw---- 1 grid asmadmin 8, 80 Apr 23 22:35 /dev/arcdisk01

brw-rw---- 1 grid asmadmin 8, 16 Apr 23 22:35 /dev/asmdisk01

brw-rw---- 1 grid asmadmin 8, 32 Apr 23 22:35 /dev/asmdisk02

brw-rw---- 1 grid asmadmin 8, 48 Apr 23 22:35 /dev/asmdisk03

brw-rw---- 1 grid asmadmin 8, 64 Apr 23 22:35 /dev/datdisk01

 

插曲：一定要確保原始盤的屬主配置正確，否則在GI執行root.sh第18步的時候會出現CLSRSC-182: Failed to add the resource for ASM diskgroup的錯誤。

 

六、安裝Grid Infrastructure（節點1執行）

1、解壓

因為從12c R2開始，Grid組件以解壓安裝的方式，所以需要將壓縮包直接解壓到$ORACLE_HOME目錄，否則解壓的目錄就是安裝目錄了。

unzip -q /tmp/linuxx64_12201_grid_home.zip -d $ORACLE_HOME

 

2、檢測安裝環境

$ORACLE_HOME/runcluvfy.sh  stage -pre crsinst -n oraclerac8281,oraclerac8282,oraclerac8283 -fixup -verbose

 

3、開始安裝

$ORACLE_HOME/gridSetup.sh

 

4、執行腳本（root）

/u01/app/oraInventory/orainstRoot.sh

/u01/app/12.2.0/grid/root.sh

 

等節點1執行成功之后，在其它節點並行執行。

 

5、檢查集群狀態

crsctl status res -t

 

6、創建其它磁盤組

asmca

 

七、安裝Database（節點1執行）

1、安裝

/u01/database/runInstaller

 

2、執行腳本（root）

/u01/app/oracle/product/12.2.0/db_1/root.sh

 

3、檢查數據庫集群狀態

srvctl status database -d orcl

附錄

1、ORACLE 12.2認證的OS軟件版本

序號	組件
1	Oracle Linux 7 with the Unbreakable Enterprise kernel: 3.8.13-35.3.1.el7uek.x86_64 or later
2	Oracle Linux 7 with the Red Hat Compatible kernel: 3.10.0-123.el7.x86_64 or later
3	Oracle Linux 6.4 with the Unbreakable Enterprise kernel 2: 2.6.39-200.24.1.el6uek.x86_64 or later
4	Oracle Linux 6 with the Red Hat Compatible kernel: 2.6.32-358.el6.x86_64 or later
5	Red Hat Enterprise Linux 7: 3.10.0-123.el7.x86_64 or later
6	Red Hat Enterprise Linux 6: 2.6.32-358.el6.x86_64 or later
7	SUSE Linux Enterprise Server 12: 3.12.28-4-default or later
8	SUSE Linux Enterprise Server 11 SP2: 3.0.13-0.27 or later

 

2、集群管理命令

啟動Cluster集群(root)

/u01/app/12.2.0/grid/bin/crsctl start cluster [-all | -n <server name>]

 

關閉Cluster集群(root)

/u01/app/12.2.0/grid/bin/crsctl start cluster [-all | -n <server name>]

 

-all              啟動集群內所有節點

-n <server name>  啟動指定節點

 

查看集群狀態(grid)

crsctl status res -t

 

查看數據庫集群狀態(oracle)

srvctl status database -d orcl

 

查看Listener狀態

srvctl status listener

 

關閉實例

srvctl stop instance -d orcl -n <server name>

 

啟動實例

srvctl start instance -d orcl -n <server name>

 

關閉數據庫

srvctl stop database -d orcl

 

啟動數據庫

srvctl start database -d orcl