redhat6.5 11.2.0.3 存儲mpath
安裝clusterware的時候節點二執行root.sh時報錯：
SQL> CREATE DISKGROUP CRS EXTERNAL REDUNDANCY DISK 'ORCL:ASM_DISK04',
'ORCL:ASM_DISK05',
'ORCL:ASM_DISK06' ATTRIBUTE 'compatible.asm'='11.2.0.0.0','au_size'='1M' /* ASMCA */
NOTE: Assigning number (1,0) to disk (ORCL:ASM_DISK04)
NOTE: Assigning number (1,1) to disk (ORCL:ASM_DISK05)
NOTE: Assigning number (1,2) to disk (ORCL:ASM_DISK06)
ORA-15186: ASMLIB error function = [asm_open(global)], error = [1], mesg = [Operation not permitted]
ORA-15025: could not open disk "ORCL:ASM_DISK01"
ORA-15186: ASMLIB error function = [asm_open(global)], error = [1], mesg = [Operation not permitted]
ORA-15025: could not open disk "ORCL:ASM_DISK02"
ORA-15186: ASMLIB error function = [asm_open(global)], error = [1], mesg = [Operation not permitted]
ORA-15025: could not open disk "ORCL:ASM_DISK03"
ORA-15186: ASMLIB error function = [asm_open(global)], error = [1], mesg = [Operation not permitted]
ORA-15025: could not open disk "ORCL:ASM_DISK04"
ORA-15186: ASMLIB error function = [asm_open(global)], error = [1], mesg = [Operation not permitted]
ORA-15025: could not open disk "ORCL:ASM_DISK05"
ORA-15186: ASMLIB error function = [asm_open(global)], error = [1], mesg = [Operation not permitted]
ORA-15025: could not open disk "ORCL:ASM_DISK06"
ORA-15186: ASMLIB error function = [asm_open(global)], error = [1], mesg = [Operation not permitted]
ORA-15025: could not open disk "ORCL:ASM_DISK07"
ORA-15186: ASMLIB error function = [asm_open(global)], error = [1], mesg = [Operation not permitted]
ORA-15025: could not open disk "ORCL:ASM_DISK08"
NOTE: Disk in mode 0x8 marked for de-assignment
NOTE: Disk in mode 0x8 marked for de-assignment
NOTE: Disk in mode 0x8 marked for de-assignment
NOTE: initiating PST update: grp = 1
GMON updating group 1 at 3 for pid 23, osid 32785
NOTE: PST update grp = 1 not completed, ret 10
GMON dismounting group 1 at 4 for pid 23, osid 32785
ERROR: diskgroup CRS was not created
ORA-15018: diskgroup cannot be created
ORA-15072: command requires at least 1 regular failure groups, discovered only 0
ORA-15080: synchronous I/O operation to a disk failed
ORA-15080: synchronous I/O operation to a disk failed
ORA-15080: synchronous I/O operation to a disk failed
ORA-15080: synchronous I/O operation to a disk failed
ERROR: CREATE DISKGROUP CRS EXTERNAL REDUNDANCY DISK 'ORCL:ASM_DISK04',
'ORCL:ASM_DISK05',
'ORCL:ASM_DISK06' ATTRIBUTE 'compatible.asm'='11.2.0.0.0','au_size'='1M' /* ASMCA */
'ORCL:ASM_DISK05',
'ORCL:ASM_DISK06' ATTRIBUTE 'compatible.asm'='11.2.0.0.0','au_size'='1M' /* ASMCA */
NOTE: Assigning number (1,0) to disk (ORCL:ASM_DISK04)
NOTE: Assigning number (1,1) to disk (ORCL:ASM_DISK05)
NOTE: Assigning number (1,2) to disk (ORCL:ASM_DISK06)
ORA-15186: ASMLIB error function = [asm_open(global)], error = [1], mesg = [Operation not permitted]
ORA-15025: could not open disk "ORCL:ASM_DISK01"
ORA-15186: ASMLIB error function = [asm_open(global)], error = [1], mesg = [Operation not permitted]
ORA-15025: could not open disk "ORCL:ASM_DISK02"
ORA-15186: ASMLIB error function = [asm_open(global)], error = [1], mesg = [Operation not permitted]
ORA-15025: could not open disk "ORCL:ASM_DISK03"
ORA-15186: ASMLIB error function = [asm_open(global)], error = [1], mesg = [Operation not permitted]
ORA-15025: could not open disk "ORCL:ASM_DISK04"
ORA-15186: ASMLIB error function = [asm_open(global)], error = [1], mesg = [Operation not permitted]
ORA-15025: could not open disk "ORCL:ASM_DISK05"
ORA-15186: ASMLIB error function = [asm_open(global)], error = [1], mesg = [Operation not permitted]
ORA-15025: could not open disk "ORCL:ASM_DISK06"
ORA-15186: ASMLIB error function = [asm_open(global)], error = [1], mesg = [Operation not permitted]
ORA-15025: could not open disk "ORCL:ASM_DISK07"
ORA-15186: ASMLIB error function = [asm_open(global)], error = [1], mesg = [Operation not permitted]
ORA-15025: could not open disk "ORCL:ASM_DISK08"
NOTE: Disk in mode 0x8 marked for de-assignment
NOTE: Disk in mode 0x8 marked for de-assignment
NOTE: Disk in mode 0x8 marked for de-assignment
NOTE: initiating PST update: grp = 1
GMON updating group 1 at 3 for pid 23, osid 32785
NOTE: PST update grp = 1 not completed, ret 10
GMON dismounting group 1 at 4 for pid 23, osid 32785
ERROR: diskgroup CRS was not created
ORA-15018: diskgroup cannot be created
ORA-15072: command requires at least 1 regular failure groups, discovered only 0
ORA-15080: synchronous I/O operation to a disk failed
ORA-15080: synchronous I/O operation to a disk failed
ORA-15080: synchronous I/O operation to a disk failed
ORA-15080: synchronous I/O operation to a disk failed
ERROR: CREATE DISKGROUP CRS EXTERNAL REDUNDANCY DISK 'ORCL:ASM_DISK04',
'ORCL:ASM_DISK05',
'ORCL:ASM_DISK06' ATTRIBUTE 'compatible.asm'='11.2.0.0.0','au_size'='1M' /* ASMCA */
此情況可能是你配置asm的時候用的oracleasm congifure -I 沒有指定需要掃描和排除的盤符前綴，第二個節點掃描的磁盤是sd*的，二用mpath多路徑是用的dm*
先查看現象：
如果用mpath 就要對應上數字 如下列的252 asm_disk01需要和 /dev/dm* 對應上，而不能和/dev/sd* 對應上數值， oracleasm配置的時候用 大 I
[root@SJCTM029 ~]# ls -l /dev/oracleasm/disks/
總用量 0
brw-rw---- 1 grid asmadmin 252, 5 5月 29 18:47 ASM_DISK01
brw-rw---- 1 grid asmadmin 252, 6 5月 29 18:47 ASM_DISK02
brw-rw---- 1 grid asmadmin 252, 8 5月 29 18:47 ASM_DISK03
brw-rw---- 1 grid asmadmin 252, 9 5月 29 18:47 ASM_DISK04
brw-rw---- 1 grid asmadmin 252, 3 5月 29 18:47 ASM_DISK05
brw-rw---- 1 grid asmadmin 252, 4 5月 29 18:47 ASM_DISK06
brw-rw---- 1 grid asmadmin 252, 2 5月 29 18:47 ASM_DISK07
brw-rw---- 1 grid asmadmin 252, 7 5月 29 18:47 ASM_DISK08
總用量 0
brw-rw---- 1 grid asmadmin 252, 5 5月 29 18:47 ASM_DISK01
brw-rw---- 1 grid asmadmin 252, 6 5月 29 18:47 ASM_DISK02
brw-rw---- 1 grid asmadmin 252, 8 5月 29 18:47 ASM_DISK03
brw-rw---- 1 grid asmadmin 252, 9 5月 29 18:47 ASM_DISK04
brw-rw---- 1 grid asmadmin 252, 3 5月 29 18:47 ASM_DISK05
brw-rw---- 1 grid asmadmin 252, 4 5月 29 18:47 ASM_DISK06
brw-rw---- 1 grid asmadmin 252, 2 5月 29 18:47 ASM_DISK07
brw-rw---- 1 grid asmadmin 252, 7 5月 29 18:47 ASM_DISK08
[root@SJCTM029 trace]# ls -l /dev/dm-*
brwxrwxrwx 1 root disk 252, 0 5月 29 16:12 /dev/dm-0
brwxrwxrwx 1 root disk 252, 1 5月 29 16:12 /dev/dm-1
brwxrwxrwx 1 root disk 252, 10 5月 29 16:12 /dev/dm-10
brw-rw---- 1 root disk 252, 2 5月 29 18:47 /dev/dm-2
brw-rw---- 1 root disk 252, 3 5月 29 18:47 /dev/dm-3
brw-rw---- 1 root disk 252, 4 5月 29 18:47 /dev/dm-4
brw-rw---- 1 root disk 252, 5 5月 29 18:47 /dev/dm-5
brw-rw---- 1 root disk 252, 6 5月 29 18:47 /dev/dm-6
brw-rw---- 1 root disk 252, 7 5月 29 18:47 /dev/dm-7
brw-rw---- 1 root disk 252, 8 5月 29 18:47 /dev/dm-8
brw-rw---- 1 root disk 252, 9 5月 29 18:47 /dev/dm-9
brwxrwxrwx 1 root disk 252, 0 5月 29 16:12 /dev/dm-0
brwxrwxrwx 1 root disk 252, 1 5月 29 16:12 /dev/dm-1
brwxrwxrwx 1 root disk 252, 10 5月 29 16:12 /dev/dm-10
brw-rw---- 1 root disk 252, 2 5月 29 18:47 /dev/dm-2
brw-rw---- 1 root disk 252, 3 5月 29 18:47 /dev/dm-3
brw-rw---- 1 root disk 252, 4 5月 29 18:47 /dev/dm-4
brw-rw---- 1 root disk 252, 5 5月 29 18:47 /dev/dm-5
brw-rw---- 1 root disk 252, 6 5月 29 18:47 /dev/dm-6
brw-rw---- 1 root disk 252, 7 5月 29 18:47 /dev/dm-7
brw-rw---- 1 root disk 252, 8 5月 29 18:47 /dev/dm-8
brw-rw---- 1 root disk 252, 9 5月 29 18:47 /dev/dm-9
===========================================
brwxrwxrwx 1 root disk 8, 0 5月 29 16:12 /dev/sda
brwxrwxrwx 1 root disk 8, 1 5月 29 16:12 /dev/sda1
brwxrwxrwx 1 root disk 8, 2 5月 29 16:12 /dev/sda2
brwxrwxrwx 1 root disk 65, 160 5月 29 16:12 /dev/sdaa
brwxrwxrwx 1 root disk 65, 176 5月 29 16:12 /dev/sdab
brwxrwxrwx 1 root disk 65, 192 5月 29 16:12 /dev/sdac
brwxrwxrwx 1 root disk 65, 208 5月 29 16:12 /dev/sdad
brwxrwxrwx 1 root disk 65, 224 5月 29 16:12 /dev/sdae
brwxrwxrwx 1 root disk 65, 240 5月 29 16:12 /dev/sdaf
brwxrwxrwx 1 root disk 66, 0 5月 29 16:12 /dev/sdag
brwxrwxrwx 1 root disk 8, 16 5月 30 2015 /dev/sdb 不能與之對應上了，如果是裸設備就和它對應上
brwxrwxrwx 1 root disk 8, 32 5月 30 2015 /dev/sdc
brwxrwxrwx 1 root disk 8, 48 5月 30 2015 /dev/sdd
brwxrwxrwx 1 root disk 8, 64 5月 30 2015 /dev/sde
brwxrwxrwx 1 root disk 8, 80 5月 29 16:12 /dev/sdf
brwxrwxrwx 1 root disk 8, 96 5月 30 2015 /dev/sdg
brwxrwxrwx 1 root disk 8, 112 5月 30 2015 /dev/sdh
brwxrwxrwx 1 root disk 8, 128 5月 30 2015 /dev/sdi
brwxrwxrwx 1 root disk 8, 144 5月 29 16:12 /dev/sdj
brwxrwxrwx 1 root disk 8, 160 5月 29 16:12 /dev/sdk
brwxrwxrwx 1 root disk 8, 176 5月 29 16:12 /dev/sdl
brwxrwxrwx 1 root disk 8, 192 5月 29 16:12 /dev/sdm
brwxrwxrwx 1 root disk 8, 208 5月 29 16:12 /dev/sdn
brwxrwxrwx 1 root disk 8, 224 5月 29 16:12 /dev/sdo
brwxrwxrwx 1 root disk 8, 240 5月 29 16:12 /dev/sdp
brwxrwxrwx 1 root disk 65, 0 5月 29 16:12 /dev/sdq
brwxrwxrwx 1 root disk 65, 16 5月 29 16:12 /dev/sdr
brwxrwxrwx 1 root disk 65, 32 5月 29 16:12 /dev/sds
brwxrwxrwx 1 root disk 65, 48 5月 29 16:12 /dev/sdt
brwxrwxrwx 1 root disk 65, 64 5月 29 16:12 /dev/sdu
brwxrwxrwx 1 root disk 65, 80 5月 29 16:12 /dev/sdv
brwxrwxrwx 1 root disk 65, 96 5月 29 16:12 /dev/sdw
brwxrwxrwx 1 root disk 65, 112 5月 29 16:12 /dev/sdx
brwxrwxrwx 1 root disk 65, 128 5月 29 16:12 /dev/sdy
brwxrwxrwx 1 root disk 65, 144 5月 29 16:12 /dev/sdz
brwxrwxrwx 1 root disk 8, 1 5月 29 16:12 /dev/sda1
brwxrwxrwx 1 root disk 8, 2 5月 29 16:12 /dev/sda2
brwxrwxrwx 1 root disk 65, 160 5月 29 16:12 /dev/sdaa
brwxrwxrwx 1 root disk 65, 176 5月 29 16:12 /dev/sdab
brwxrwxrwx 1 root disk 65, 192 5月 29 16:12 /dev/sdac
brwxrwxrwx 1 root disk 65, 208 5月 29 16:12 /dev/sdad
brwxrwxrwx 1 root disk 65, 224 5月 29 16:12 /dev/sdae
brwxrwxrwx 1 root disk 65, 240 5月 29 16:12 /dev/sdaf
brwxrwxrwx 1 root disk 66, 0 5月 29 16:12 /dev/sdag
brwxrwxrwx 1 root disk 8, 16 5月 30 2015 /dev/sdb 不能與之對應上了，如果是裸設備就和它對應上
brwxrwxrwx 1 root disk 8, 32 5月 30 2015 /dev/sdc
brwxrwxrwx 1 root disk 8, 48 5月 30 2015 /dev/sdd
brwxrwxrwx 1 root disk 8, 64 5月 30 2015 /dev/sde
brwxrwxrwx 1 root disk 8, 80 5月 29 16:12 /dev/sdf
brwxrwxrwx 1 root disk 8, 96 5月 30 2015 /dev/sdg
brwxrwxrwx 1 root disk 8, 112 5月 30 2015 /dev/sdh
brwxrwxrwx 1 root disk 8, 128 5月 30 2015 /dev/sdi
brwxrwxrwx 1 root disk 8, 144 5月 29 16:12 /dev/sdj
brwxrwxrwx 1 root disk 8, 160 5月 29 16:12 /dev/sdk
brwxrwxrwx 1 root disk 8, 176 5月 29 16:12 /dev/sdl
brwxrwxrwx 1 root disk 8, 192 5月 29 16:12 /dev/sdm
brwxrwxrwx 1 root disk 8, 208 5月 29 16:12 /dev/sdn
brwxrwxrwx 1 root disk 8, 224 5月 29 16:12 /dev/sdo
brwxrwxrwx 1 root disk 8, 240 5月 29 16:12 /dev/sdp
brwxrwxrwx 1 root disk 65, 0 5月 29 16:12 /dev/sdq
brwxrwxrwx 1 root disk 65, 16 5月 29 16:12 /dev/sdr
brwxrwxrwx 1 root disk 65, 32 5月 29 16:12 /dev/sds
brwxrwxrwx 1 root disk 65, 48 5月 29 16:12 /dev/sdt
brwxrwxrwx 1 root disk 65, 64 5月 29 16:12 /dev/sdu
brwxrwxrwx 1 root disk 65, 80 5月 29 16:12 /dev/sdv
brwxrwxrwx 1 root disk 65, 96 5月 29 16:12 /dev/sdw
brwxrwxrwx 1 root disk 65, 112 5月 29 16:12 /dev/sdx
brwxrwxrwx 1 root disk 65, 128 5月 29 16:12 /dev/sdy
brwxrwxrwx 1 root disk 65, 144 5月 29 16:12 /dev/sdz
解決辦法：
vi /etc/sysconfig/oracleasm
# ORACLEASM_SCANORDER: Matching patterns to order disk scanning
ORACLEASM_SCANORDER="dm"
# ORACLEASM_SCANEXCLUDE: Matching patterns to exclude disks from scan
ORACLEASM_SCANEXCLUDE="sd"
ORACLEASM_SCANORDER="dm"
# ORACLEASM_SCANEXCLUDE: Matching patterns to exclude disks from scan
ORACLEASM_SCANEXCLUDE="sd"
可以通過以下方法來確認是否遭遇了此問題。
# ls -l /dev/oracleasm/disks
brw-rw---- 1 oracle dba 3, 65 May 14 12:08 CRSVOL
# cat /proc/partitions
3 65 4974448 sda
253 1 4974448 dm-1
./crsctl stop crs
./crsctl disable crs
oracleasm stop
./crsctl disable crs
oracleasm stop
oracleasm exit
oracleasm init
oracleasm scandisks
ls -l /dev/oracleasm/disks/
./crsctl enable crs
./crsctl start crs
oracleasm init
oracleasm scandisks
ls -l /dev/oracleasm/disks/
./crsctl enable crs
./crsctl start crs
節點二再次執行root.sh就沒問題了
http://www.tuicool.com/articles/UreyMb + my-oracle-support
MOS Note [1050164.1]