数据库实例正，数据磁盘组online,asm磁盘组正常访问。
故障现象：
[root@p750db2://]#su - grid
$ crsctl stat res -t
CRS-4535: Cannot communicate with Cluster Ready Services
CRS-4000: Command Status failed, or completed with errors.
$ ps -ef|grep has
root 6029498 1 0 Jan 01 - 1971:22 /oracle/app/11.2.0/grid/bin/ohasd.bin reboot
root 6225994 1 0 Jan 01 - 0:00 /bin/sh /etc/init.ohasd run
grid 23986302 21364882 0 10:49:58 pts/2 0:00 grep has
$ ps -ef|grep pmon
grid 24838256 21364882 0 10:50:07 pts/2 0:00 grep pmon
oracle 14942614 1 0 Aug 31 - 32:24 ora_pmon_db2
grid 21758402 1 0 Aug 31 - 20:40 asm_pmon_+ASM2
$ crsctl stat res -t
CRS-4535: Cannot communicate with Cluster Ready Services
CRS-4000: Command Status failed, or completed with errors.
$ which crsctl
/oracle/app/11.2.0/grid/bin/crsctl
$ exit
[root@p750db2://]#/oracle/app/11.2.0/grid/bin/crsctl start crs
CRS-4640: Oracle High Availability Services is already active
CRS-4000: Command Start failed, or completed with errors.
[root@p750db2://]#/oracle/app/11.2.0/grid/bin/crsctl check has
CRS-4638: Oracle High Availability Services is online
[root@p750db2://]#/oracle/app/11.2.0/grid/bin/crsctl check crs
CRS-4638: Oracle High Availability Services is online
CRS-4535: Cannot communicate with Cluster Ready Services
CRS-4529: Cluster Synchronization Services is online
CRS-4533: Event Manager is online
[root@p750db2://]#/oracle/app/11.2.0/grid/bin/crsctl stat res -t -init
--------------------------------------------------------------------------------
NAME TARGET STATE SERVER STATE_DETAILS
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ora.asm
1 ONLINE ONLINE p750db2 Started
ora.cluster_interconnect.haip
1 ONLINE ONLINE p750db2
ora.crf
1 ONLINE ONLINE p750db2
ora.crsd
1 ONLINE OFFLINE
ora.cssd
1 ONLINE ONLINE p750db2
ora.cssdmonitor
1 ONLINE ONLINE p750db2
ora.ctssd
1 ONLINE ONLINE p750db2 OBSERVER
ora.diskmon
1 OFFLINE OFFLINE
ora.drivers.acfs
1 ONLINE ONLINE p750db2
ora.evmd
1 ONLINE ONLINE p750db2
ora.gipcd
1 ONLINE ONLINE p750db2
ora.gpnpd
1 ONLINE ONLINE p750db2
ora.mdnsd
1 ONLINE ONLINE p750db2 故障解决
检查asm磁盘，发现存放ocr 磁盘组offline
手动mount OCR磁盘组
[root@p750db2://]#su - grid
$ asmcmd
ASMCMD> ls
ARCH/
DATA/
ASMCMD> mount -a
ORA-15032: not all alterations performed
ORA-15017: diskgroup "DATA" cannot be mounted
ORA-15013: diskgroup "DATA" is already mounted
ORA-15017: diskgroup "ARCH" cannot be mounted
ORA-15013: diskgroup "ARCH" is already mounted (DBD ERROR: OCIStmtExecute)
ASMCMD> lsdg
State Type Rebal Sector Block AU Total_MB Free_MB Req_mir_free_MB Usable_file_MB Offline_disks Voting_files Name
MOUNTED EXTERN N 512 4096 1048576 1024000 377578 0 377578 0 N ARCH/
MOUNTED EXTERN N 512 4096 1048576 1024000 516385 0 516385 0 N DATA/
MOUNTED HIGH N 512 4096 1048576 25600 24092 10240 4617 0 Y GRID/
ASMCMD> exit启动crsd 进程
$ exit
[root@p750db2://]#/oracle/app/11.2.0/grid/bin/crsctl start resource ora.crsd -init
CRS-2672: Attempting to start 'ora.crsd' on 'p750db2'
CRS-2676: Start of 'ora.crsd' on 'p750db2' succeeded故障解决，检查集群状态
[root@p750db2://]#
[root@p750db2://]#/oracle/app/11.2.0/grid/bin/crsctl stat res -t
--------------------------------------------------------------------------------
NAME TARGET STATE SERVER STATE_DETAILS
--------------------------------------------------------------------------------
Local Resources
--------------------------------------------------------------------------------
ora.ARCH.dg
ONLINE ONLINE p750db1
ONLINE ONLINE p750db2
ONLINE ONLINE p750db3
ONLINE ONLINE p750db4
ora.DATA.dg
ONLINE ONLINE p750db1
ONLINE ONLINE p750db2
ONLINE ONLINE p750db3
ONLINE ONLINE p750db4
ora.GRID.dg
ONLINE ONLINE p750db1
ONLINE ONLINE p750db2
ONLINE ONLINE p750db3
ONLINE ONLINE p750db4
ora.LISTENER.lsnr
ONLINE ONLINE p750db1
ONLINE ONLINE p750db2
ONLINE ONLINE p750db3
ONLINE OFFLINE p750db4
ora.asm
ONLINE ONLINE p750db1 Started
ONLINE ONLINE p750db2 Started
ONLINE ONLINE p750db3 Started
ONLINE ONLINE p750db4 Started
ora.gsd
OFFLINE OFFLINE p750db1
OFFLINE OFFLINE p750db2
OFFLINE OFFLINE p750db3
OFFLINE OFFLINE p750db4
ora.net1.network
ONLINE ONLINE p750db1
ONLINE ONLINE p750db2
ONLINE ONLINE p750db3
ONLINE ONLINE p750db4
ora.ons
ONLINE ONLINE p750db1
ONLINE ONLINE p750db2
ONLINE ONLINE p750db3
ONLINE ONLINE p750db4
ora.registry.acfs
ONLINE ONLINE p750db1
ONLINE ONLINE p750db2
ONLINE ONLINE p750db3
ONLINE ONLINE p750db4
--------------------------------------------------------------------------------
Cluster Resources
--------------------------------------------------------------------------------
ora.LISTENER_SCAN1.lsnr
1 ONLINE ONLINE p750db2
ora.cvu
1 ONLINE ONLINE p750db1
ora.db.db
1 ONLINE ONLINE p750db1 Open
2 ONLINE ONLINE p750db2 Open
3 ONLINE ONLINE p750db3 Open
4 ONLINE ONLINE p750db4 Open
ora.oc4j
1 ONLINE ONLINE p750db1
ora.p750db1.vip
1 ONLINE ONLINE p750db1
ora.p750db2.vip
1 ONLINE ONLINE p750db2
ora.p750db3.vip
1 ONLINE ONLINE p750db3
ora.p750db4.vip
1 ONLINE INTERMEDIATE p750db2 FAILED OVER
ora.scan1.vip
1 ONLINE ONLINE p750db2
[root@p750db2://]#