1. 腳本目的
當java(tomcat)進程還存在，但是無法處理任何請求，如假死、沒有任何響應等時候，執行dump.sh腳本，快照當前系統及jvm的信息。
2. dump.sh腳本
#!/usr/bin/env bash
declare SNAPSHOT_BASE_DIRECTORY=$1
declare JAVA_HOME=$2
declare PID=$3
# remove the last '/' char
set JAVA_HOME=${JAVA_HOME/%\//}
set SNAPSHOT_BASE_DIRECTORY=${SNAPSHOT_BASE_DIRECTORY/%\//}
declare FULL_TIME_FORMAT="+%Y-%m-%d_%H_%M_%S_%N"
declare SHORT_TIME_FORMAT="+%H-%M-%S-%N"
declare TIMESTAMP=`date ""${FULL_TIME_FORMAT}"" `
declare RESULT_DIRECTORY=${SNAPSHOT_BASE_DIRECTORY}/${TIMESTAMP}
declare RUN_LOG_PATH="${RESULT_DIRECTORY}/run.log"
run_log(){
declare log_msg="$(date ""${FULL_TIME_FORMAT}""): $1"
echo ${log_msg}
echo ${log_msg} >>${RUN_LOG_PATH}
}
echo_blank_line(){
echo
}
is_no_null(){
if [ -z $1 ];then
return 0;
else
return 1;
fi
}
is_valid_number(){
is_no_null $1
if [ $? -eq 0 ];then
return 0;
fi
if [ $1 -gt 0 ] 2>/dev/null ;then
return 1;
else
return 0;
fi
}
valid_param(){
is_no_null ${SNAPSHOT_BASE_DIRECTORY}
if [ $? -eq 0 ];then
echo "Param 1 SNAPSHOT_BASE_DIRECTORY is no exist and exit"
exit 1;
else
echo "tip: SNAPSHOT_BASE_DIRECTORY=${SNAPSHOT_BASE_DIRECTORY}"
fi
if [ ! -d ${RESULT_DIRECTORY} ]; then
mkdir -p ${RESULT_DIRECTORY}
run_log "tip: Create RESULT_DIRECTORY=${RESULT_DIRECTORY}"
fi
echo "tip: RESULT_DIRECTORY=${RESULT_DIRECTORY}"
if [ -d "${JAVA_HOME}" ];then
run_log "tip: JAVA HOME: ${JAVA_HOME}"
else
run_log "Param2 JAVA_HOME is no exist and exit"
exit 2;
fi
is_valid_number ${PID}
if [ $? -eq 0 ];then
run_log "Param3 PID is invalid and exit"
exit 3;
fi
}
valid_param
echo "RESULT_DIRECTORY=${RESULT_DIRECTORY}"
machine_dump(){
declare filename=$1;
type $1 >/dev/null 2>&1 && {
run_log "Start $1 $2 dump"
declare timestamp=$(date ""${SHORT_TIME_FORMAT}"");
run_log "Execute $1 $2 >> ${RESULT_DIRECTORY}/machine_${filename}_${timestamp}.dump"
$1 $2 >> ${RESULT_DIRECTORY}/machine_${filename}_${timestamp}.dump
run_log "End $1 $2 dump"
echo_blank_line
}
}
# 3 times interval 1s
declare machine_static_frequency="1 3"
# machine real time statistics information
machine_dump uptime
machine_dump free -glt
machine_dump vmstat "-t ${machine_static_frequency}"
machine_dump mpstat "-A ${machine_static_frequency}"
machine_dump iostat "${machine_static_frequency}"
machine_dump netstat "-an"
machine_dump lsof "-p ${PID}"
# machine history statistics information
machine_dump sar -A
jvm_dump(){
declare filename;
if [ ! -z $3 ];then
filename=$3
else
filename=$1
fi
run_log "Start $1 $2 dump"
declare timestamp=$(date ""${SHORT_TIME_FORMAT}"");
run_log "${JAVA_HOME}/bin/$1 $2 >> ${RESULT_DIRECTORY}/jvm_${filename}_${timestamp}.dump"
${JAVA_HOME}/bin/$1 $2 >> ${RESULT_DIRECTORY}/jvm_${filename}_${timestamp}.dump
run_log "End $1 $2 dump"
echo_blank_line
}
# 8 times interval 1s
declare jvm_static_frequency="1000 8"
# jvm real time statistics information
jvm_dump jinfo "${PID}"
# topH have must together with jstack
machine_dump top "-H -b -n 1 -p ${PID}" "topH"
jvm_dump jstack "-F ${PID}"
jvm_dump jstack "-F -l ${PID}" "jstack-deadlock"
jvm_dump jstat "-gcutil ${PID} ${jvm_static_frequency}" "jstat-gcutil"
jvm_dump jstat "-gccapacity ${PID} ${jvm_static_frequency}" "jstat-gccapacity"
jvm_dump jmap "-F -heap ${PID}" "jmap-heap"
jvm_dump jmap "-F -histo ${PID}" "jmap-histo"
# statistics all kinds of tcp status
cat ${RESULT_DIRECTORY}/machine_netstat*.dump | awk '/^tcp/ {++S[$NF]} END {for(a in S) print a, S[a]}' > ${RESULT_DIRECTORY}/machine_netstat_status_statistics.dump
3. 執行demo
# 正常執行
/data/sh/dump.sh /data/rigger /usr/local/java/jdk1.8.0_05 16751
# 調試執行
bash -x /data/sh/dump.sh /data/rigger /usr/local/java/jdk1.8.0_05 16751
- 參數說明
參數 | 說明 -- | -- /data/sh/dump.sh | dump腳本的絕對路徑 /data/rigger | dump文件保存的根路徑，注意，會在該路徑下在生成時間戳文件夾【注意】：rigger後面別+/ /usr/local/java/jdk1.8.0_05 | java_home路徑，【注意】：jdk1.8.0_05後面別+/ 16751 | java(tomcat)-pid
4. 執行結果
[root@chow rigger]$ ll
total 4
drwxr-xr-x 2 root root 4096 Mar 31 15:34 2017-03-31_15-31-33-801995527
[root@chow rigger]$ cd 2017-03-31_15-31-33-801995527/
[root@chow 2017-03-31_15-31-33-801995527]$ ll
total 1108
-rw-r--r-- 1 root root 8617 Mar 31 15:31 jvm_jinfo_15-31-38-646484941.dump
-rw-r--r-- 1 root root 1688 Mar 31 15:33 jvm_jmap-heap_15-33-26-169241399.dump
-rw-r--r-- 1 root root 224388 Mar 31 15:34 jvm_jmap-histo_15-33-31-900738428.dump
-rw-r--r-- 1 root root 36390 Mar 31 15:31 jvm_jstack_15-31-41-120341421.dump
-rw-r--r-- 1 root root 38276 Mar 31 15:33 jvm_jstack-deadlock_15-31-43-547938349.dump
-rw-r--r-- 1 root root 1481 Mar 31 15:33 jvm_jstat-gccapacity_15-33-18-916960208.dump
-rw-r--r-- 1 root root 738 Mar 31 15:33 jvm_jstat-gcutil_15-33-11-650829600.dump
-rw-r--r-- 1 root root 353 Mar 31 15:31 machine_free_15-31-33-825915457.dump
-rw-r--r-- 1 root root 1055 Mar 31 15:31 machine_iostat_15-31-35-878499063.dump
-rw-r--r-- 1 root root 16243 Mar 31 15:31 machine_lsof_15-31-38-316510021.dump
-rw-r--r-- 1 root root 24546 Mar 31 15:31 machine_mpstat_15-31-35-851353492.dump
-rw-r--r-- 1 root root 175841 Mar 31 15:31 machine_netstat_15-31-37-893697066.dump
-rw-r--r-- 1 root root 0 Mar 31 15:34 machine_netstat_status_statistics.dump
-rw-r--r-- 1 root root 547932 Mar 31 15:31 machine_sar_15-31-38-566908017.dump
-rw-r--r-- 1 root root 15704 Mar 31 15:31 machine_topH_15-31-40-596637210.dump
-rw-r--r-- 1 root root 72 Mar 31 15:31 machine_uptime_15-31-33-814570794.dump
-rw-r--r-- 1 root root 507 Mar 31 15:31 machine_vmstat_15-31-33-836673013.dump
-rw-r--r-- 1 root root 4439 Mar 31 15:34 run.log
5. 腳本說明
- 腳本快照信息，主要包含兩部分：機器信息和jvm信息
- 機器信息，包含:)實時信息和歷史信息[sar]，部分實時信息會截取執行腳本時候的3次快照信息，間隔爲1s;最後會統計下tcp的各種狀態數量。
- jvm信息，含jinfo[jvm配置信息]、jstack[線程及死鎖]、jstat[實時jvm垃圾回收信息]及jmap[堆及堆中對象信息]。