#!/bin/bash
# @Function
# Find out the highest cpu consumed threads of java, and print the stack of these threads.
#
# @Usage
# $ ./show-busy-java-threads
#
# @online-doc https://github.com/oldratlee/useful-scripts/blob/master/docs/java.md#-show-busy-java-threads
# @author Jerry Lee (oldratlee at gmail dot com)
# @author superhj1987 (superhj1987 at 126 dot com)
#@author scan 20200509
##readonly用來定義只讀變量，一旦使用readonly定義的變量在腳本中就不能更改
## 獲取腳本名字
readonly PROG="`basename $0`"
##-a：定義只讀數組變量；
##echo ${COMMAND_LINE[0]}
readonly -a COMMAND_LINE=("$0" "$@")
# Get current user name via whoami command
# See https://www.lifewire.com/current-linux-user-whoami-command-3867579
# Because if run command by `sudo -u`, env var $USER is not rewritten/correct, just inherited from outside!
readonly USER="`whoami`"
################################################################################
# util functions
################################################################################
# NOTE: $'foo' is the escape sequence syntax of bash
##echo "$ec[1;31m$@$eend" 定義顏色的始末
##$@ 所有參數的內容
readonly ec=$'\033' # escape char
readonly eend=$'\033[0m' # escape end
colorEcho() {
local color=$1
shift
# if stdout is console, turn on color output.
[ -t 1 ] && echo "$ec[1;${color}m$@$eend" || echo "$@"
}
colorPrint() {
local color=$1
shift
colorEcho "$color" "$@"
[ -n "$append_file" -a -w "$append_file" ] && echo "$@" >> "$append_file"
[ -n "$store_dir" -a -w "$store_dir" ] && echo "$@" >> "${store_file_prefix}$PROG"
}
normalPrint() {
echo "$@"
[ -n "$append_file" -a -w "$append_file" ] && echo "$@" >> "$append_file"
[ -n "$store_dir" -a -w "$store_dir" ] && echo "$@" >> "${store_file_prefix}$PROG"
}
redPrint() {
colorPrint 31 "$@"
}
greenPrint() {
colorPrint 32 "$@"
}
yellowPrint() {
colorPrint 33 "$@"
}
bluePrint() {
colorPrint 36 "$@"
}
die() {
redPrint "Error: $@" 1>&2
exit 1
}
logAndRun() {
echo "$@"
echo
"$@"
}
logAndCat() {
echo "$@"
echo
cat
}
usage() {
#Remove any current binding for keyseq.
#
local -r exit_code="$1"
shift
# -n the length of STRING is nonzero
#-a both EXPRESSION1 and EXPRESSION2 are true
#
[ -n "$exit_code" -a "$exit_code" != 0 ] && local -r out=/dev/stderr || local -r out=/dev/stdout
(( $# > 0 )) && { echo "$@"; echo; } > $out
> $out cat <<EOF
Usage: ${PROG} [OPTION]... [delay [count]]
Find out the highest cpu consumed threads of java, and print the stack of these threads.
Example:
${PROG} # show busy java threads info
${PROG} 1 # update every 1 second, (stop by eg: CTRL+C)
${PROG} 3 10 # update every 3 seconds, update 10 times
Output control:
-p, --pid <java pid> find out the highest cpu consumed threads from
the specified java process, default from all java process.
-c, --count <num> set the thread count to show, default is 5.
-a, --append-file <file> specifies the file to append output as log.
-S, --store-dir <dir> specifies the directory for storing intermediate files, and keep files.
default store intermediate files at tmp dir, and auto remove after run.
use this option to keep files so as to review jstack/top/ps output later.
delay the delay between updates in seconds.
count the number of updates.
delay/count arguments imitates the style of vmstat command.
jstack control:
-s, --jstack-path <path> specifies the path of jstack command.
-F, --force set jstack to force a thread dump.
use when jstack <pid> does not respond (process is hung).
-m, --mix-native-frames set jstack to print both java and native frames (mixed mode).
-l, --lock-info set jstack with long listing. Prints additional information about locks.
CPU usage calculation control:
-d, --top-delay specifies the delay between top samples, default is 0.5 (second).
get thread cpu percentage during this delay interval.
more info see top -d option. eg: -d 1 (1 second).
-P, --use-ps use ps command to find busy thread(cpu usage) instead of top command,
default use top command, because cpu usage of ps command is expressed as
the percentage of time spent running during the *entire lifetime*
of a process, this is not ideal in general.
Miscellaneous:
-h, --help display this help and exit.
EOF
exit $exit_code
}
################################################################################
# Check os support
################################################################################
uname | grep '^Linux' -q || die "$PROG only support Linux, not support `uname` yet!"
################################################################################
# parse options
################################################################################
# NOTE: ARGS can not be declared as readonly!!
# readonly declaration make exit code of assignment to be always 0, aka. the exit code of `getopt` in subshell is discarded.
# tested on bash 4.2.46
##-n, --name <程序名> 將錯誤報告給的程序名
##-a --alternative 允許長選項以 - 開始
## -l, --longoptions <長選項> 要識別的長選項
ARGS=`getopt -n "$PROG" -a -o p:c:a:s:S:Pd:Fmlh -l count:,pid:,append-file:,jstack-path:,store-dir:,use-ps,top-delay:,force,mix-native-frames,lock-info,help -- "$@"`
[ $? -ne 0 ] && { echo; usage 1; }
eval set -- "${ARGS}"
while true; do
case "$1" in
-c|--count)
count="$2"
shift 2
;;
-p|--pid)
pid="$2"
shift 2
;;
-a|--append-file)
append_file="$2"
shift 2
;;
-s|--jstack-path)
jstack_path="$2"
shift 2
;;
-S|--store-dir)
store_dir="$2"
shift 2
;;
-P|--use-ps)
use_ps=true
shift
;;
-d|--top-delay)
top_delay="$2"
shift 2
;;
-F|--force)
force=-F
shift
;;
-m|--mix-native-frames)
mix_native_frames=-m
shift
;;
-l|--lock-info)
more_lock_info=-l
shift
;;
-h|--help)
usage
;;
--)
shift
break
;;
esac
done
##如果count有定義,那麼就是count對應的值,否則就是5
count=${count:-5}
##如果$1存在就是$1,否則就是0
update_delay=${1:-0}
# -z the length of STRING is zero,
[ -z "$1" ] && update_count=1 || update_count=${2:-0}
(( update_count < 0 )) && update_count=0
##如果top_delay 定義,那麼就是top_delay,否則就是0.5
top_delay=${top_delay:-0.5}
##如果use_ps定義,那麼就是use_ps,否則就是false
use_ps=${use_ps:-false}
# check the directory of append-file(-a) mode, create if not exsit.
##創建輸出文件
##-n the length of STRING is nonzero
##
if [ -n "$append_file" ]; then
##-e FILE exists 文件存在
if [ -e "$append_file" ]; then
##-f FILE exists and is a regular file 文件存在且是一個普通文件
[ -f "$append_file" ] || die "$append_file(specified by option -a, for storing run output files) exists but is not a file!"
##-w 文件存在且FILE exists and write permission is granted
[ -w "$append_file" ] || die "file $append_file(specified by option -a, for storing run output files) exists but is not writable!"
else
## 獲取文件對應的目錄
append_file_dir="$(dirname "$append_file")"
##文件存在
if [ -e "$append_file_dir" ]; then
##-d FILE exists and is a directory
[ -d "$append_file_dir" ] || die "directory $append_file_dir(specified by option -a, for storing run output files) exists but is not a directory!"
##-w FILE exists and write permission is granted
[ -w "$append_file_dir" ] || die "directory $append_file_dir(specified by option -a, for storing run output files) exists but is not writable!"
else
mkdir -p "$append_file_dir" || die "fail to create directory $append_file_dir(specified by option -a, for storing run output files)!"
fi
fi
fi
# check store directory(-S) mode, create directory if not exsit.
##-n the length of STRING is nonzero
if [ -n "$store_dir" ]; then
##-e FILE exists
if [ -e "$store_dir" ]; then
##-d FILE exists and is a directory
[ -d "$store_dir" ] || die "$store_dir(specified by option -S, for storing output files) exists but is not a directory!"
##
[ -w "$store_dir" ] || die "directory $store_dir(specified by option -S, for storing output files) exists but is not writable!"
else
mkdir -p "$store_dir" || die "fail to create directory $store_dir(specified by option -S, for storing output files)!"
fi
fi
################################################################################
# check the existence of jstack command
################################################################################
##-n the length of STRING is nonzero
if [ -n "$jstack_path" ]; then
## FILE exists and is a regular file
[ -f "$jstack_path" ] || die "$jstack_path is NOT found!"
## FILE exists and execute (or search) permission is granted
[ -x "$jstack_path" ] || die "$jstack_path is NOT executalbe!"
elif which jstack &> /dev/null; then
jstack_path="`which jstack`"
else
[ -n "$JAVA_HOME" ] || die "jstack not found on PATH and No JAVA_HOME setting! Use -s option set jstack path manually."
[ -f "$JAVA_HOME/bin/jstack" ] || die "jstack not found on PATH and \$JAVA_HOME/bin/jstack($JAVA_HOME/bin/jstack) file does NOT exists! Use -s option set jstack path manually."
[ -x "$JAVA_HOME/bin/jstack" ] || die "jstack not found on PATH and \$JAVA_HOME/bin/jstack($JAVA_HOME/bin/jstack) is NOT executalbe! Use -s option set jstack path manually."
jstack_path="$JAVA_HOME/bin/jstack"
fi
################################################################################
# biz logic
################################################################################
readonly run_timestamp="`date "+%Y-%m-%d_%H:%M:%S.%N"`"
readonly uuid="${PROG}_${run_timestamp}_${RANDOM}_$$"
readonly tmp_store_dir="/tmp/${uuid}"
##-n the length of STRING is nonzero
if [ -n "$store_dir" ]; then
readonly store_file_prefix="$store_dir/${run_timestamp}_"
else
readonly store_file_prefix="$tmp_store_dir/${run_timestamp}_"
fi
mkdir -p "$tmp_store_dir"
cleanupWhenExit() {
rm -rf "$tmp_store_dir" &> /dev/null
}
trap "cleanupWhenExit" EXIT
headInfo() {
colorEcho "0;34;42" ================================================================================
echo "$(date "+%Y-%m-%d %H:%M:%S.%N") [$(( i + 1 ))/$update_count]: ${COMMAND_LINE[@]}"
colorEcho "0;34;42" ================================================================================
echo
}
## 傳了pid,就指定了pid
if [ -n "${pid}" ]; then
readonly ps_process_select_options="-p $pid"
else
##-C 這將選擇在cmdlist中給定可執行名稱的進程
readonly ps_process_select_options="-C java -C jsvc"
fi
# output field: pid, thread id(lwp), pcpu, user
# order by pcpu(percentage of cpu usage)
findBusyJavaThreadsByPs() {
# 1. sort by %cpu by ps option `--sort -pcpu`
# 2. use wide output(unlimited width) by ps option `-ww`
# avoid trunk user column to username_fo+ or $uid alike
##-w Wide output. Use this option twice for unlimited width.
##-L Show threads, possibly with LWP and NLWP columns
##-o 選項允許您指定在運行 ps 命令時顯示哪些列
##LWP (thread ID) columns
##pcpu cpu utilization
## --sort spec specify sorting order. Sorting syntax is [+|-]key[,[+|-]key[,...]]
##Choose a multi-letter key from the STANDARD FORMAT SPECIFIERS section. The "+" is
##默認是遞增
##optional since default direction is increasing numerical or
## lexicographic order. Identical to k. For example: ps jax --sort=uid,-ppid,+pid
local -a ps_cmd_line=(ps $ps_process_select_options -wwLo pid,lwp,pcpu,user --sort -pcpu --no-headers)
local -r ps_out="$("${ps_cmd_line[@]}")"
if [ -n "$store_dir" ]; then
echo "$ps_out" | logAndCat "${ps_cmd_line[@]}" > "${store_file_prefix}$(( i + 1 ))_ps"
fi
##modifyby scan
##
#echo "$ps_out" | head -n "${count}"
echo "$ps_out" | sort -rk3 |head -n "${count}"
}
# top with output field: thread id, %cpu
__top_threadId_cpu() {
# 1. sort by %cpu by top option `-o %CPU`
# unfortunately, top version 3.2 does not support -o option(supports from top version 3.3+),
# use
# HOME="$tmp_store_dir" top -H -b -n 1
# combined
# sort
# instead of
# HOME="$tmp_store_dir" top -H -b -n 1 -o '%CPU'
# 2. change HOME env var when run top,
# so as to prevent top command output format being change by .toprc user config file unexpectedly
# 3. use option `-d 0.5`(update interval 0.5 second) and `-n 2`(update 2 times),
# and use second time update data to get cpu percentage of thread in 0.5 second interval
# 4. top v3.3, there is 1 black line between 2 update;
# but top v3.2, there is 2 blank lines between 2 update!
##-a 只讀數組
##-H： 設置線程模式 -b Starts top in ’Batch mode’ -d: 啓動時設置刷新時間間隔 -n: n : 更新的次數，完成後將會退出 top
local -a top_cmd_line=(top -H -b -d $top_delay -n 2)
##${nums[@]} 獲取數組中的所有元素
##切換top運行目錄
## 獲取cpu最高的線程
local -r top_out=$(HOME="$tmp_store_dir" "${top_cmd_line[@]}")
if [ -n "$store_dir" ]; then
echo "$top_out" | logAndCat "${top_cmd_line[@]}" > "${store_file_prefix}$(( i + 1 ))_top"
fi
#只輸出java進程,輸出線程號,CPU 字段
echo "$top_out" |
awk 'BEGIN { blockIndex = 0; currentLineHasText = 0; prevLineHasText = 0; } {
currentLineHasText = ($0 != "")
if (prevLineHasText && !currentLineHasText)
blockIndex++ # from text line to empty line, increase block index
if (blockIndex == 3 && ($NF == "java" || $NF == "jsvc")) # $NF(last field) is command field
# only print 4th text block(blockIndex == 3), aka. process info of second top update
print $1 " " $9 # $1 is thread id field, $9 is %cpu field
prevLineHasText = currentLineHasText # update prevLineHasText
}' | sort -k2,2nr
}
__complete_pid_user_by_ps() {
# ps output field: pid, thread id(lwp), user
local -a ps_cmd_line=(ps $ps_process_select_options -wwLo pid,lwp,user --no-headers)
local -r ps_out="$("${ps_cmd_line[@]}")"
##-n 非空
if [ -n "$store_dir" ]; then
echo "$ps_out" | logAndCat "${ps_cmd_line[@]}" > "${store_file_prefix}$(( i + 1 ))_ps"
fi
local idx=0
local -a line
##從__top_threadId_cpu 函數讀入線程號,cpu
while IFS=" " read -a line ; do
(( idx < count )) || break
local threadId="${line[0]}"
#echo $threadId
local pcpu="${line[1]}"
#echo $pcpu
# output field: pid, threadId, pcpu, user
##通過線程號關聯
## $1 進程號 $2線程號 $3用戶
## 判斷$2==threadId
local output_fields="$( echo "$ps_out" |
awk -v "threadId=$threadId" -v "pcpu=$pcpu" '$2==threadId {
printf "%s %s %s %s\n", $1, threadId, pcpu, $3; exit
}' )"
if [ -n "$output_fields" ]; then
(( idx++ ))
echo "$output_fields"
fi
done
}
# output format is same as function findBusyJavaThreadsByPs
#| 第一個命令command 1執行的結果作爲command2的輸入傳給command 2
findBusyJavaThreadsByTop() {
__top_threadId_cpu | __complete_pid_user_by_ps
}
printStackOfThreads() {
local -a line
local idx=0
redPrint "使用的JVM工具路徑:"
echo $jstack_path
sleep 2
while IFS=" " read -a line ; do
##打印數組內容
redPrint "pid,lwp,pcpu,user"
redPrint ${line[*]}
local pid="${line[0]}"
local threadId="${line[1]}"
local threadId0x="0x`printf %x ${threadId}`"
local pcpu="${line[2]}"
local user="${line[3]}"
(( idx++ ))
local jstackFile="${store_file_prefix}$(( i + 1 ))_jstack_${pid}"
[ -f "${jstackFile}" ] || {
local -a jstack_cmd_line=( "$jstack_path" ${force} $mix_native_frames $more_lock_info ${pid} )
if [ "${user}" == "${USER}" ]; then
# run without sudo, when java process user is current user
logAndRun "${jstack_cmd_line[@]}" > ${jstackFile}
elif [ $UID == 0 ]; then
# if java process user is not current user, must run jstack with sudo
logAndRun sudo -u "${user}" "${jstack_cmd_line[@]}" > ${jstackFile}
else
# current user is not root user, so can not run with sudo; print error message and rerun suggestion
redPrint "[$idx] Fail to jstack busy(${pcpu}%) thread(${threadId}/${threadId0x}) stack of java process(${pid}) under user(${user})."
redPrint "User of java process($user) is not current user($USER), need sudo to rerun:"
yellowPrint " sudo ${COMMAND_LINE[@]}"
normalPrint
continue
fi || {
redPrint "[$idx] Fail to jstack busy(${pcpu}%) thread(${threadId}/${threadId0x}) stack of java process(${pid}) under user(${user})."
normalPrint
rm "${jstackFile}" &> /dev/null
continue
}
}
bluePrint "[$idx] Busy(${pcpu}%) thread(${threadId}/${threadId0x}) stack of java process(${pid}) under user(${user}):"
if [ -n "$mix_native_frames" ]; then
local sed_script="/--------------- $threadId ---------------/,/^---------------/ {
/--------------- $threadId ---------------/b # skip first seperator line
/^---------------/d # delete second seperator line
p
}"
elif [ -n "$force" ]; then
local sed_script="/^Thread ${threadId}:/,/^$/ {
/^$/d; p # delete end seperator line
}"
else
local sed_script="/ nid=${threadId0x} /,/^$/ {
/^$/d; p # delete end seperator line
}"
fi
{
sed "$sed_script" -n ${jstackFile}
echo
} | tee ${append_file:+-a "$append_file"} ${store_dir:+-a "${store_file_prefix}$PROG"}
done
}
################################################################################
# Main
################################################################################
main() {
local i
# if update_count <= 0, infinite loop till user interrupted (eg: CTRL+C)
for (( i = 0; update_count <= 0 || i < update_count; ++i )); do
(( i > 0 )) && sleep "$update_delay"
[ -n "$append_file" -o -n "$store_dir" ] && headInfo | tee ${append_file:+-a "$append_file"} ${store_dir:+-a "${store_file_prefix}$PROG"} > /dev/null
(( update_count != 1 )) && headInfo
if $use_ps; then
findBusyJavaThreadsByPs
else
findBusyJavaThreadsByTop
fi | printStackOfThreads
redPrint "jstat 列說明"
cat <<EOF
E（Eden區,新生代）
O(老年代)
FGC（Full GC）的次數
FGCT（Full GC Time）的累計時間
EOF
##時間間隔2s,查詢次數10次
jstat -gcutil $pid 2000 10
redPrint "classname 類型說明"
cat <<EOF
[C is a char[]
[S is a short[]
[I is a int[]
[B is a byte[]
[[I is a int[][]
EOF
jmap -histo $pid
done
}
main
