通常情況下，Linux的網卡中斷是由一個CPU核心來處理的，當承擔高流量的場景下，會出現一些詭異的情況（網卡尚未達到瓶頸，但是卻出現丟包的情況）
這種時候，我們最好看下網卡中斷是不是缺少調優。
優化3要點：網卡多隊列+irq affinity親緣性設置+關閉irqbalance
目前阿里雲官方提供的centos和ubuntu鏡像裏面，已經自帶了優化腳本，內容如下:
centos7的腳本路徑在 /usr/sbin/ecs_mq_rps_rfs 具體內容如下：
#!/bin/bash
# This is the default setting of networking multiqueue and irq affinity
# 1. enable multiqueue if available
# 2. irq affinity optimization
# 3. stop irqbalance service
# set and check multiqueue
function set_check_multiqueue()
{
eth=$1
log_file=$2
queue_num=$(ethtool -l $eth | grep -ia5 'pre-set' | grep -i combined | awk {'print $2'})
if [ $queue_num -gt 1 ]; then
# set multiqueue
ethtool -L $eth combined $queue_num
# check multiqueue setting
cur_q_num=$(ethtool -l $eth | grep -iA5 current | grep -i combined | awk {'print $2'})
if [ "X$queue_num" != "X$cur_q_num" ]; then
echo "Failed to set $eth queue size to $queue_num" >> $log_file
echo "after setting, pre-set queue num: $queue_num , current: $cur_q_num" >> $log_file
return 1
else
echo "OK. set $eth queue size to $queue_num" >> $log_file
fi
else
echo "only support $queue_num queue; no need to enable multiqueue on $eth" >> $log_file
fi
}
#set irq affinity
function set_irq_smpaffinity()
{
log_file=$1
node_dir=/sys/devices/system/node
for i in $(ls -d $node_dir/node*); do
i=${i/*node/}
done
echo "max node :$i" >> $log_file
node_cpumax=$(cat /sys/devices/system/node/node${i}/cpulist |awk -F- '{print $NF}')
irqs=($(cat /proc/interrupts |grep virtio |grep put | awk -F: '{print $1}'))
core=0
for irq in ${irqs[@]};do
VEC=$core
if [ $VEC -ge 32 ];then
let "IDX = $VEC / 32"
MASK_FILL=""
MASK_ZERO="00000000"
for ((i=1; i<=$IDX;i++))
do
MASK_FILL="${MASK_FILL},${MASK_ZERO}"
done
let "VEC -= 32 * $IDX"
MASK_TMP=$((1<<$VEC))
MASK=$(printf "%X%s" $MASK_TMP $MASK_FILL)
else
MASK_TMP=$((1<<$VEC))
MASK=$(printf "%X" $MASK_TMP)
fi
echo $MASK > /proc/irq/$irq/smp_affinity
echo "mask:$MASK, irq:$irq" >> $log_file
core=$(((core+1)%(node_cpumax+1)))
done
}
# stop irqbalance service
function stop_irqblance()
{
log_file=$1
ret=0
if [ "X" != "X$(ps -ef | grep irqbalance | grep -v grep)" ]; then
if which systemctl;then
systemctl stop irqbalance
else
service irqbalance stop
fi
if [ $? -ne 0 ]; then
echo "Failed to stop irqbalance" >> $log_file
ret=1
fi
else
echo "OK. irqbalance stoped." >> $log_file
fi
return $ret
}
# main logic
function main()
{
ecs_network_log=/var/log/ecs_network_optimization.log
ret_value=0
echo "running $0" > $ecs_network_log
echo "======== ECS network setting starts $(date +'%Y-%m-%d %H:%M:%S') ========" >> $ecs_network_log
# we assume your NIC interface(s) is/are like eth*
eth_dirs=$(ls -d /sys/class/net/eth*)
if [ "X$eth_dirs" = "X" ]; then
echo "ERROR! can not find any ethX in /sys/class/net/ dir." >> $ecs_network_log
ret_value=1
fi
for i in $eth_dirs
do
cur_eth=$(basename $i)
echo "optimize network performance: current device $cur_eth" >> $ecs_network_log
# only optimize virtio_net device
driver=$(basename $(readlink $i/device/driver))
if ! echo $driver | grep -q virtio; then
echo "ignore device $cur_eth with driver $driver" >> $ecs_network_log
continue
fi
echo "set and check multiqueue on $cur_eth" >> $ecs_network_log
set_check_multiqueue $cur_eth $ecs_network_log
if [ $? -ne 0 ]; then
echo "Failed to set multiqueue on $cur_eth" >> $ecs_network_log
ret_value=1
fi
done
stop_irqblance $ecs_network_log
set_irq_smpaffinity $ecs_network_log
echo "======== ECS network setting END $(date +'%Y-%m-%d %H:%M:%S') ========" >> $ecs_network_log
return $ret_value
}
# program starts here
main
exit $?查詢的rps綁定情況的腳本 get_rps.sh
#!/bin/bash # 獲取當前rps情況 for i in $(ls /sys/class/net/eth0/queues/rx-*/rps_cpus); do echo $i cat $i done
參考文檔:
https://help.aliyun.com/knowledge_detail/52559.html 【推薦閱讀】
https://www.jianshu.com/p/09bb5d5a72ba
https://help.aliyun.com/document_detail/25378.html#g6
https://tech.meituan.com/2018/03/16/redis-high-concurrency-optimization.html 【推薦閱讀】