NFS配置及優化

NFS配置

1. 不同的內容服務採用不同的文件系統，文件類型的可以是ext4，數據庫類型的可以是xfs
2. linux系統的安裝，多硬盤，系統盤和數據盤分開，使用lvm方式，數據盤大小要一致（不一致導致存儲浪費）
3. 完成NFS 服務的安裝，不同的Linux版本不一樣，具體的安裝方法可以baidu或者google，進行磁盤分區，如果使用ssd可以使用lvm緩存機制
4. 條帶化創建vg

5. /boot需要單獨的物理分區設置

   pvcreate針對現有的所有的硬盤

   vgcreate或者是vgextende 現有的vg

6. lvcreate -i 3（pv個數） -I 64 -l 1533 -n stripe_lv vg01
   lvcreate -L 50G -i2 -I64 -n gfslv vg0
   
   lvcreate --extents 100%FREE --stripes 5 --stripesize 256 --name nfs_lv ubuntu-vg
   
   擴展
   pvcreate之後，vgextend, 再lvextend
   
   http://tldp.org/HOWTO/LVM-HOWTO/recipethreescsistripe.html
   https://blog.csdn.net/cybertan/article/details/6773214
   
   vgextend
   lvextend -l +100%FREE  /dev/ubuntu-vg/nfs_lv
   resize2fs -p -F /dev/mapper/ubuntu--vg-nfs_lv
7. linux disk 優化
   NSF 服務優化
   磁盤調度算法
   echo deadline > /sys/block/sda/queue/scheduler
   echo deadline > /sys/block/sdb/queue/scheduler
   磁盤隊列
   echo 512 > /sys/block/sda/queue/nr_requests
   echo 512 > /sys/block/sdb/queue/nr_requests
   預讀塊
   echo 4096 >  /sys/block/sda/queue/read_ahead_kb
   echo 4096 >  /sys/block/sdb/queue/read_ahead_kb
   寫過期設置
   echo 1000 > /sys/block/sda/queue/iosched/write_expire
   echo 1000 > /sys/block/sdb/queue/iosched/write_expire
   讀過期設置，寫得1/2
   echo 500 > /sys/block/sda/queue/iosched/read_expire
   echo 500 > /sys/block/sdb/queue/iosched/read_expire
   linux I/O調度方式啓用異步方式，提高讀寫性能
   echo 40 > /proc/sys/vm/dirty_ratio
   echo 1500 > /proc/sys/vm/dirty_expire_centisecs
   echo 1000 > /proc/sys/vm/dirty_writeback_centisecs
   echo 20 > /proc/sys/vm/dirty_background_ratio
8. LVM mount優化 （關閉記錄文件系統atime）
   mount -o remount,noatime,nodiratime /dev/mapper/ubuntu--vg-nfs_lv 
9. 修改文件日誌的提交時間
   mount –o remount,commit=15 DEVICE
10. nfs mount優化
11. 內核及網絡優化

    net.core.netdev_max_backlog = 30000

    net.core.somaxconn = 262144

    net.core.rmem_default = 8388608

    net.core.wmem_default = 8388608

    net.core.rmem_max=16777216

    net.core.wmem_max=16777216

    net.ipv4.ip_local_port_range = 2048 65000

    net.ipv4.tcp_mem = 786432 1048576 1572864

    net.ipv4.tcp_rmem = 4096 87380 16777216

    net.ipv4.tcp_wmem = 4096 65536 16777216

    net.ipv4.tcp_fin_timeout = 10

    #net.ipv4.tcp_tw_recycle = 1

    net.ipv4.tcp_tw_reuse = 1

    net.ipv4.tcp_max_tw_buckets = 6000

    net.ipv4.tcp_timestamps = 0

    net.ipv4.tcp_window_scaling = 1

    net.ipv4.tcp_sack = 1

    net.ipv4.tcp_no_metrics_save=1

    net.ipv4.tcp_keepalive_time = 1200

    net.ipv4.tcp_syncookies = 1

    net.ipv4.tcp_max_orphans = 262144

    net.ipv4.tcp_max_syn_backlog = 262144

    net.ipv4.tcp_synack_retries = 2

    net.ipv4.tcp_syn_retries = 2

    #net.ipv4.netfilter.ip_conntrack_tcp_timeout_established = 1800

    #net.ipv4.netfilter.ip_conntrack_max = 131070

    kernel.sysrq = 0

    kernel.core_uses_pid = 1

    kernel.msgmnb = 65536

    kernel.msgmax = 65536

    kernel.shmmax = 68719476736

    kernel.shmall = 4294967296

    kernel.randomize_va_space = 1

    #kernel.exec-shield = 1

    net.ipv4.ip_forward = 0

    net.ipv4.neigh.default.gc_stale_time=120

    net.ipv4.conf.default.accept_source_route = 0

    net.ipv4.conf.default.rp_filter=0

    net.ipv4.conf.all.rp_filter=0

    net.ipv4.conf.all.arp_announce=2

    net.ipv4.conf.lo.arp_announce=2

    fs.file-max = 2000000

    fs.nr_open = 2000000

12. 打開net bbr算法
    # enable bbr

    net.core.default_qdisc = fq

    net.ipv4.tcp_congestion_control = bbr