生產服務器內存泄漏的排查過程

{"type":"doc","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"最近在排查生產上，應用佔用內存過大的問題，排查出來是jdk8bug+jetty服務器內存泄漏導致的，將過程記錄下來，大家一起探討。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":1},"content":[{"type":"text","text":"一、環境信息","attrs":{}}]},{"type":"bulletedlist","content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"操作系統：centos、","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"內存分配器：glibc2.17","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"JDK版本：jdk8u_101.b13","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"配置：4C8G 即4CPU+8G物理內存。","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"應用信息：java應用，垃圾收集器使用G1，-xmx=-xms = 4G","attrs":{}}]}],"attrs":{}}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":1},"content":[{"type":"text","text":"二、現象","attrs":{}}]},{"type":"bulletedlist","content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"jetty啓動應用後，Java應用進程佔用1.4~1.5G。","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"運行一段時間後，應用進程佔用5G+。","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"物理內存free -h,剩餘100-300M之間。","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"部分機器開始使用swap空間，範圍是：100～700M，一直緩慢持續上升。","attrs":{}}]}],"attrs":{}}],"attrs":{}},{"type":"heading","attrs":{"align":null,"level":1},"content":[{"type":"text","text":"三、排查過程","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"現狀：","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"free -h 顯示如下：","attrs":{}}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/2d/2d2d9d23b586c77c43bd35f5f199e165.png","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 操作系統開始使用內存，大家都知道，只有物理內存喫緊的時候纔會使用swap，也就是說明剩餘的物理內存剩餘不多了。操作系統啓動了 應急策略：使用swap將部分不活躍的內存轉移到swap區域以便騰出更多的內存空間。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 使用sar和vmstat命令，可以看到在頻繁的發生pageout到swap區域，暫未發生整個進程的內存頁swap out：","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 執行sar -B 1，顯示如下：","attrs":{}}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/97/97e276d6dcbea08bdd60bbdaa89a21e7.png","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"執行vmstat 1，顯示如下：","attrs":{}}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/43/4312c911c35bf607d45ae47a1ca6814a.png","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"觀察到我們的應用進程佔據的內存很大，top -pH 該進程，顯示如下：","attrs":{}}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/a5/a5eaf8acf7f1394eb92d63a3309c7ad4.gif","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 由上圖可知，應用進程佔用了5.6 G， 虛擬內存佔用了9.4G(那麼大！！！)，線程的個數是700個。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"在開發環境模擬生產流量運行一段時間後，監控","attrs":{}},{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"堆內堆外","attrs":{}},{"type":"text","text":"的內存使用情況，通過arthas工具（或者jconsole）觀察如下：","attrs":{}}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/20/2025cb2531a0ecc08fa00ad73ca4ea91.gif","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 通過上圖可知，我們的應用堆外內存的佔用情況是240M左右：","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" code_cache(95M) + metaspace(122M) + compressed_class_space(14M) + direct(724K) +mapped(0K) = 232M。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"按照配置堆大小是4G：-xms =-xmx =4G，JVM堆外內存佔據240M來計算的話，進程應該佔用的內存上限爲：","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"上限大小","attrs":{}},{"type":"text","text":" = 堆（4G）+ 非堆（JVM堆外，code_cache,、metaspace、compressed_class、direct等，240M） + 線程（400M，其實遠遠不到） = 4.625G。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 但是現在實際佔用已超過這個上限將近","attrs":{}},{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"1G","attrs":{}},{"type":"text","text":" 。這就足夠說明該應用的內存佔用是有問題的。是否存在內存泄漏需要排查下，主要排查地方如下：","attrs":{}}]},{"type":"bulletedlist","content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"1、堆內存，看看是否存在堆內存泄漏。","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"2、堆外內存（僅僅指JVM的堆外內存），看看是否存在泄漏。","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"3、Native Memory內存泄漏，這部分內存的使用是JVM管控和追蹤不到的地方。","attrs":{}}]}],"attrs":{}}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"3.1 堆內存和堆外內存排查","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 堆內存排查的最好方式是將應用的內存快照dump一份，使用Eclipse的MAT（Memory Analyzer Tool）工具進行分析，官網地址：","attrs":{}},{"type":"link","attrs":{"href":"https://www.eclipse.org/mat/","title":""},"content":[{"type":"text","text":"Memory Analyzer Tool","attrs":{}}]},{"type":"text","text":" ,它是一個強大的基於Eclipse的內存分析工具，也可以 獨立的安裝軟件，可以分析內存的使用情況，幫助我們找到內存泄露，減少內存消耗。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 使用命令進行dump內存，","attrs":{}},{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"切記這個操作會對應用造成影響，生產上使用要謹慎","attrs":{}},{"type":"text","text":" ，","attrs":{}}]},{"type":"codeblock","attrs":{"lang":"shell"},"content":[{"type":"text","text":"jmap -dump:live,format=b,file=appMem.bin pid ","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 將bin讀入到MAT後，選擇顯示可疑內存分析，呈現的結果如下：","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/c4/c45f8ee60e7919c62b9e91d65894b0c3.gif","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/d0/d09c802af8c216d23553ef98ce662b60.gif","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"*** ","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" MAT提示我們有兩個可疑的內存泄漏點：","attrs":{}}]},{"type":"bulletedlist","content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"com.alibab.dubbo.common.URL","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"com.alibaba.druid.pool.DruidDataSource","attrs":{}}]}],"attrs":{}}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 第一點我們的應用依賴的服務機器數量比較多，服務的數量也很多，所以有1萬的URL對象不足爲奇，同時alibba的URL也不涉及到資源的操作，所以可疑排除內存泄漏。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 第二點說明數據庫鏈接數，也沒有 什麼可疑的點。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 從可疑點並不能看出來什麼，也可以從首頁的Overview的各個指標，比如Histogram、Top Consume和Big Object等。可以參考","attrs":{}},{"type":"link","attrs":{"href":"https://blog.csdn.net/Jin_Kwok/article/details/80326088","title":""},"content":[{"type":"text","text":"MAT使用詳細介紹","attrs":{}}]},{"type":"text","text":" 這個教程。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 同時也通過atrthas、jconsole、jvisualvm進行跟蹤應用程序的跟蹤，可以看到堆內存的分佈：新生代Eden、老年代、堆外的幾個指標也很正常。同時通過arthas也可以生成內存使用的火焰圖，並沒有使用direct內存。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 這時候還是不能確認堆和堆外內存的使用是否正常，這時候使用NMT（Native Memory Tracking）對JVM的內存使用情況進行跟蹤。官網地址是：","attrs":{}},{"type":"link","attrs":{"href":"https://docs.oracle.com/javase/8/docs/technotes/guides/troubleshoot/tooldescr007.html","title":""},"content":[{"type":"text","text":"NMT的使用","attrs":{}}]},{"type":"text","text":",可以在應用的啓動參數中加上如下參數：","attrs":{}}]},{"type":"codeblock","attrs":{"lang":"shell"},"content":[{"type":"text","text":"-XX:NativeMemoryTracking=detail","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/80/8075fc6d3918c0d000eff941937a6591.gif","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 爲了追蹤NMT的各個內存區域的使用情況，編寫了一個每1小時採集的腳本，對NMT的各個內存區域使用情況進行統計分析：","attrs":{}}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/b2/b26895fcb09affbef9e42820c9c5e099.gif","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 運行幾天後，各個區域的內存使用無異常。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 到這兒基本上可以確定","attrs":{}},{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"應用的JVM堆是沒有問題的。","attrs":{}},{"type":"text","text":" ","attrs":{}}]},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"3.2 Native Memory排查","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 通過上面的分析可知，JVM的內存使用是沒有問題的，這時候就需要考慮JVM無法追蹤和監控的內存：原生內存。根據/proc/meminfo整理下操作系統的內存分佈情況:","attrs":{}}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/b4/b41a98fb5b8d57f6dd7b742c16fa1a96.gif","alt":null,"title":"","style":[{"key":"width","value":"25%"},{"key":"bordertype","value":"none"}],"href":"","fromPaste":false,"pastePass":false}},{"type":"paragraph","attrs":{"indent":1,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"對各個區域進行整理得出如下的內存分佈：","attrs":{}}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/72/7286af89fd5cce7df41e59299aaf770f.jpeg","alt":"linux內存分佈圖","title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 這時候寫了一個操作系統的meminfo的內存定時採集器，這個採集器主要定時採集/proc/meminfo的各個區域的內存情況：","attrs":{}}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/81/8182b21688c72f2c7c1cfad747b53933.gif","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"從上圖可以看到如下的變化：","attrs":{}}]},{"type":"bulletedlist","content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"MemFree持續下降並保持平穩，這也符合生產上的現狀，內存可用率持續下降。","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"Active(annon)、AnonHugePage、Dirty這兩個區域持續上漲。","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"其他的指標都是變化後基本上不再變化。","attrs":{}}]}],"attrs":{}}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" Active(annon)、AnonHugePage這個主要是進程的匿名內存塊使用在增長，Dirty是指這個進程使用的物理內存大小，是記錄邏輯地址與物理地址額映射關係，是真正被操作系統感知到使用的物理內存，這個值也一直在增長，說明我們使用的物理內存越來越多，但是我們使用Native Memory Tracking跟蹤程序，JVM整體各個區域是平穩的，不增長的。這時候疑點就出來了：","attrs":{}},{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"爲什麼JVM外的內存使用一直在增長？？。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 這時候可以使用pmap命令對進程使用的地址空間和大小進行查看，基本命令如下所示：","attrs":{}}]},{"type":"codeblock","attrs":{"lang":"shell"},"content":[{"type":"text","text":"pmap -x pid | sort -k 2 -r -n | head -50 ","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 執行命令後發現大量的64M內存塊不在Native Memory Tracking的地址管理空間中，統計了下大概有45+個64M的邏輯內存塊，第一列是地址，第二列是邏輯地址大小，第三列是實際使用的物理內存大小，對申請的物理內存（即第二列）排序後後如下： ","attrs":{}}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/ab/abce8710ef1c123b7f6200120041a8fe.gif","alt":null,"title":"","style":[{"key":"width","value":"50%"},{"key":"bordertype","value":"none"}],"href":"","fromPaste":false,"pastePass":false}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"又對第三列進行排序，看看內存的使用物理內存的情況：","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" ","attrs":{}}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/1c/1c52bc887716c067b7fa6e5e3f5643f6.png","alt":null,"title":"","style":[{"key":"width","value":"50%"},{"key":"bordertype","value":"none"}],"href":"","fromPaste":false,"pastePass":false}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"上面兩個圖，這些64M塊的邏輯地址（第一列）都可以在/proc/pid/smap中，也就是說這些地址確實是屬於這個進程申請的，但是爲什麼會申請那麼多的邏輯地址塊同時已經使用了那麼多物理內存呢？","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"選擇其中一個64M的內存塊地址，去/proc/pid/smap中找到這個塊的啓始地址，使用gdb工具將這地址塊的內存dump出來，看看是什麼內容：","attrs":{}}]},{"type":"codeblock","attrs":{"lang":"shell"},"content":[{"type":"text","text":"gdb --batch --pid -ex \"dump memory gdb_64M.dump 0x765e38000000 0x7f5e3bffa000\"","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"生成的gdb_64M.dump, 通過strings gdb_64M.dump 命令查看內存塊的內容，如下：","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/45/454ae8afd8e97c22358b97f57429185c.png","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 可以看到基本上都是一些類名稱，應該是存儲的解壓jar包的內容。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 把思路擴大點，看看爲什麼會產生64M這種東西，外事不決問谷歌，通過關鍵字“64M”、“內存”等關鍵字搜索相關的信息，確實搜到了相關信息，總結如下：","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"blockquote","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"64M內存塊是glibc申請的，glibc是一種內存管理器，內部存在一個內存池，應用頻繁申請的小內存和釋放內存都通過glibc內存管理器申請和釋放，這樣能夠減少與操作系統交互的次數，提高性能和效率。但是glibc持有的內存池因爲存在內存碎片問題，導致內存的釋放條件苛刻，導致進程 持有很大一塊內存，特別是多線程競爭激烈的情況下，容易出現進程持有大量的內存，導致內存耗盡，表現爲內存泄漏。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"glibc每次向操作系統申請一大塊內存（32位系統1M， 64位系統64M）進行切分成小塊進行管理。","attrs":{}}]}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/24/2429fbf49f4cfa02dcbcad460ecb7aca.png","alt":"在這裏插入圖片描述","title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 瞭解這些信息後，知道了原來是glibc在搞鬼，一直持有內存，不容易釋放，所以導致內存佔用過大。那就驗證下45+個64M是不是glibc申請的：","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"使用strace工具對應用對操作系統的內存申請(brk和mmap)與釋放（mummap）進行跟蹤，爲了能夠從應用啓動的時候就進行追蹤，在jetty啓動的時候就跟蹤：基本命令如下：","attrs":{}}]},{"type":"codeblock","attrs":{"lang":"shell"},"content":[{"type":"text","text":"strace -f -e \"brk,mmap,munmap\" -tt -o straceLog ./jetty.sh start","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/67/67202b45998ceb78734c0fa439019479.gif","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/fa/fa4b920e17bac62ecb6e0ec511726219.gif","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 第二個圖爲什麼申請的是2倍的64M，這是因爲glibc在申請內存時沒有采用鎖機制，而是採用CAS的方式，當多個線程同時申請64M的操作時，只有一個成功，其他都失敗。這時候失敗的線程就會第二次申請，這次申請的內存是128M，申請成功後 ，會釋放前面的64M，保留後面的64M地址空間。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 從上面的次數可以統計出48個64M內存塊。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 這兒基本上確定了64M內存是glibc申請和管理的。現在業界比較成熟的內存管理器有如下：","attrs":{}}]},{"type":"bulletedlist","content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"jemalloc – FreeBSD ","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"Firefoxtcmalloc – Google","attrs":{}}]}],"attrs":{}}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 這兩個內存管理器對內存的管理都比較有效，減少了內存碎片的產生，接下來使用tcmalloc替換 glibc後 ,安裝tcmalloc並在/etc/profile 文件中添加如下環境變量（注意tcmalloc是自己的安裝地址）","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"sudo vim /etc/profile：","attrs":{}}]},{"type":"codeblock","attrs":{"lang":"shell"},"content":[{"type":"text","text":"LD_PRELOAD=\"/usr/lib/libtcmalloc.so","attrs":{}}]},{"type":"paragraph","attrs":{"indent":1,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"然後執行source /etc/profile使得環境變量生效。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"經過幾天的測試後，結果如下：","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 操作系統中也沒有了64M的內存塊了，啓動時的內存佔用也變小了。 ","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 可以看出tcmalloc對內存的管理要好於glibc，可以替換成tcmalloc，應用進程佔用更少的內存。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"按道理到這兒就應該結束了，已經找到多出來的內存是什麼，是什麼導致了64內存塊的存在，怎麼優化下內存的管理方式減少內存塊個數。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 按道理查到這兒應該也可以了。","attrs":{}}]},{"type":"bulletedlist","content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"應用程序沒有出現內存泄漏。","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"64M內存塊是glibc持有的，且glibc因內存池和內存碎片的原因，容易使得進程佔用內存過大。","attrs":{}}]}],"attrs":{}}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 可是好奇害死貓～～～，還是想知道下爲什麼進程需要那麼多的64M內存。因爲根據glibc的特性，只有內存競爭激烈時纔會產生大量的64M內存塊，所以接下來想查明64M內存塊到底是誰使用的，怎麼去優化下不使用那麼多內存。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"3.3 服務器內存泄漏排查過程","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"現在需要查清楚的是：","attrs":{}}]},{"type":"bulletedlist","content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"是誰頻繁使用使用內存？","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"申請64M做什麼？","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"使用後爲什麼一直沒釋放？是否存在Native內存泄漏？","attrs":{}}]}],"attrs":{}}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"針對第一個問題：","attrs":{}},{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"是誰頻繁使用使用內存？","attrs":{}},{"type":"text","text":" ","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"編寫了一個Linux的hook函數，去統計線程的malloc次數，具體的可以參考","attrs":{}},{"type":"link","attrs":{"href":"https://blog.csdn.net/whbing1471/article/details/112394403","title":""},"content":[{"type":"text","text":"Linux的hook機制：自定義動態鏈接庫hook","attrs":{}}]},{"type":"text","text":",","attrs":{}}]},{"type":"codeblock","attrs":{"lang":"shell"},"content":[{"type":"text","text":"LD_PRELOAD=./malloc_hook.so ./jetty.sh start > mallocLog","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/e6/e6598d56e05474e81a21dcca8ced10f7.png","alt":null,"title":"","style":[{"key":"width","value":"25%"},{"key":"bordertype","value":"none"}],"href":"","fromPaste":false,"pastePass":false}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"使用如下命令對結果文檔進行統計：","attrs":{}}]},{"type":"codeblock","attrs":{"lang":"shell"},"content":[{"type":"text","text":"cat mallocLog | awk '{print $1}' | less | sort | uniq -c | sort -rn | less","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"統計出的結果如下：","attrs":{}}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/d4/d4c9654e3791f0e6d2ee6b090d5d5d95.png","alt":null,"title":"","style":[{"key":"width","value":"25%"},{"key":"bordertype","value":"none"}],"href":"","fromPaste":false,"pastePass":false}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"使用malloc最多線程pid=60153，通過jstack查看，這個線程是jetty的線程：","attrs":{}}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/13/1343eda210932f7cdc17f15de65e02b5.png","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"從堆棧可以看出jetty主線程在解壓jar包，難道是使用Inflater解壓jar需要那麼多內存，一直都聽說使用Inflater不當容易造成內存泄漏，是不是因爲使用Inflater造成內存泄漏了。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"針對第一個問題，來回答第二個問題，需要使用gperftools，這是google的內存分析利器，關於安裝和配置，可以參考官網，github地址：","attrs":{}},{"type":"link","attrs":{"href":"https://github.com/gperftools/gperftools","title":""},"content":[{"type":"text","text":"gperftools","attrs":{}}]},{"type":"text","text":".使用這個工具的前提是需要使用tcmalloc，這個已經在前文的配置中配置過了，使用如下命令重新啓動jetty即可：","attrs":{}}]},{"type":"codeblock","attrs":{"lang":"shell"},"content":[{"type":"text","text":"HEAPPROFILE=\"home/user/heap-gperftools/gzip\" ./jetty.sh start","attrs":{}}]},{"type":"blockquote","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"一定要注意，這個路徑存在目錄的話這個目錄一定要存在，不存在創建好。同時在這個目錄下會生成很多文件，一定要篩選出來進程的heap。還存在其他的參數控制，我採用的默認：","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"# 此配置表示進程每分配X字節內存輸出一個文件,默認是1G輸出一個文件，這個設置的是4G","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"export HEAP_PROFILE_ALLOCATION_INTERVAL=4073741824","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"# 此配置表示進程每使用X字節內存輸出一個文件，默認是100M輸出一個文件，這個設置的4G","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"export HEAP_PROFILE_INUSE_INTERVAL=4048576000","attrs":{}}]}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"啓動後會輸出以下信息：","attrs":{}}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/ea/ea6c615593ba4e2cedec402eb22b1475.png","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"從輸出信息也可以看出","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"生成的heap文件使用如下命令進行解析和排序：","attrs":{}}]},{"type":"codeblock","attrs":{"lang":"shell"},"content":[{"type":"text","text":"/home/xxx/gperftools/bin/pprof –text /usr/local/java8/bin/java gzip.0001.heap ","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/c0/c0866a7a6401c9914c1b1e69f413faa0.png","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"對結果排序後，得到的結果如下：","attrs":{}}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/96/9634b4f34c669b3321b3344630e45761.png","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/d1/d11f047a8066e9ee486c5ace1465b2ff.png","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 剛開始的第一張inflater使用內存不是很明顯 ，從第2張到第15張，inflater使用就非常的明顯，infalter是用於解壓縮器，使用那麼兇猛，一定是在在解壓東西。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 這時候使用btrace命令跟蹤某個函數的調用，btrace的腳本如下：","attrs":{}}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/b3/b320aee56b94dcf117e86c44241a97b8.png","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"跟蹤的調用結果如下：","attrs":{}}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/70/702c7bcb85ce45de3cd625c0e5e20d2c.png","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"從圖中可以看到，確實是jetty啓動線程使用解壓jar包。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"現在基本上結論是：","attrs":{}},{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"申請的64M，是jetty啓動時，掃描和解壓jar包在頻繁的使用內存。","attrs":{}},{"type":"text","text":" ","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 現在需要查明第三個問題：使用後爲什麼一直沒釋放？是否存在Native內存泄漏？","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"現在知道是因爲解壓jar包需要使用內存，glibc纔會申請64M內存，並且放在內存池中，可是解壓完成後爲什麼內存沒有釋放？","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 這時候就需要了解下jetty的啓動過程了。jetty啓動時爲了支持Servlet規範中的註解方式（使得不再需要在web.xml文件中進行Servlet的部署描述，簡化開發流程），jetty在啓動時會掃描class、lib包，使用ServiceLoader，將使用註解方式聲明的Servlet、Listener註冊到jetty容器，在掃描jar包的時候調用了inflate，分配了大量的內存。","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 現在從ServiceLoader這個類入手，分析類的加載過程，分析jdk的源碼，看看是否有內存泄漏，通過查看源碼和搜索相關問題進行確認，jdk8存在bug，有以下問題：","attrs":{}}]},{"type":"bulletedlist","content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"jdk8打開jar時是默認緩存的，緩存會佔用內存。","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"因爲緩存機制，jdk8的ServiceLoader和UrlClassLoader同時加載一個jar包文件，會導致這個jar包的文件句柄泄漏，文件句柄泄漏對redeploy危害巨大的，但是對我們這種重啓的應用危害性不大。","attrs":{}}]}],"attrs":{}}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"針對以上兩個問題的帖子如下：","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"link","attrs":{"href":"https://bugs.java.com/bugdatabase/view_bug.do?bug_id=8156014","title":""},"content":[{"type":"text","text":"JDK-8156014 : (sl) File handle leaked for jar of class loaded with java.util.ServiceLoader","attrs":{}}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"link","attrs":{"href":"https://bugs.java.com/bugdatabase/view_bug.do?bug_id=8013099","title":""},"content":[{"type":"text","text":"JDK-8013099 : (sl) ServiceLoader interferes with URLClassLoader.close()","attrs":{}}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"通過查看jdk源碼得知，jdk9-111已經修復了這個問題，將緩存去掉，如下所示：","attrs":{}}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/a6/a63cf3ec4cc4dc5914964d9a309b07ff.png","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"通過使用解決這個問題的jdk11，重啓應用後佔用的內存從：","attrs":{}},{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"1.4下降到1G。","attrs":{}},{"type":"text","text":" ","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"通過pmap查看使用中的64M的內存塊從啓動之初的","attrs":{}},{"type":"text","marks":[{"type":"strong","attrs":{}}],"text":"15+個降低到5+個。","attrs":{}},{"type":"text","text":" 優化前和優化後效果如下：","attrs":{}}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/97/9749874e98520a1896cd22565b60046b.png","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"還是有不少的使用的64M存在，是不是哪兒還有緩存內存或者內存泄漏，","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"通過查找jetty產品的發佈log，查查這些版本線都解決了那些bug，網站：","attrs":{}},{"type":"link","attrs":{"href":"https://gitee.com/mirrors/jetty/raw/jetty-9.4.x/VERSION.txt","title":""},"content":[{"type":"text","text":"https://gitee.com/mirrors/jetty/raw/jetty-9.4.x/VERSION.txt","attrs":{}}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"同時也在jetty的論壇上，通過搜索 memory、leak等關鍵字搜索哪些bug是目前使用版本存在，被高版本修復的，正好定位到一個bug：","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"link","attrs":{"href":"https://github.com/eclipse/jetty.project/issues/575","title":""},"content":[{"type":"text","text":"Memory leak while scanning annotations #575 ","attrs":{}}]},{"type":"text","text":" ","attrs":{}}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"確定是jetty掃描資源時的泄漏，這時通過源碼確認這個bug並且確定這個bug是在那個時候版本上進行修復的：9.3.4，修復的情況如下，使用try-catch進行資源關閉：","attrs":{}}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/80/802c10bd6d77339ba93fdaa477c35112.png","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":" 如果想了解爲什麼關閉inputstram會導致內存泄漏 ，可以看看源碼，這個也是和jdk8的默認緩存相關，默認緩存了又不關閉通過 InputStream去關閉或者清理對應的Inflater（end()或者reset()）,這不就導致內存泄漏了嗎！！！關閉流程如下所示：","attrs":{}}]},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/a0/a0c911f7e3570b2485cc60faeef255ee.png","alt":null,"title":null,"style":[{"key":"width","value":"75%"},{"key":"bordertype","value":"none"}],"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"知道這個問題後，將jetty版本升級到解決 這個問題的較高版本，重啓應用後：","attrs":{}}]},{"type":"bulletedlist","content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"內存佔用從1G降低到600～700M。","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"64M的使用塊1～2個。","attrs":{}}]}],"attrs":{}}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":1},"content":[{"type":"text","text":"四、結論及優化方案","attrs":{}}]},{"type":"bulletedlist","content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"jdk8有bug，這個bug不影響我們的應用，jdk較高版本在ServiceLoader時禁用了緩存，節省了300M的堆外內存使用。-----解決方案：升級到jdk9-111及以上 版本。","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"AnnotationParser.java 存在未關閉文件流的問題，同時由於jdk8的緩存機制，導致內存泄露。----解決方案：升級jetty-9.3.4及以上版本。","attrs":{}}]}],"attrs":{}},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"glibc 64M大內存塊問題，這是glibc的產品特性，如果多線程高併發的應用，建議使用tcmalloc或者jemalloc，能更好的管理內存。","attrs":{}}]}],"attrs":{}}],"attrs":{}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"大功告成了！！！！！！以上的建議在上生產前，請做好生產壓測驗證。","attrs":{}}]}]}