一次完整的JVM堆外內存泄漏故障排查記錄

{"type":"doc","content":[{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"前言"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"記錄一次線上JVM堆外內存泄漏問題的排查過程與思路，其中夾帶一些"},{"type":"text","marks":[{"type":"strong"}],"text":"JVM內存分配機制"},{"type":"text","text":"以及*"},{"type":"text","marks":[{"type":"italic"}],"text":"常用的JVM問題排查指令和工具分享"},{"type":"text","text":"*，希望對大家有所幫助。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"在整個排查過程中，我也走了不少彎路，但是在文章中我仍然會把完整的思路和想法寫出來，當做一次經驗教訓，給後人參考，文章最後也總結了下內存泄漏問題快速排查的幾個原則。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"本文的主要內容："}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"故障描述和排查過程"}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"故障原因和解決方案分析"}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"JVM堆內內存和堆外內存分配原理"}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"常用的進程內存泄漏排查指令和工具介紹和使用"}]}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"blockquote","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"文章撰寫不易，請大家多多支持我的原創技術公衆號：後端技術漫談"}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"故障描述"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"8月12日中午午休時間，我們商業服務收到告警，服務進程佔用容器的物理內存（16G）超過了80%的閾值，並且還在不斷上升。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/b6/b6a0441d79e7a4e61ed7a6e7c982ef97.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"監控系統調出圖表查看："}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/e6/e6d4f03c8933366e4989b65491c5335c.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"像是Java進程發生了內存泄漏，而我們堆內存的限制是4G，這種大於4G快要喫滿內存應該是JVM堆外內存泄漏。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"確認了下當時服務進程的啓動配置："}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"-Xms4g -Xmx4g -Xmn2g -Xss1024K -XX:PermSize=256m -XX:MaxPermSize=512m -XX:ParallelGCThreads=20 -XX:+UseConcMarkSweepGC -XX:+UseParNewGC -XX:+UseCMSCompactAtFullCollection -XX:CMSInitiatingOccupancyFraction=80"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"雖然當天沒有上線新代碼，但是"},{"type":"text","marks":[{"type":"strong"}],"text":"當天上午我們正在使用消息隊列推送歷史數據的修復腳本，該任務會大量調用我們服務其中的某一個接口"},{"type":"text","text":"，所以初步懷疑和該接口有關。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"下圖是該調用接口當天的訪問量變化："}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/65/65f87da50056aa4ec6bef22d155a41b4.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"可以看到案發當時調用量相比正常情況（每分鐘200+次）提高了很多（每分鐘5000+次）。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"我們暫時讓腳本停止發送消息，該接口調用量下降到每分鐘200+次，容器內存不再以極高斜率上升，一切似乎恢復了正常。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"接下來排查這個接口是不是發生了內存泄漏。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"排查過程"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"首先我們先回顧下Java進程的內存分配，方便我們下面排查思路的闡述。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"以我們線上使用的JDK1.8版本爲例"},{"type":"text","text":"。JVM內存分配網上有許多"},{"type":"link","attrs":{"href":"https://www.cnblogs.com/duanxz/p/3520829.html","title":""},"content":[{"type":"text","text":"總結"}]},{"type":"text","text":"，我就不再進行二次創作。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"JVM內存區域的劃分爲兩塊：堆區和非堆區。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"堆區：就是我們熟知的新生代老年代。"}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"非堆區：非堆區如圖中所示，有元數據區和直接內存。"}]}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/7f/7f8e40210014834a29424801be0531ac.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"這裏需要額外注意的是：永久代（JDK8的原生去）存放JVM運行時使用的類，永久代的對象在full GC時進行垃圾收集。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"複習完了JVM的內存分配，讓我們回到故障上來。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":3},"content":[{"type":"text","text":"堆內存分析"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"雖說一開始就基本確認與堆內存無關，因爲泄露的內存佔用超過了堆內存限制4G，但是我們爲了保險起見先看下堆內存有什麼線索。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"我們觀察了新生代和老年代內存佔用曲線以及回收次數統計，和往常一樣沒有大問題，我們接着在事故現場的容器上dump了一份JVM堆內存的日誌。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":4},"content":[{"type":"text","text":"堆內存Dump"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"堆內存快照dump命令："}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"jmap -dump:live,format=b,file=xxxx.hprof pid"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"blockquote","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"畫外音：你也可以使用jmap -histo:live pid直接查看堆內存存活的對象。"}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"導出後，將Dump文件下載回本地，然後可以使用Eclipse的MAT（Memory Analyzer）或者JDK自帶的JVisualVM打開日誌文件。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"使用MAT打開文件如圖所示："}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/70/7014cceacf720992e884015247a6585d.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"可以看到堆內存中，有一些nio有關的大對象，比如正在接收消息隊列消息的nioChannel，還有nio.HeapByteBuffer，但是數量不多，不能作爲判斷的依據，先放着觀察下。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"下一步，我開始瀏覽該接口代碼，接口內部主要邏輯是調用集團的WCS客戶端，將數據庫表中數據查表後寫入WCS，沒有其他額外邏輯"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"發覺沒有什麼特殊邏輯後，我開始懷疑WCS客戶端封裝是否存在內存泄漏，這樣懷疑的理由是，WCS客戶端底層是由SCF客戶端封裝的，作爲RPC框架，其底層通訊傳輸協議有可能會申請直接內存。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"是不是我的代碼出發了WCS客戶端的Bug，導致不斷地申請直接內存的調用，最終喫滿內存。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"我聯繫上了WCS的值班人，將我們遇到的問題和他們描述了一下，他們回覆我們，會在他們本地執行下寫入操作的壓測，看看能不能復現我們的問題。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"既然等待他們的反饋還需要時間，我們就準備先自己琢磨下原因。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"我將懷疑的目光停留在了直接內存上，懷疑是由於接口調用量過大，客戶端對nio使用不當，導致使用ByteBuffer申請過多的直接內存。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"blockquote","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"畫外音：最終的結果證明，這一個先入爲主的思路導致排查過程走了彎路。在問題的排查過程中，用合理的猜測來縮小排查範圍是可以的，但最好先把每種可能性都列清楚，在發現自己深入某個可能性無果時，要及時回頭仔細審視其他可能性。"}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":3},"content":[{"type":"text","text":"沙箱環境復現"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"爲了能還原當時的故障場景，我在沙箱環境申請了一臺壓測機器，來確保和線上環境一致。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"首先我們先模擬內存溢出的情況（大量調用接口）："}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"我們讓腳本繼續推送數據，調用我們的接口，我們持續觀察內存佔用。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"當開始調用後，內存便開始持續增長，並且看起來沒有被限制住（沒有因爲限制觸發Full GC）。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/92/92e66c9d391107e8b0356e493848ee19.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"接着我們來模擬下平時正常調用量的情況（正常量調用接口）："}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"我們將該接口平時正常的調用量（比較小，且每10分鐘進行一次批量調用）切到該壓測機器上，得到了下圖這樣的老生代內存和物理內存趨勢："}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/99/998b485b4f312935db2658ffe2cdd157.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/d6/d612c11849878ddf90ea6b0eb4bfed92.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"問題來了：爲何內存會不斷往上走喫滿內存呢？"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"當時猜測是由於JVM進程並沒有對於直接內存大小進行限制（-XX:MaxDirectMemorySize），所以堆外內存不斷上漲，並不會觸發FullGC操作。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"上圖能夠得出兩個結論："}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"在內存泄露的接口調用量很大的時候，如果恰好堆內老生代等其他情況一直不滿足FullGC條件，就一直不會FullGC，直接內存一路上漲。"}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"而在平時低調用量的情況下， 內存泄漏的比較慢，FullGC總會到來，回收掉泄露的那部分，這也是平時沒有出問題，正常運行了很久的原因。"}]}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"由於上面提到，我們進程的啓動參數中並沒有限制直接內存，於是我們將-XX:MaxDirectMemorySize配置加上，再次在沙箱環境進行了測驗。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"結果發現，進程佔用的物理內存依然會不斷上漲，超出了我們設置的限制，“看上去”配置似乎沒起作用。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"這讓我很訝異，難道JVM對內存的限制出現了問題？"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"到了這裏，能夠看出我排查過程中思路執着於直接內存的泄露，一去不復返了。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"blockquote","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"畫外音：我們應該相信JVM對內存的掌握，如果發現參數失效，多從自己身上找原因，看看是不是自己使用參數有誤。"}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":3},"content":[{"type":"text","text":"直接內存分析"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"爲了更進一步的調查清楚直接內存裏有什麼，我開始對直接內存下手。由於直接內存並不能像堆內存一樣，很容易的看出所有佔用的對象，我們需要一些命令來對直接內存進行排查，我有用了幾種辦法，來查看直接內存裏到底出現了什麼問題。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":4},"content":[{"type":"text","text":"查看進程內存信息 pmap"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"pmap - report memory map of a process(查看進程的內存映像信息)"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"pmap命令用於報告進程的內存映射關係，是Linux調試及運維一個很好的工具。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"pmap -x pid 如果需要排序 | sort -n -k3**"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"執行後我得到了下面的輸出，刪減輸出如下:"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"..\n00007fa2d4000000 8660 8660 8660 rw--- [ anon ]\n00007fa65f12a000 8664 8664 8664 rw--- [ anon ]\n00007fa610000000 9840 9832 9832 rw--- [ anon ]\n00007fa5f75ff000 10244 10244 10244 rw--- [ anon ]\n00007fa6005fe000 59400 10276 10276 rw--- [ anon ]\n00007fa3f8000000 10468 10468 10468 rw--- [ anon ]\n00007fa60c000000 10480 10480 10480 rw--- [ anon ]\n00007fa614000000 10724 10696 10696 rw--- [ anon ]\n00007fa6e1c59000 13048 11228 0 r-x-- libjvm.so\n00007fa604000000 12140 12016 12016 rw--- [ anon ]\n00007fa654000000 13316 13096 13096 rw--- [ anon ]\n00007fa618000000 16888 16748 16748 rw--- [ anon ]\n00007fa624000000 37504 18756 18756 rw--- [ anon ]\n00007fa62c000000 53220 22368 22368 rw--- [ anon ]\n00007fa630000000 25128 23648 23648 rw--- [ anon ]\n00007fa63c000000 28044 24300 24300 rw--- [ anon ]\n00007fa61c000000 42376 27348 27348 rw--- [ anon ]\n00007fa628000000 29692 27388 27388 rw--- [ anon ]\n00007fa640000000 28016 28016 28016 rw--- [ anon ]\n00007fa620000000 28228 28216 28216 rw--- [ anon ]\n00007fa634000000 36096 30024 30024 rw--- [ anon ]\n00007fa638000000 65516 40128 40128 rw--- [ anon ]\n00007fa478000000 46280 46240 46240 rw--- [ anon ]\n0000000000f7e000 47980 47856 47856 rw--- [ anon ]\n00007fa67ccf0000 52288 51264 51264 rw--- [ anon ]\n00007fa6dc000000 65512 63264 63264 rw--- [ anon ]\n00007fa6cd000000 71296 68916 68916 rwx-- [ anon ]\n00000006c0000000 4359360 2735484 2735484 rw--- [ anon ]"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"可以看出，最下面一行是堆內存的映射，佔用4G，其他上面有非常多小的內存佔用，不過通過這些信息我們依然看不出問題。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":4},"content":[{"type":"text","text":"堆外內存跟蹤 NativeMemoryTracking "}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"blockquote","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"Native Memory Tracking (NMT) 是Hotspot VM用來分析VM內部內存使用情況的一個功能。我們可以利用jcmd（jdk自帶）這個工具來訪問NMT的數據。"}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"NMT必須先通過VM啓動參數中打開，不過要注意的是，打開NMT會帶來5%-10%的性能損耗。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"-XX:NativeMemoryTracking=[off | summary | detail]\n# off: 默認關閉\n# summary: 只統計各個分類的內存使用情況.\n# detail: Collect memory usage by individual call sites."}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"然後運行進程，可以使用下面的命令查看直接內存："}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"jcmd VM.native_memory [summary | detail | baseline | summary.diff | detail.diff | shutdown] [scale= KB | MB | GB]\n \n# summary: 分類內存使用情況.\n# detail: 詳細內存使用情況，除了summary信息之外還包含了虛擬內存使用情況。\n# baseline: 創建內存使用快照，方便和後面做對比\n# summary.diff: 和上一次baseline的summary對比\n# detail.diff: 和上一次baseline的detail對比\n# shutdown: 關閉NMT"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"我們使用："}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"jcmd pid VM.native_memory detail scale=MB > temp.txt"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"得到如圖結果："}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/96/96044f08d3c80a1297e9d1730dc50638.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/9c/9cdbd542d579d3b1c81855fec28a741b.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"上圖中給我們的信息，都不能很明顯的看出問題，至少我當時依然不能通過這幾次信息看出問題。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"排查似乎陷入了僵局。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":3},"content":[{"type":"text","text":"山重水複疑無路"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"在排查陷入停滯的時候，我們得到了來自WCS和SCF方面的回覆，"},{"type":"text","marks":[{"type":"strong"}],"text":"兩方都確定了他們的封裝沒有內存泄漏的存在"},{"type":"text","text":"，WCS方面沒有使用直接內存，而SCF雖然作爲底層RPC協議，但是也不會遺留這麼明顯的內存bug，否則應該線上有很多反饋。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":4},"content":[{"type":"text","text":"查看JVM內存信息 jmap"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"此時，找不到問題的我再次新開了一個沙箱容器，運行服務進程，然後運行jmap命令，看一看JVM內存的"},{"type":"text","marks":[{"type":"strong"}],"text":"實際配置"},{"type":"text","text":"："}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"jmap -heap pid"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"得到結果："}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"Attaching to process ID 1474, please wait...\nDebugger attached successfully.\nServer compiler detected.\nJVM version is 25.66-b17\n\nusing parallel threads in the new generation.\nusing thread-local object allocation.\nConcurrent Mark-Sweep GC\n\nHeap Configuration:\n MinHeapFreeRatio = 40\n MaxHeapFreeRatio = 70\n MaxHeapSize = 4294967296 (4096.0MB)\n NewSize = 2147483648 (2048.0MB)\n MaxNewSize = 2147483648 (2048.0MB)\n OldSize = 2147483648 (2048.0MB)\n NewRatio = 2\n SurvivorRatio = 8\n MetaspaceSize = 21807104 (20.796875MB)\n CompressedClassSpaceSize = 1073741824 (1024.0MB)\n MaxMetaspaceSize = 17592186044415 MB\n G1HeapRegionSize = 0 (0.0MB)\n\nHeap Usage:\nNew Generation (Eden + 1 Survivor Space):\n capacity = 1932787712 (1843.25MB)\n used = 1698208480 (1619.5378112792969MB)\n free = 234579232 (223.71218872070312MB)\n 87.86316621615607% used\nEden Space:\n capacity = 1718091776 (1638.5MB)\n used = 1690833680 (1612.504653930664MB)\n free = 27258096 (25.995346069335938MB)\n 98.41346682518548% used\nFrom Space:\n capacity = 214695936 (204.75MB)\n used = 7374800 (7.0331573486328125MB)\n free = 207321136 (197.7168426513672MB)\n 3.4349974840697497% used\nTo Space:\n capacity = 214695936 (204.75MB)\n used = 0 (0.0MB)\n free = 214695936 (204.75MB)\n 0.0% used\nconcurrent mark-sweep generation:\n capacity = 2147483648 (2048.0MB)\n used = 322602776 (307.6579818725586MB)\n free = 1824880872 (1740.3420181274414MB)\n 15.022362396121025% used\n\n29425 interned Strings occupying 3202824 bytes"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"輸出的信息中，看得出老年代和新生代都蠻正常的，元空間也只佔用了20M，直接內存看起來也是2g..."}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"嗯？爲什麼"},{"type":"codeinline","content":[{"type":"text","text":"MaxMetaspaceSize = 17592186044415 MB"}]},{"type":"text","text":"？"},{"type":"text","marks":[{"type":"strong"}],"text":"看起來就和沒限制一樣"},{"type":"text","text":"。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"再仔細看看我們的啓動參數："}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"-Xms4g -Xmx4g -Xmn2g -Xss1024K -XX:PermSize=256m -XX:MaxPermSize=512m -XX:ParallelGCThreads=20 -XX:+UseConcMarkSweepGC -XX:+UseParNewGC -XX:+UseCMSCompactAtFullCollection -XX:CMSInitiatingOccupancyFraction=80"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"配置的是"},{"type":"codeinline","content":[{"type":"text","text":"-XX:PermSize=256m -XX:MaxPermSize=512m"}]},{"type":"text","text":"，也就是永久代的內存空間。"},{"type":"text","marks":[{"type":"strong"}],"text":"而1.8後，Hotspot虛擬機已經移除了永久代，使用了元空間代替。"},{"type":"text","text":" 由於我們線上使用的是JDK1.8，"},{"type":"text","marks":[{"type":"strong"}],"text":"所以我們對於元空間的最大容量根本就沒有做限制"},{"type":"text","text":"，"},{"type":"codeinline","content":[{"type":"text","text":"-XX:PermSize=256m -XX:MaxPermSize=512m"}]},{"type":"text","text":" 這兩個參數對於1.8就是過期的參數。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"下面的圖描述了從1.7到1.8，永久代的變更："}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/15/15e9ca6e59a7483450315444a1bc12cb.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"那會不會是元空間內存泄露了呢？"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"我選擇了在本地進行測試，方便更改參數，也方便使用JVisualVM工具直觀的看出內存變化。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":4},"content":[{"type":"text","text":"使用JVisualVM觀察進程運行"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"首先限制住元空間，使用參數"},{"type":"codeinline","content":[{"type":"text","text":"-XX:MetaspaceSize=64m -XX:MaxMetaspaceSize=128m"}]},{"type":"text","text":"，然後在本地循環調用出問題的接口。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"得到如圖："}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/17/17f2b922a6d36ce0296aadbd22eec84a.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"可以看出，在元空間耗盡時，系統出發了Full GC，元空間內存得到回收，並且卸載了很多類。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"然後我們將元空間限制去掉，也就是使用之前出問題的參數："}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"-Xms4g -Xmx4g -Xmn2g -Xss1024K -XX:ParallelGCThreads=20 -XX:+UseConcMarkSweepGC -XX:+UseParNewGC -XX:+UseCMSCompactAtFullCollection -XX:CMSInitiatingOccupancyFraction=80 -XX:MaxDirectMemorySize=2g -XX:+UnlockDiagnosticVMOptions"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"得到如圖："}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/14/1464a342f890df4cfb0c993d26ac345c.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"可以看出，元空間在不斷上漲，並且已裝入的類隨着調用量的增加也在不斷上漲，呈現正相關趨勢。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":3},"content":[{"type":"text","text":"柳暗花明又一村"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"問題一下子明朗了起來，"},{"type":"text","marks":[{"type":"strong"}],"text":"隨着每次接口的調用，極有可能是某個類都在不斷的被創建，佔用了元空間的內存"},{"type":"text","text":"。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":4},"content":[{"type":"text","text":"觀察JVM類加載情況 -verbose"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"blockquote","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"在調試程序時，有時需要查看程序加載的類、內存回收情況、調用的本地接口等。這時候就需要-verbose命令。在myeclipse可以通過右鍵設置（如下），也可以在命令行輸入java -verbose來查看。"}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"-verbose:class 查看類加載情況\n-verbose:gc 查看虛擬機中內存回收情況\n-verbose:jni 查看本地方法調用的情況"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"我們在本地環境，添加啓動參數"},{"type":"codeinline","content":[{"type":"text","text":"-verbose:class"}]},{"type":"text","text":"循環調用接口。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"可以看到生成了無數"},{"type":"codeinline","content":[{"type":"text","text":"com.alibaba.fastjson.serializer.ASMSerializer_1_WlkCustomerDto"}]},{"type":"text","text":":"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"[Loaded com.alibaba.fastjson.serializer.ASMSerializer_1_WlkCustomerDto from file:/C:/Users/yangzhendong01/.m2/repository/com/alibaba/fastjson/1.2.71/fastjson-1.2.71.jar]\n[Loaded com.alibaba.fastjson.serializer.ASMSerializer_1_WlkCustomerDto from file:/C:/Users/yangzhendong01/.m2/repository/com/alibaba/fastjson/1.2.71/fastjson-1.2.71.jar]\n[Loaded com.alibaba.fastjson.serializer.ASMSerializer_1_WlkCustomerDto from file:/C:/Users/yangzhendong01/.m2/repository/com/alibaba/fastjson/1.2.71/fastjson-1.2.71.jar]\n[Loaded com.alibaba.fastjson.serializer.ASMSerializer_1_WlkCustomerDto from file:/C:/Users/yangzhendong01/.m2/repository/com/alibaba/fastjson/1.2.71/fastjson-1.2.71.jar]\n[Loaded com.alibaba.fastjson.serializer.ASMSerializer_1_WlkCustomerDto from file:/C:/Users/yangzhendong01/.m2/repository/com/alibaba/fastjson/1.2.71/fastjson-1.2.71.jar]\n[Loaded com.alibaba.fastjson.serializer.ASMSerializer_1_WlkCustomerDto from file:/C:/Users/yangzhendong01/.m2/repository/com/alibaba/fastjson/1.2.71/fastjson-1.2.71.jar]"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"當調用了很多次，積攢了一定的類時，我們手動執行Full GC，進行類加載器的回收，我們發現大量的fastjson相關類被回收。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/db/dbb2974b290f6936de67c526553d1f4b.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"如果在回收前，使用jmap查看類加載情況，同樣也可以發現大量的fastjson相關類："}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"jmap -clstats 7984"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/83/836aba0a9281bda9ca25c356d46164fe.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"這下有了方向，"},{"type":"text","marks":[{"type":"strong"}],"text":"這次仔細排查代碼"},{"type":"text","text":"，查看代碼邏輯裏哪裏用到了fastjson，發現瞭如下代碼："}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"/**\n * 返回Json字符串.駝峯轉_\n * @param bean 實體類.\n */\npublic static String buildData(Object bean) {\n try {\n SerializeConfig CONFIG = new SerializeConfig();\n CONFIG.propertyNamingStrategy = PropertyNamingStrategy.SnakeCase;\n return jsonString = JSON.toJSONString(bean, CONFIG);\n } catch (Exception e) {\n return null;\n }\n}"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":3},"content":[{"type":"text","text":"問題根因"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"我們在調用wcs前將駝峯字段的實體類序列化成下劃線字段，**這需要使用fastjson的SerializeConfig，而我們在靜態方法中對其進行了實例化。SerializeConfig創建時默認會創建一個ASM代理類用來實現對目標對象的序列化。也就是上面被頻繁創建的類"},{"type":"codeinline","content":[{"type":"text","text":"com.alibaba.fastjson.serializer.ASMSerializer_1_WlkCustomerDto"}]},{"type":"text","text":",如果我們複用SerializeConfig，fastjson會去尋找已經創建的代理類，從而複用。但是如果new SerializeConfig()，則找不到原來生成的代理類，就會一直去生成新的WlkCustomerDto代理類。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"下面兩張圖時問題定位的源碼："}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/57/5729a480f23d39f271b8654e8c252bbd.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/d2/d2b45e02ffe540bcb0314ed031eec54a.png","alt":null,"title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"我們將SerializeConfig作爲類的靜態變量，問題得到了解決。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"codeblock","attrs":{"lang":""},"content":[{"type":"text","text":"private static final SerializeConfig CONFIG = new SerializeConfig();\n\nstatic {\n CONFIG.propertyNamingStrategy = PropertyNamingStrategy.SnakeCase;\n}"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":4},"content":[{"type":"text","text":"fastjson SerializeConfig 做了什麼"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"SerializeConfig介紹："}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"blockquote","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"SerializeConfig的主要功能是配置並記錄每種Java類型對應的序列化類（ObjectSerializer接口的實現類），比如Boolean.class使用BooleanCodec（看命名就知道該類將序列化和反序列化實現寫到一起了）作爲序列化實現類，float[].class使用FloatArraySerializer作爲序列化實現類。這些序列化實現類，有的是FastJSON中默認實現的（比如Java基本類），有的是通過ASM框架生成的（比如用戶自定義類），有的甚至是用戶自定義的序列化類（比如Date類型框架默認實現是轉爲毫秒，應用需要轉爲秒）。當然，這就涉及到是使用ASM生成序列化類還是使用JavaBean的序列化類類序列化的問題，這裏判斷根據就是是否Android環境（環境變量\"java.vm.name\"爲\"dalvik\"或\"lemur\"就是Android環境），但判斷不僅這裏一處，後續還有更具體的判斷。"}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"理論上來說，每個SerializeConfig實例若序列化相同的類，都會找到之前生成的該類的代理類，來進行序列化。們的服務在每次接口被調用時，都實例化一個ParseConfig對象來配置Fastjson反序列的設置，而未禁用ASM代理的情況下，由於每次調用ParseConfig都是一個新的實例，因此永遠也檢查不到已經創建的代理類，所以Fastjson便不斷的創建新的代理類，並加載到metaspace中，最終導致metaspace不斷擴張，將機器的內存耗盡。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":4},"content":[{"type":"text","text":"升級JDK1.8纔會出現問題"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"導致問題發生的原因還是值得重視。爲什麼在升級之前不會出現這個問題？這就要分析jdk1.8和1.7自帶的hotspot虛擬機的差異了。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"blockquote","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"從jdk1.8開始，自帶的hostspot虛擬機取消了過去的永久區，而新增了metaspace區，從功能上看，metaspace可以認爲和永久區類似，其最主要的功用也是存放類元數據，但實際的機制則有較大的不同。"}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":">"}]},{"type":"blockquote","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"首先，metaspace默認的最大值是整個機器的物理內存大小，所以metaspace不斷擴張會導致java程序侵佔系統可用內存，最終系統沒有可用的內存；而永久區則有固定的默認大小，不會擴張到整個機器的可用內存。當分配的內存耗盡時，兩者均會觸發full gc，但不同的是永久區在full gc時，以堆內存回收時類似的機制去回收永久區中的類元數據（Class對象），只要是根引用無法到達的對象就可以回收掉，而metaspace判斷類元數據是否可以回收，是根據加載這些類元數據的Classloader是否可以回收來判斷的，只要Classloader不能回收，通過其加載的類元數據就不會被回收。這也就解釋了我們這兩個服務爲什麼在升級到1.8之後纔出現問題，因爲在之前的jdk版本中，雖然每次調用fastjson都創建了很多代理類，在永久區中加載類很多代理類的Class實例，但這些Class實例都是在方法調用是創建的，調用完成之後就不可達了，因此永久區內存滿了觸發full gc時，都會被回收掉。"}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":">"}]},{"type":"blockquote","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"而使用1.8時，因爲這些代理類都是通過主線程的Classloader加載的，這個Classloader在程序運行的過程中永遠也不會被回收，因此通過其加載的這些代理類也永遠不會被回收，這就導致metaspace不斷擴張，最終耗盡機器的內存了。"}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"這個問題並不侷限於fastjson，只要是需要通過程序加載創建類的地方，就有可能出現這種問題。"},{"type":"text","marks":[{"type":"strong"}],"text":"尤其是在框架中，往往大量採用類似ASM、javassist等工具進行字節碼增強，而根據上面的分析，在jdk1.8之前，因爲大多數情況下動態加載的Class都能夠在full gc時得到回收，因此不容易出現問題"},{"type":"text","text":"，也因此很多框架、工具包並沒有針對這個問題做一些處理，一旦升級到1.8之後，這些問題就可能會暴露出來。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"總結"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"問題解決了，接下來複盤下整個排查問題的流程，整個流程暴露了我很多問題，最主要的就是"},{"type":"text","marks":[{"type":"strong"}],"text":"對於JVM不同版本的內存分配還不夠熟悉"},{"type":"text","text":"，導致了對於老生代和元空間判斷失誤，走了很多彎路，在直接內存中排查了很久，浪費了很多時間。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"其次，排查需要的"},{"type":"text","marks":[{"type":"strong"}],"text":"一是仔細，二是全面，"},{"type":"text","text":"，最好將所有可能性先行整理好，不然很容易陷入自己設定好的排查範圍內，走進死衚衕不出來。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"最後，總結一下這次的問題帶來的收穫："}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"JDK1.8開始，自帶的hostspot虛擬機取消了過去的永久區，而新增了metaspace區，從功能上看，metaspace可以認爲和永久區類似，其最主要的功用也是存放類元數據，但實際的機制則有較大的不同。"}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"對於JVM裏面的內存需要在啓動時進行限制，包括我們熟悉的堆內存，也要包括直接內存和元生區，這是保證線上服務正常運行最後的兜底。"}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"使用類庫，請多注意代碼的寫法，儘量不要出現明顯的內存泄漏。"}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"對於使用了ASM等字節碼增強工具的類庫，在使用他們時請多加小心（尤其是JDK1.8以後）。"}]}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"blockquote","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"文章撰寫不易，請大家多多支持我的原創技術公衆號：後端技術漫談"}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"參考"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"觀察程序運行時類加載的過程"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"blog.csdn.net/tenderhearted/article/details/39642275"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"Metaspace整體介紹（永久代被替換原因、元空間特點、元空間內存查看分析方法）"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"https://www.cnblogs.com/duanxz/p/3520829.html"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"java內存佔用異常問題常見排查流程（含堆外內存異常）"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"https://my.oschina.net/haitaohu/blog/3024843"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"JVM源碼分析之堆外內存完全解讀"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"http://lovestblog.cn/blog/2015/05/12/direct-buffer/"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"JVM 類的卸載"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"https://www.cnblogs.com/caoxb/p/12735525.html"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"fastjson在jdk1.8上面開啓asm"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"https://github.com/alibaba/fastjson/issues/385"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"fastjson：PropertyNamingStrategy_cn"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"https://github.com/alibaba/fastjson/wiki/PropertyNamingStrategy_cn"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"警惕動態代理導致的Metaspace內存泄漏問題"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"https://blog.csdn.net/xyghehehehe/article/details/78820135"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":2},"content":[{"type":"text","text":"關注我"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"我是一名後端開發工程師。主要關注後端開發，數據安全，爬蟲，物聯網，邊緣計算等方向，歡迎交流。"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":3},"content":[{"type":"text","text":"各大平臺都可以找到我"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"微信公衆號：後端技術漫談"}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"**Github："},{"type":"link","attrs":{"href":"https://github.com/qqxx6661","title":""},"content":[{"type":"text","text":"@qqxx6661"}]},{"type":"text","text":"**"}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"CSDN："},{"type":"link","attrs":{"href":"http://blog.csdn.net/qqxx6661","title":""},"content":[{"type":"text","text":"@蠻三刀把刀"}]}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"知乎："},{"type":"link","attrs":{"href":"https://www.zhihu.com/people/yang-zhen-dong-1/","title":""},"content":[{"type":"text","text":"@後端技術漫談"}]}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"簡書："},{"type":"link","attrs":{"href":"https://www.jianshu.com/u/b5f225ca2376","title":""},"content":[{"type":"text","text":"@蠻三刀把刀"}]}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"掘金："},{"type":"link","attrs":{"href":"https://juejin.im/user/5b48015ce51d45191462ba55","title":""},"content":[{"type":"text","text":"@蠻三刀把刀"}]}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"騰訊雲+社區："},{"type":"link","attrs":{"href":"https://cloud.tencent.com/developer/user/1706868","title":""},"content":[{"type":"text","text":"@後端技術漫談"}]}]}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":3},"content":[{"type":"text","text":"原創文章主要內容"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"bulletedlist","content":[{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"後端開發實戰"}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"Java面試知識"}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"設計模式/數據結構/算法題解"}]}]},{"type":"listitem","content":[{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","text":"讀書筆記/逸聞趣事/程序人生"}]}]}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"heading","attrs":{"align":null,"level":3},"content":[{"type":"text","text":"個人公衆號：後端技術漫談"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"image","attrs":{"src":"https://static001.geekbang.org/infoq/04/042e7e6040753ab6d56a46960a2b0767.gif","alt":"個人公衆號：後端技術漫談","title":null,"style":null,"href":null,"fromPaste":true,"pastePass":true}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null},"content":[{"type":"text","marks":[{"type":"strong"}],"text":"如果文章對你有幫助，不妨點贊，收藏起來~"}]},{"type":"paragraph","attrs":{"indent":0,"number":0,"align":null,"origin":null}}]}