ClickHouse使用之五 ——clickhouse-go內存泄露解決

這個代碼運行2億條記錄，發現內存使用一直增加，內存滿了以後，直接被killed

 

func (p *ClickHouseClient) CountAllTxTypees(startId int, endId int, SpaceStore SpaceInterface) (web3datas []Web3Data) {
	sql := fmt.Sprintf(`
	SELECT game_name, Address,
		   CASE tx_type
			   WHEN 'mint' THEN 'created_nft_count'
			   WHEN 'transfer' THEN 'nft_transactions_count'
			   ELSE 'chain_interaction_count'
		   END AS attr_name,
		   COUNT(*) AS attr_value
	FROM tx_hashes 
	where id >= %d and id < %d 
	GROUP BY game_name, Address, tx_type`, startId, endId)

	rows, err := conn.Query(context.Background(), sql, startId, endId)
	if err != nil {
		panic(err)
	}

	for rows.Next() {
		var (
			GameName, UserAddress, AttrName string
			AttrValue                       *uint64
		)
		if err := rows.Scan(
			&GameName,
			&UserAddress,
			&AttrName,
			&AttrValue,
		); err != nil {
			log.Fatal(err)
		}

		spaceId, existed := SpaceStore.GetID(GameName)
		if !existed {
			fmt.Printf("GameName:%s not existed!!! \n", GameName)
			return
		}

		web3datas = append(web3datas, Web3Data{SpaceId: int32(*spaceId), UserAddress: UserAddress, AttrName: AttrName, AttrValue: int64(*AttrValue)})
	}
	return
}

然後第一感覺應該是內存泄露，看了下是否有沒有關閉的：

發現有2個close，一個是conn的close，一個是rows的close

剛開始我是加的Rows的close，然後發現內存還是增加；

後來我索引把每個連接，再執行操作的時候重新連接clickhouse, 然後斷開連接，這樣果然有效，內存不增加了。

最終代碼：

func (p *ClickHouseClient) CountAllTxTypees(startId int, endId int, SpaceStore SpaceInterface) (web3datas []Web3Data) {
	// 連接每次用完都close，不然程序內存會一直增加
	conn := p.connect()
	defer conn.Close()

	sql := fmt.Sprintf(`
	SELECT game_name, Address,
		   CASE tx_type
			   WHEN 'mint' THEN 'created_nft_count'
			   WHEN 'transfer' THEN 'nft_transactions_count'
			   ELSE 'chain_interaction_count'
		   END AS attr_name,
		   COUNT(*) AS attr_value
	FROM tx_hashes 
	where id >= %d and id < %d 
	GROUP BY game_name, Address, tx_type`, startId, endId)

	rows, err := conn.Query(context.Background(), sql, startId, endId)
	if err != nil {
		panic(err)
	}

	defer rows.Close()
	for rows.Next() {
		var (
			GameName, UserAddress, AttrName string
			AttrValue                       *uint64
		)
		if err := rows.Scan(
			&GameName,
			&UserAddress,
			&AttrName,
			&AttrValue,
		); err != nil {
			log.Fatal(err)
		}

		spaceId, existed := SpaceStore.GetID(GameName)
		if !existed {
			fmt.Printf("GameName:%s not existed!!! \n", GameName)
			return
		}

		web3datas = append(web3datas, Web3Data{SpaceId: int32(*spaceId), UserAddress: UserAddress, AttrName: AttrName, AttrValue: int64(*AttrValue)})
	}
	return
}