ClickHouse使用之五 ——clickhouse-go内存泄露解决

这个代码运行2亿条记录，发现内存使用一直增加，内存满了以后，直接被killed

 

func (p *ClickHouseClient) CountAllTxTypees(startId int, endId int, SpaceStore SpaceInterface) (web3datas []Web3Data) {
	sql := fmt.Sprintf(`
	SELECT game_name, Address,
		   CASE tx_type
			   WHEN 'mint' THEN 'created_nft_count'
			   WHEN 'transfer' THEN 'nft_transactions_count'
			   ELSE 'chain_interaction_count'
		   END AS attr_name,
		   COUNT(*) AS attr_value
	FROM tx_hashes 
	where id >= %d and id < %d 
	GROUP BY game_name, Address, tx_type`, startId, endId)

	rows, err := conn.Query(context.Background(), sql, startId, endId)
	if err != nil {
		panic(err)
	}

	for rows.Next() {
		var (
			GameName, UserAddress, AttrName string
			AttrValue                       *uint64
		)
		if err := rows.Scan(
			&GameName,
			&UserAddress,
			&AttrName,
			&AttrValue,
		); err != nil {
			log.Fatal(err)
		}

		spaceId, existed := SpaceStore.GetID(GameName)
		if !existed {
			fmt.Printf("GameName:%s not existed!!! \n", GameName)
			return
		}

		web3datas = append(web3datas, Web3Data{SpaceId: int32(*spaceId), UserAddress: UserAddress, AttrName: AttrName, AttrValue: int64(*AttrValue)})
	}
	return
}

然后第一感觉应该是内存泄露，看了下是否有没有关闭的：

发现有2个close，一个是conn的close，一个是rows的close

刚开始我是加的Rows的close，然后发现内存还是增加；

后来我索引把每个连接，再执行操作的时候重新连接clickhouse, 然后断开连接，这样果然有效，内存不增加了。

最终代码：

func (p *ClickHouseClient) CountAllTxTypees(startId int, endId int, SpaceStore SpaceInterface) (web3datas []Web3Data) {
	// 连接每次用完都close，不然程序内存会一直增加
	conn := p.connect()
	defer conn.Close()

	sql := fmt.Sprintf(`
	SELECT game_name, Address,
		   CASE tx_type
			   WHEN 'mint' THEN 'created_nft_count'
			   WHEN 'transfer' THEN 'nft_transactions_count'
			   ELSE 'chain_interaction_count'
		   END AS attr_name,
		   COUNT(*) AS attr_value
	FROM tx_hashes 
	where id >= %d and id < %d 
	GROUP BY game_name, Address, tx_type`, startId, endId)

	rows, err := conn.Query(context.Background(), sql, startId, endId)
	if err != nil {
		panic(err)
	}

	defer rows.Close()
	for rows.Next() {
		var (
			GameName, UserAddress, AttrName string
			AttrValue                       *uint64
		)
		if err := rows.Scan(
			&GameName,
			&UserAddress,
			&AttrName,
			&AttrValue,
		); err != nil {
			log.Fatal(err)
		}

		spaceId, existed := SpaceStore.GetID(GameName)
		if !existed {
			fmt.Printf("GameName:%s not existed!!! \n", GameName)
			return
		}

		web3datas = append(web3datas, Web3Data{SpaceId: int32(*spaceId), UserAddress: UserAddress, AttrName: AttrName, AttrValue: int64(*AttrValue)})
	}
	return
}