ClickHouse使用之五 ——clickhouse-go内存泄露解决
这个代码运行2亿条记录,发现内存使用一直增加,内存满了以后,直接被killed
func (p *ClickHouseClient) CountAllTxTypees(startId int, endId int, SpaceStore SpaceInterface) (web3datas []Web3Data) {
sql := fmt.Sprintf(`
SELECT game_name, Address,
CASE tx_type
WHEN 'mint' THEN 'created_nft_count'
WHEN 'transfer' THEN 'nft_transactions_count'
ELSE 'chain_interaction_count'
END AS attr_name,
COUNT(*) AS attr_value
FROM tx_hashes
where id >= %d and id < %d
GROUP BY game_name, Address, tx_type`, startId, endId)
rows, err := conn.Query(context.Background(), sql, startId, endId)
if err != nil {
panic(err)
}
for rows.Next() {
var (
GameName, UserAddress, AttrName string
AttrValue *uint64
)
if err := rows.Scan(
&GameName,
&UserAddress,
&AttrName,
&AttrValue,
); err != nil {
log.Fatal(err)
}
spaceId, existed := SpaceStore.GetID(GameName)
if !existed {
fmt.Printf("GameName:%s not existed!!! \n", GameName)
return
}
web3datas = append(web3datas, Web3Data{SpaceId: int32(*spaceId), UserAddress: UserAddress, AttrName: AttrName, AttrValue: int64(*AttrValue)})
}
return
}
然后第一感觉应该是内存泄露,看了下是否有没有关闭的:
发现有2个close,一个是conn的close,一个是rows的close
刚开始我是加的Rows的close,然后发现内存还是增加;
后来我索引把每个连接,再执行操作的时候重新连接clickhouse, 然后断开连接,这样果然有效,内存不增加了。
最终代码:
func (p *ClickHouseClient) CountAllTxTypees(startId int, endId int, SpaceStore SpaceInterface) (web3datas []Web3Data) {
// 连接每次用完都close,不然程序内存会一直增加
conn := p.connect()
defer conn.Close()
sql := fmt.Sprintf(`
SELECT game_name, Address,
CASE tx_type
WHEN 'mint' THEN 'created_nft_count'
WHEN 'transfer' THEN 'nft_transactions_count'
ELSE 'chain_interaction_count'
END AS attr_name,
COUNT(*) AS attr_value
FROM tx_hashes
where id >= %d and id < %d
GROUP BY game_name, Address, tx_type`, startId, endId)
rows, err := conn.Query(context.Background(), sql, startId, endId)
if err != nil {
panic(err)
}
defer rows.Close()
for rows.Next() {
var (
GameName, UserAddress, AttrName string
AttrValue *uint64
)
if err := rows.Scan(
&GameName,
&UserAddress,
&AttrName,
&AttrValue,
); err != nil {
log.Fatal(err)
}
spaceId, existed := SpaceStore.GetID(GameName)
if !existed {
fmt.Printf("GameName:%s not existed!!! \n", GameName)
return
}
web3datas = append(web3datas, Web3Data{SpaceId: int32(*spaceId), UserAddress: UserAddress, AttrName: AttrName, AttrValue: int64(*AttrValue)})
}
return
}