Hive学习笔记——metastore listener

除了使用hive hook来记录hive上用户的操作之外，还可以使用hive metastore listener来进行记录，参考：

https://towardsdatascience.com/apache-hive-hooks-and-metastore-listeners-a-tale-of-your-metadata-903b751ee99f

hive metastore的接口有3种，分别是

 

Property	Abstract class
hive.metastore.pre.event.listeners	org.apache.hadoop.hive.metastore.MetaStorePreEventListener
hive.metastore.end.function.listeners	org.apache.hadoop.hive.metastore.MetaStoreEndFunctionListener
hive.metastore.event.listeners	org.apache.hadoop.hive.metastore.MetaStoreEventListener

下面的代码中继承了MetaStoreEventListener，实现的功能是在建表和修改表的时候，日志输出一下table的元数据信息

package com.bigdata.hive;

import com.fasterxml.jackson.databind.ObjectMapper;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.metastore.MetaStoreEventListener;
import org.apache.hadoop.hive.metastore.events.AlterTableEvent;
import org.apache.hadoop.hive.metastore.events.CreateTableEvent;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.IOException;

public class MyListener extends MetaStoreEventListener {

    private static final Logger LOGGER = LoggerFactory.getLogger(MyListener.class);
    private static final ObjectMapper objMapper = new ObjectMapper();

    public MyListener(Configuration config) {
        super(config);
        logWithHeader(" created ");
    }

    @Override
    public void onCreateTable(CreateTableEvent event) {
        logWithHeader(event.getTable());
    }

    @Override
    public void onAlterTable(AlterTableEvent event) {
        logWithHeader(event.getOldTable());
        logWithHeader(event.getNewTable());
    }

    private void logWithHeader(Object obj) {
        LOGGER.info("[CustomListener][Thread: " + Thread.currentThread().getName() + "] | " + objToStr(obj));
    }

    private String objToStr(Object obj) {
        try {
            return objMapper.writeValueAsString(obj);
        } catch (IOException e) {
            LOGGER.error("Error on conversion", e);
        }
        return null;
    }

}

进行打包

mvn clean package

将jar包放到/var/lib/hive目录下，并修改用户组

lintong@master:/var/lib/hive$ ls -al| grep jar
-rw-r--r--   1 hive hive 363002613 12月 26 16:30 bigdata-1.0-SNAPSHOT-jar-with-dependencies.jar

在CDH界面上配置metastore listener，注意是在metastore的配置项中，而不是hiveserver2的

 

 

然后重启hive

之后在HUE上运行建表语句

create table test_table2(id int,name string)

查看hive metastore的日志，可以看到日志中打印了所创建的hive表的元数据信息

lintong@master:/var/log/hive$ tail -f hadoop-cmf-hive-HIVEMETASTORE-master.log.out | grep Listen
2021-12-26 21:43:53,737 INFO  com.bigdata.hive.MyListener: [main]: [CustomListener][Thread: main] | " created "
2021-12-26 21:45:15,224 INFO  com.bigdata.hive.MyListener: [pool-5-thread-14]: [CustomListener][Thread: pool-5-thread-14] | {"tableName":"CM_TEST_TABLE","dbName":"cloudera_manager_metastore_canary_test_db_hive_hivemetastore_0a5c4e82140edb166b3b3c29b6817024","owner":null,"createTime":1640526315,"lastAccessTime":0,"retention":0,"sd":{"cols":[{"name":"s","type":"string","comment":"test string","setType":true,"setComment":true,"setName":true},{"name":"f","type":"float","comment":"test float","setType":true,"setComment":true,"setName":true},{"name":"a","type":"array<map<string,struct<p1:int,p2:int>>>","comment":"test complex type","setType":true,"setComment":true,"setName":true}],"location":"hdfs://master:8020/user/hue/.cloudera_manager_hive_metastore_canary/hive_HIVEMETASTORE_0a5c4e82140edb166b3b3c29b6817024/cm_test_table","inputFormat":null,"outputFormat":null,"compressed":false,"numBuckets":1,"serdeInfo":{"name":"CM_TEST_TABLE","serializationLib":null,"parameters":{"serialization.format":"1"},"setSerializationLib":false,"setParameters":true,"parametersSize":1,"setName":true},"bucketCols":[],"sortCols":[],"parameters":{},"skewedInfo":null,"storedAsSubDirectories":false,"setParameters":true,"setSkewedInfo":false,"colsSize":3,"colsIterator":[{"name":"s","type":"string","comment":"test string","setType":true,"setComment":true,"setName":true},{"name":"f","type":"float","comment":"test float","setType":true,"setComment":true,"setName":true},{"name":"a","type":"array<map<string,struct<p1:int,p2:int>>>","comment":"test complex type","setType":true,"setComment":true,"setName":true}],"setCols":true,"setLocation":true,"setInputFormat":false,"setOutputFormat":false,"setCompressed":true,"setNumBuckets":true,"setSerdeInfo":true,"bucketColsSize":0,"bucketColsIterator":[],"setBucketCols":true,"sortColsSize":0,"sortColsIterator":[],"setSortCols":true,"setStoredAsSubDirectories":false,"parametersSize":0},"partitionKeys":[{"name":"p1","type":"string","comment":"partition-key-1","setType":true,"setComment":true,"setName":true},{"name":"p2","type":"int","comment":"partition-key-2","setType":true,"setComment":true,"setName":true}],"parameters":{"transient_lastDdlTime":"1640526315"},"viewOriginalText":null,"viewExpandedText":null,"tableType":null,"privileges":null,"temporary":false,"ownerType":"USER","partitionKeysSize":2,"setParameters":true,"setTableType":false,"setTableName":true,"setOwner":false,"setRetention":true,"partitionKeysIterator":[{"name":"p1","type":"string","comment":"partition-key-1","setType":true,"setComment":true,"setName":true},{"name":"p2","type":"int","comment":"partition-key-2","setType":true,"setComment":true,"setName":true}],"setPartitionKeys":true,"setViewOriginalText":false,"setViewExpandedText":false,"setPrivileges":false,"setTemporary":false,"setOwnerType":true,"setDbName":true,"setCreateTime":true,"setLastAccessTime":true,"setSd":true,"parametersSize":1}