HiveConnection.py

python\小脚本\HiveConnection.py

# -*- coding:utf-8 -*- 
# Author : sll
# @Time  : 2018/11/1 下午4:48

from pyhive import hive
from pandas import DataFrame

HIVE_CONFIG = {
    "host": "10.3.4.25",
    "port": 7001,
    "username": 'username',
    "password": 'password',
    'auth': "CUSTOM",
    'database': 'bi_app_database'
}


class HiveConn(object):

    def __init__(self):
        self.connection = hive.connect(**HIVE_CONFIG)
        self.cur = self.connection.cursor()

    def hive_execute(self, hql_query, is_DF=False, is_save_file=False, filename=""):
        self.cur.execute("set hive.execution.engine=tez")
        self.cur.execute("set tez.queue.name=develop")
        self.cur.execute(hql_query)
        result = self.cur.fetchall()
        if is_DF:
            return DataFrame(list(result))
        elif is_save_file and filename:
            df = DataFrame(list(result))
            df.to_csv(filename, index=False, header=None, encoding='utf-8', sep=',')
            if df.empty:
                raise Exception("Hive_execute: Return dataframe is emtpy")
            else:
                return df.count()
        else:
            return result

    def hive_close(self):
        try:
            self.cur.close()
            self.connection.close()
        except Exception:
            pass
        return