Python ES日志库CMRESHandler修复[[types removal] Specifying types in bulk requests is deprecated.]警告

背景

Elasticsearch版本前两天升级到了7.x，每次打印日志都提示了[[types removal] Specifying types in bulk requests is deprecated.]警告，网上查了一通，发现是7.x版本后，类型已经弃用，在CMRESHandler的Issues中看到了遇到同样问题的朋友，并向作者提交了一个PR，目前还是未合并状态，所以自己先本地进行重写，后续等作者合并代码并发布最新版本后，再使用原生的

Issues链接

https://github.com/cmanaha/python-elasticsearch-logger/issues/76

PR链接

https://github.com/cmanaha/python-elasticsearch-logger/pull/79

本地重写方案

from elasticsearch import helpers as eshelpers
from elasticsearch import Elasticsearch, RequestsHttpConnection
try:
    from requests_kerberos import HTTPKerberosAuth, DISABLED
    CMR_KERBEROS_SUPPORTED = True
except ImportError:
    CMR_KERBEROS_SUPPORTED = False

try:
    from requests_aws4auth import AWS4Auth
    AWS4AUTH_SUPPORTED = True
except ImportError:
    AWS4AUTH_SUPPORTED = False

class PrivateCMRESHandler(CMRESHandler):
    '''
    重写CMRESHandler下的__get_es_client方法和flush方法，不指定es_doc_type，修复[[types removal] Specifying types in bulk requests is deprecated.]警告
    '''
    def __get_es_client(self):
        if self.auth_type == PrivateCMRESHandler.AuthType.NO_AUTH:
            if self._client is None:
                self._client = Elasticsearch(hosts=self.hosts,
                                             use_ssl=self.use_ssl,
                                             verify_certs=self.verify_certs,
                                             connection_class=RequestsHttpConnection,
                                             serializer=self.serializer)
            return self._client

        if self.auth_type == PrivateCMRESHandler.AuthType.BASIC_AUTH:
            if self._client is None:
                return Elasticsearch(hosts=self.hosts,
                                     http_auth=self.auth_details,
                                     use_ssl=self.use_ssl,
                                     verify_certs=self.verify_certs,
                                     connection_class=RequestsHttpConnection,
                                     serializer=self.serializer)
            return self._client

        if self.auth_type == PrivateCMRESHandler.AuthType.KERBEROS_AUTH:
            if not CMR_KERBEROS_SUPPORTED:
                raise EnvironmentError("Kerberos module not available. Please install \"requests-kerberos\"")
            # For kerberos we return a new client each time to make sure the tokens are up to date
            return Elasticsearch(hosts=self.hosts,
                                 use_ssl=self.use_ssl,
                                 verify_certs=self.verify_certs,
                                 connection_class=RequestsHttpConnection,
                                 http_auth=HTTPKerberosAuth(mutual_authentication=DISABLED),
                                 serializer=self.serializer)

        if self.auth_type == PrivateCMRESHandler.AuthType.AWS_SIGNED_AUTH:
            if not AWS4AUTH_SUPPORTED:
                raise EnvironmentError("AWS4Auth not available. Please install \"requests-aws4auth\"")
            if self._client is None:
                awsauth = AWS4Auth(self.aws_access_key, self.aws_secret_key, self.aws_region, 'es')
                self._client = Elasticsearch(
                    hosts=self.hosts,
                    http_auth=awsauth,
                    use_ssl=self.use_ssl,
                    verify_certs=True,
                    connection_class=RequestsHttpConnection,
                    serializer=self.serializer
                )
            return self._client

        raise ValueError("Authentication method not supported")

    def flush(self):
        """ Flushes the buffer into ES
        :return: None
        """
        if self._timer is not None and self._timer.is_alive():
            self._timer.cancel()
        self._timer = None

        if self._buffer:
            try:
                with self._buffer_lock:
                    logs_buffer = self._buffer
                    self._buffer = []
                actions = (
                    {
                        '_index': self._index_name_func.__func__(self.es_index_name),
                        '_source': log_record
                    }
                    for log_record in logs_buffer
                )
                eshelpers.bulk(
                    client=self.__get_es_client(),
                    actions=actions,
                    stats_only=True
                )
            except Exception as exception:
                if self.raise_on_indexing_exceptions:
                    raise exception

调用

# 添加 CMRESHandler
es_handler = PrivateCMRESHandler(hosts=[{'host': self.ELASTIC_SEARCH_HOST, 'port': self.ELASTIC_SEARCH_PORT}],
						  # 可以配置对应的认证权限
						  auth_type=PrivateCMRESHandler.AuthType.BASIC_AUTH,
						  auth_details=self.AUTH_DETAILS,
						  es_index_name=self.ELASTIC_SEARCH_INDEX,
						  # 一个月分一个 Index
						  index_name_frequency=PrivateCMRESHandler.IndexNameFrequency.MONTHLY,
						  # 额外增加环境标识
						  es_additional_fields={'environment': self.APP_ENVIRONMENT}
						  )
es_handler.setLevel(level=self.es_output_level)
formatter = self.formatter
es_handler.setFormatter(formatter)
self.logger.addHandler(es_handler)

另外还找到一个已修复该问题的第三方库

链接

https://github.com/IMInterne/python-elasticsearch-ecs-logger

安装

pip install ElasticECSHandler