日志配置

utils.log_wraper.py

import logging
import absl.logging

logger_initialized = {}
global_log_file = []


# issue: https://github.com/tensorflow/tensorflow/issues/26691
# refer to: https://github.com/dhalperi/pybatfish/blob/f8ddd3938148f9a5d9c14c371a099802c564fac3/pybatfish/client/capirca.py#L33-L50
# https://github.com/abseil/abseil-py/issues/99
logging.root.removeHandler(absl.logging._absl_handler)
# https://github.com/abseil/abseil-py/issues/102
absl.logging._warn_preinit_stderr = False


def get_logger(name, log_file=None, log_level=logging.INFO):
    """Initialize and get a logger by name.

    If the logger has not been initialized, this method will initialize the
    logger by adding one or two handlers, otherwise the initialized logger will
    be directly returned. During initialization, a StreamHandler will always be
    added. If `log_file` is specified and the process rank is 0, a FileHandler
    will also be added.

    Args:
        name (str): Logger name.
        log_file (str | None): The log filename. If specified, a FileHandler
            will be added to the logger.
        log_level (int): The logger level. Note that only the process of
            rank 0 is affected, and other processes will set the level to
            "Error" thus be silent most of the time.

    Returns:
        logging.Logger: The expected logger.
    """
    logger = logging.getLogger(name)
    if name in logger_initialized:
        return logger
    # handle hierarchical names
    # e.g., logger "a" is initialized, then logger "a.b" will skip the
    # initialization since it is a child of "a".
    for logger_name in logger_initialized:
        if name.startswith(logger_name):
            return logger

    stream_handler = logging.StreamHandler()
    handlers = [stream_handler]
    rank = 0

    if log_file is not None and len(global_log_file) == 0:
        global_log_file.append(log_file)
    if log_file is None and len(global_log_file) > 0:
        log_file = global_log_file[0]

    # only rank 0 will add a FileHandler
    if rank == 0 and log_file is not None:
        file_handler = logging.FileHandler(log_file, 'a')
        handlers.append(file_handler)

    formatter = logging.Formatter(
        '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
    for handler in handlers:
        handler.setFormatter(formatter)
        handler.setLevel(log_level)
        logger.addHandler(handler)

    if rank == 0:
        logger.setLevel(log_level)
    else:
        logger.setLevel(logging.ERROR)

    logger_initialized[name] = True

    return logger


def PCHECK(expr, msg, *args):
    if not expr:
        get_logger('main').error(msg, *args)

loger.py

import logging
import os
from .utils import get_logger


LOGGERS = [
    {
        "logfile_name": "gongxu_task.log",
        "logger_list": ['test_task_demo', 'bra_mon_im_ex', 'bra_product_sch', 'bra_weekly_export', 'bra_weekly_ship',
                        'corndeep_data', 'global_quarter_predict_orig', 'licht_balance_orig', 'thai_product_sch',
                        'thaisugar_mon_export']
    },
    {
        "logfile_name": "jiage_task.log",
        "logger_list": ['test_jiage_task_demo', 'Raw_Brasil_Spot', 'Raw_India', 'Raw_Licht_Spot', 'Raw_LpKp', 'Raw_Petrobras_Spot']}
]

LOG_DIR = None


def check_folder(folder):
    if not os.path.exists(folder):
        os.makedirs(folder)


def set_logger_dir(log_dir='./logs', log_level=logging.INFO):
    global LOG_DIR
    if LOG_DIR is not None:
        return
    else:
        LOG_DIR = log_dir

    check_folder(log_dir)

    for LOGGER in LOGGERS:
        log_file = os.path.join(log_dir, LOGGER["logfile_name"])
        file_handler = logging.FileHandler(log_file, 'a')
        formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
        # formatter = logging.Formatter('%(asctime)s - %(filename)s[line:%(lineno)d] - %(levelname)s - %(message)s')
        file_handler.setFormatter(formatter)
        file_handler.setLevel(log_level)

        for name in LOGGER["logger_list"]:
            logger = get_logger(name)
            for handler in logger.handlers:
                logger.removeHandler(handler)
            logger.addHandler(file_handler)

应用logger

import time
from app.common.utils import get_logger

logger = get_logger('test_task_demo')
jiage_logger = get_logger('test_jiage_task_demo')


def run():
    print("run test_demo")
    logger.info("run test_demo")
    logger.warning("run test_demo warning")
    time.sleep(1)
    logger.info("run test_demo end")
    logger.info("gongxu_task running...")
    # logger.error("run test_demo err")
    jiage_logger.info("run test_demo")
    jiage_logger.warning("run test_demo warning")
    time.sleep(1)
    jiage_logger.info("run test_demo end")
    jiage_logger.info("jiage_task running...")
    print("run test_demo end")
    print("task running...")