Logging Server Update

from os import path, makedirs
from socket import gethostname, gethostbyname
from logging import config, Filter, getLogger, INFO, DEBUG, WARNING, ERROR, CRITICAL, FATAL, NOTSET
from multiprocessing import Process

from zmq import Context
from zmq.backend.cython.constants import SUB, SUBSCRIBE, PUSH, PULL

PATH = path.dirname(path.realpath(__file__))
PATH_LOGS = path.join(PATH, '../logs')
LOGGER_FORMAT = '%(levelname)s:%(asctime)s,%(msecs)d:%(process)d:%(name)s:%(filename)s:%(funcName)s:' \
                '%(lineno)d:%(message)s'
LOG_FILE_BACKUP_COUNT = 3
LOG_FILE_MAXIMUM_BYTES = 5242880  # 5MB

_nameToLevel = {
    'CRITICAL': CRITICAL,
    'FATAL': FATAL,
    'ERROR': ERROR,
    'WARN': WARNING,
    'WARNING': WARNING,
    'INFO': INFO,
    'DEBUG': DEBUG,
    'NOTSET': NOTSET,
}


def spawn(task, *args):
    process = Process(target=task, args=args)
    process.daemon = True
    process.start()
    return process


class Logger:
    def __init__(self, name, frontend, backend):
        super().__init__()

        logger = 'local'
        self.ctx = None
        self.name = name
        self.backend = backend
        self.frontend = frontend
        self.processes = []
        self.logger = getLogger(logger)
        self.hostname = gethostname()
        self.ipaddr = gethostbyname(self.hostname)

        _path = path.join(PATH_LOGS, logger)
        if not path.exists(_path):
            makedirs(_path)

        self.logger.setLevel(DEBUG)
        config.dictConfig(log_config)

    def start(self, nbr_workers):
        ctx = Context()

        # Clients connect to the frontend
        front_socket = ctx.socket(SUB)
        front_socket.setsockopt_string(SUBSCRIBE, '')
        front_socket.bind(self.frontend)

        # Workers connect to the backend
        back_socket = ctx.socket(PUSH)
        back_socket.bind(self.backend)

        for idx in range(nbr_workers):
            process = spawn(self.worker, idx)
            self.processes.append(process)

        while True:
            request = front_socket.recv_multipart()
            back_socket.send_multipart(request)

    def worker(self, index):
        socket = Context().socket(PULL)
        socket.connect(self.backend)

        remote_logger = getLogger('remote')  # Handles logs for remote clients
        remote_logger.setLevel(DEBUG)

        message = f'{self.hostname}:{self.ipaddr}:{self.name}-{index}:started'
        self.logger.info(message)

        while True:
            errorlevel, message = socket.recv_multipart()
            message = message.decode('utf-8')
            errorlevel = errorlevel.decode('utf-8')
            remote_logger.log(_nameToLevel[errorlevel], message)


class Equals(Filter):
    def __init__(self, param):
        super().__init__()
        self.errorlevel = param if param else NOTSET

    def filter(self, record):
        return True if record.levelno == self.errorlevel else False


class LessThan(Filter):
    def __init__(self, param):
        super().__init__()
        self.errorlevel = param if param else WARNING

    def filter(self, record):
        return True if record.levelno < self.errorlevel else False


class GreaterThan(Filter):
    def __init__(self, param):
        super().__init__()
        self.errorlevel = param if param else INFO

    def filter(self, record):
        return True if record.levelno > self.errorlevel else False


log_config = {
    'version': 1,
    'formatters': {
        'remote': {'format': ''},
        'default': {'format': LOGGER_FORMAT, 'datefmt': '%Y/%m/%d %H:%M:%S'}
    },
    'loggers': {
        'local': {
            'handlers': [
                'stdout',
                'stderr',
                'debug',
                'info',
                'warning',
                'error',
                'critical'
            ]
        },
        'remote': {
            'handlers': [
                'stdout2',
                'stderr2',
                'debug2',
                'info2',
                'warning2',
                'error2',
                'critical2'
            ]
        },
    },
    'filters': {
        'stdout': {
            '()': LessThan,
            'param': WARNING
        },
        'stderr': {
            '()': GreaterThan,
            'param': INFO
        },
        'info': {
            '()': Equals,
            'param': INFO
        },
        'warning': {
            '()': Equals,
            'param': WARNING
        },
        'error': {
            '()': Equals,
            'param': ERROR
        },
        'critical': {
            '()': Equals,
            'param': CRITICAL
        },
    },
    'handlers': {
        'stdout': {
            'class': 'logging.StreamHandler',
            'level': DEBUG,
            'stream': 'ext://sys.stdout',
            'formatter': 'default',
            'filters': ['stdout']
        },
        'stdout2': {
            'class': 'logging.StreamHandler',
            'level': DEBUG,
            'stream': 'ext://sys.stdout',
            'formatter': 'remote',
            'filters': ['stdout']
        },
        'stderr': {
            'class': 'logging.StreamHandler',
            'level': WARNING,
            'stream': 'ext://sys.stderr',
            'formatter': 'default',
            'filters': ['stderr']
        },
        'stderr2': {
            'class': 'logging.StreamHandler',
            'level': WARNING,
            'stream': 'ext://sys.stderr',
            'formatter': 'remote',
            'filters': ['stderr']
        },
        'debug': {
            'class': 'logging.handlers.RotatingFileHandler',
            'filename': path.join(PATH_LOGS, 'local', 'debug.log'),
            'level': DEBUG,
            'maxBytes': LOG_FILE_MAXIMUM_BYTES,
            'backupCount': LOG_FILE_BACKUP_COUNT,
            'formatter': 'default',
        },
        'debug2': {
            'class': 'logging.handlers.RotatingFileHandler',
            'filename': path.join(PATH_LOGS, 'debug.log'),
            'level': DEBUG,
            'maxBytes': LOG_FILE_MAXIMUM_BYTES,
            'backupCount': LOG_FILE_BACKUP_COUNT,
            'formatter': 'remote',
        },
        'info': {
            'class': 'logging.handlers.RotatingFileHandler',
            'filename': path.join(PATH_LOGS, 'local', 'info.log'),
            'level': INFO,
            'maxBytes': LOG_FILE_MAXIMUM_BYTES,
            'backupCount': LOG_FILE_BACKUP_COUNT,
            'formatter': 'default',
            'filters': ['info'],
        },
        'info2': {
            'class': 'logging.handlers.RotatingFileHandler',
            'filename': path.join(PATH_LOGS, 'info.log'),
            'level': INFO,
            'maxBytes': LOG_FILE_MAXIMUM_BYTES,
            'backupCount': LOG_FILE_BACKUP_COUNT,
            'formatter': 'remote',
            'filters': ['info'],
        },
        'warning': {
            'class': 'logging.handlers.RotatingFileHandler',
            'filename': path.join(PATH_LOGS, 'local', 'warn.log'),
            'level': WARNING,
            'maxBytes': LOG_FILE_MAXIMUM_BYTES,
            'backupCount': LOG_FILE_BACKUP_COUNT,
            'formatter': 'default',
            'filters': ['warning'],
        },
        'warning2': {
            'class': 'logging.handlers.RotatingFileHandler',
            'filename': path.join(PATH_LOGS, 'warn.log'),
            'level': WARNING,
            'maxBytes': LOG_FILE_MAXIMUM_BYTES,
            'backupCount': LOG_FILE_BACKUP_COUNT,
            'formatter': 'remote',
            'filters': ['warning'],
        },
        'error': {
            'class': 'logging.handlers.RotatingFileHandler',
            'filename': path.join(PATH_LOGS, 'local', 'error.log'),
            'level': ERROR,
            'maxBytes': LOG_FILE_MAXIMUM_BYTES,
            'backupCount': LOG_FILE_BACKUP_COUNT,
            'formatter': 'default',
            'filters': ['error'],
        },
        'error2': {
            'class': 'logging.handlers.RotatingFileHandler',
            'filename': path.join(PATH_LOGS, 'error.log'),
            'level': ERROR,
            'maxBytes': LOG_FILE_MAXIMUM_BYTES,
            'backupCount': LOG_FILE_BACKUP_COUNT,
            'formatter': 'remote',
            'filters': ['error'],
        },
        'critical': {
            'class': 'logging.handlers.RotatingFileHandler',
            'filename': path.join(PATH_LOGS, 'local', 'critical.log'),
            'level': CRITICAL,
            'maxBytes': LOG_FILE_MAXIMUM_BYTES,
            'backupCount': LOG_FILE_BACKUP_COUNT,
            'formatter': 'default',
            'filters': ['critical'],
        },
        'critical2': {
            'class': 'logging.handlers.RotatingFileHandler',
            'filename': path.join(PATH_LOGS, 'critical.log'),
            'level': CRITICAL,
            'maxBytes': LOG_FILE_MAXIMUM_BYTES,
            'backupCount': LOG_FILE_BACKUP_COUNT,
            'formatter': 'remote',
            'filters': ['critical'],
        },
    },
}