revamp logging

hyperglass/api/__init__.py

@@ -14,7 +14,8 @@ from starlette.staticfiles import StaticFiles
 from starlette.middleware.cors import CORSMiddleware
 
 # Project
-from hyperglass.util import log, cpu_count
+from hyperglass.log import log
+from hyperglass.util import cpu_count
 from hyperglass.constants import TRANSPORT_REST, __version__
 from hyperglass.api.events import on_startup, on_shutdown
 from hyperglass.api.routes import docs, query, queries, routers, import_certificate

hyperglass/api/models/__init__.py

@@ -1,6 +1 @@
 """Query & Response Validation Models."""
-
-# Project
-from hyperglass.api.models import query, types, rfc8522, response, validators
-
-# flake8: noqa: F401

hyperglass/api/models/query.py

@@ -7,7 +7,7 @@ import hashlib
 from pydantic import BaseModel, StrictStr, validator
 
 # Project
-from hyperglass.util import log
+from hyperglass.log import log
 from hyperglass.exceptions import InputInvalid
 from hyperglass.configuration import params, devices
 from hyperglass.api.models.types import SupportedQuery

hyperglass/api/models/validators.py

@@ -5,7 +5,8 @@ import re
 from ipaddress import ip_network
 
 # Project
-from hyperglass.util import log, get_containing_prefix
+from hyperglass.log import log
+from hyperglass.util import get_containing_prefix
 from hyperglass.exceptions import InputInvalid, InputNotAllowed
 from hyperglass.configuration import params
 

hyperglass/api/routes.py

@@ -10,7 +10,8 @@ from starlette.requests import Request
 from fastapi.openapi.docs import get_redoc_html, get_swagger_ui_html
 
 # Project
-from hyperglass.util import log, clean_name, import_public_key
+from hyperglass.log import log
+from hyperglass.util import clean_name, import_public_key
 from hyperglass.cache import Cache
 from hyperglass.encode import jwt_decode
 from hyperglass.exceptions import HyperglassError

hyperglass/cli/echo.py

@@ -19,7 +19,7 @@ def cmd_help(emoji="", help_text="", supports_color=False):
     return help_str
 
 
-def _base_formatter(state, text, callback, **kwargs):
+def _base_formatter(_text, _state, _callback, *args, **kwargs):
     """Format text block, replace template strings with keyword arguments.
 
     Arguments:
@@ -31,29 +31,36 @@ def _base_formatter(state, text, callback, **kwargs):
     Returns:
         {str|ClickException} -- Formatted output
     """
-    fmt = Message(state)
+    fmt = Message(_state)
 
-    if callback is None:
-        callback = style
+    if _callback is None:
+        _callback = style
+
+    nargs = ()
+    for i in args:
+        if not isinstance(i, str):
+            nargs += (str(i),)
+        else:
+            nargs += (i,)
 
     for k, v in kwargs.items():
         if not isinstance(v, str):
             v = str(v)
         kwargs[k] = style(v, **fmt.kw)
 
-    text_all = re.split(r"(\{\w+\})", text)
+    text_all = re.split(r"(\{\w+\})", _text)
     text_all = [style(i, **fmt.msg) for i in text_all]
-    text_all = [i.format(**kwargs) for i in text_all]
+    text_all = [i.format(*nargs, **kwargs) for i in text_all]
 
     if fmt.emoji:
         text_all.insert(0, fmt.emoji)
 
     text_fmt = "".join(text_all)
 
-    return callback(text_fmt)
+    return _callback(text_fmt)
 
 
-def info(text, callback=echo, **kwargs):
+def info(text, *args, **kwargs):
     """Generate formatted informational text.
 
     Arguments:
@@ -63,10 +70,10 @@ def info(text, callback=echo, **kwargs):
     Returns:
         {str} -- Informational output
     """
-    return _base_formatter(state="info", text=text, callback=callback, **kwargs)
+    return _base_formatter(_state="info", _text=text, _callback=echo, *args, **kwargs)
 
 
-def error(text, callback=CliError, **kwargs):
+def error(text, *args, **kwargs):
     """Generate formatted exception.
 
     Arguments:
@@ -76,10 +83,10 @@ def error(text, callback=CliError, **kwargs):
     Raises:
         ClickException: Raised after formatting
     """
-    raise _base_formatter(state="error", text=text, callback=callback, **kwargs)
+    raise _base_formatter(text, "error", CliError, *args, **kwargs)
 
 
-def success(text, callback=echo, **kwargs):
+def success(text, *args, **kwargs):
     """Generate formatted success text.
 
     Arguments:
@@ -89,10 +96,12 @@ def success(text, callback=echo, **kwargs):
     Returns:
         {str} -- Success output
     """
-    return _base_formatter(state="success", text=text, callback=callback, **kwargs)
+    return _base_formatter(
+        _state="success", _text=text, _callback=echo, *args, **kwargs
+    )
 
 
-def warning(text, callback=echo, **kwargs):
+def warning(text, *args, **kwargs):
     """Generate formatted warning text.
 
     Arguments:
@@ -102,10 +111,12 @@ def warning(text, callback=echo, **kwargs):
     Returns:
         {str} -- Warning output
     """
-    return _base_formatter(state="warning", text=text, callback=callback, **kwargs)
+    return _base_formatter(
+        _state="warning", _text=text, _callback=echo, *args, **kwargs
+    )
 
 
-def label(text, callback=echo, **kwargs):
+def label(text, *args, **kwargs):
     """Generate formatted info text with accented labels.
 
     Arguments:
@@ -115,10 +126,10 @@ def label(text, callback=echo, **kwargs):
     Returns:
         {str} -- Label output
     """
-    return _base_formatter(state="label", text=text, callback=callback, **kwargs)
+    return _base_formatter(_state="label", _text=text, _callback=echo, *args, **kwargs)
 
 
-def status(text, callback=echo, **kwargs):
+def status(text, *args, **kwargs):
     """Generate formatted status text.
 
     Arguments:
@@ -128,4 +139,4 @@ def status(text, callback=echo, **kwargs):
     Returns:
         {str} -- Status output
     """
-    return _base_formatter(state="status", text=text, callback=callback, **kwargs)
+    return _base_formatter(_state="status", _text=text, _callback=echo, *args, **kwargs)

hyperglass/compat/_sshtunnel.py

@@ -47,8 +47,7 @@ from binascii import hexlify
 import paramiko
 
 # Project
-from hyperglass.util import log
-from hyperglass.constants import LOG_FMT
+from hyperglass.log import log
 from hyperglass.configuration import params
 
 if params.debug:
@@ -83,12 +82,6 @@ SSH_CONFIG_FILE = os.path.join(DEFAULT_SSH_DIRECTORY, "config")
 ########################
 
 
-class DefaultHandlers:
-    sink = sys.stdout
-    format = LOG_FMT
-    level = "INFO"
-
-
 def check_host(host):
     assert isinstance(host, str), "IP is not a string ({0})".format(type(host).__name__)
 

hyperglass/configuration/__init__.py

@@ -6,7 +6,6 @@ import copy
 import json
 import math
 from pathlib import Path
-from datetime import datetime
 
 # Third Party
 import yaml
@@ -14,10 +13,15 @@ from aiofile import AIOFile
 from pydantic import ValidationError
 
 # Project
-from hyperglass.util import log, check_path, set_app_path
+from hyperglass.log import (
+    log,
+    set_log_level,
+    enable_file_logging,
+    enable_syslog_logging,
+)
+from hyperglass.util import check_path, set_app_path
 from hyperglass.constants import (
     CREDIT,
-    LOG_HANDLER,
     DEFAULT_HELP,
     DEFAULT_TERMS,
     DEFAULT_DETAILS,
@@ -82,66 +86,6 @@ STATIC_PATH = CONFIG_PATH / "static"
 CONFIG_MAIN, CONFIG_DEVICES, CONFIG_COMMANDS = _check_config_files(CONFIG_PATH)
 
 
-def _set_log_level(debug):
-    """Set log level based on debug state.
-
-    Arguments:
-        debug {bool} -- Debug state from config file
-
-    Returns:
-        {bool} -- True
-    """
-    stdout_handler = LOG_HANDLER.copy()
-
-    if debug:
-        log_level = "DEBUG"
-        stdout_handler["level"] = log_level
-        os.environ["HYPERGLASS_LOG_LEVEL"] = log_level
-        log.configure(handlers=[stdout_handler])
-
-    if debug:
-        log.debug("Debugging enabled")
-    return True
-
-
-def _set_file_logging(log_directory, log_format, log_max_size):
-    """Set up file-based logging from configuration parameters."""
-
-    if log_format == "json":
-        log_file_name = "hyperglass_log.json"
-        structured = True
-    else:
-        log_file_name = "hyperglass_log.log"
-        structured = False
-
-    log_file = log_directory / log_file_name
-
-    if log_format == "text":
-        now_str = "hyperglass logs for " + datetime.utcnow().strftime(
-            "%B %d, %Y beginning at %H:%M:%S UTC"
-        )
-        now_str_y = len(now_str) + 6
-        now_str_x = len(now_str) + 4
-        log_break = (
-            "#" * now_str_y,
-            "\n#" + " " * now_str_x + "#\n",
-            "#  ",
-            now_str,
-            "  #",
-            "\n#" + " " * now_str_x + "#\n",
-            "#" * now_str_y,
-        )
-
-        with log_file.open("a+") as lf:
-            lf.write(f'\n\n{"".join(log_break)}\n\n')
-
-    log.add(log_file, rotation=log_max_size, serialize=structured)
-
-    log.debug("Logging to file enabled")
-
-    return True
-
-
 def _config_required(config_path: Path) -> dict:
     try:
         with config_path.open("r") as cf:
@@ -224,11 +168,8 @@ async def _config_devices():
 
 user_config = _config_optional(CONFIG_MAIN)
 
-# Logging Config
-_debug = user_config.get("debug", True)
-
 # Read raw debug value from config to enable debugging quickly.
-_set_log_level(_debug)
+set_log_level(logger=log, debug=user_config.get("debug", True))
 
 _user_commands = _config_optional(CONFIG_COMMANDS)
 _user_devices = _config_required(CONFIG_DEVICES)
@@ -247,13 +188,25 @@ except ValidationError as validation_errors:
             error_msg=error["msg"],
         )
 
+# Re-evaluate debug state after config is validated
+set_log_level(logger=log, debug=params.debug)
+
 # Set up file logging once configuration parameters are initialized.
-_set_file_logging(
-    log_directory=params.log_directory,
-    log_format=params.log_format,
-    log_max_size=params.log_max_size,
+enable_file_logging(
+    logger=log,
+    log_directory=params.logging.directory,
+    log_format=params.logging.format,
+    log_max_size=params.logging.max_size,
 )
 
+# Set up syslog logging if enabled.
+if params.logging.syslog is not None and params.logging.syslog.enable:
+    enable_syslog_logging(
+        logger=log,
+        syslog_host=params.logging.syslog.host,
+        syslog_port=params.logging.syslog.port,
+    )
+
 # Perform post-config initialization string formatting or other
 # functions that require access to other config levels. E.g.,
 # something in 'params.web.text' needs to be formatted with a value
@@ -288,10 +241,6 @@ except KeyError:
     pass
 
 
-# Re-evaluate debug state after config is validated
-_set_log_level(params.debug)
-
-
 def _build_frontend_networks():
     """Build filtered JSON structure of networks for frontend.
 

hyperglass/configuration/markdown.py

@@ -1,7 +1,7 @@
 """Markdown processing utility functions."""
 
 # Project
-from hyperglass.util import log
+from hyperglass.log import log
 
 
 def _get_file(path_obj):

hyperglass/configuration/models/logging.py

@@ -0,0 +1,28 @@
+"""Validate logging configuration."""
+
+# Standard Library
+from typing import Optional
+from pathlib import Path
+
+# Third Party
+from pydantic import ByteSize, StrictInt, StrictStr, StrictBool, DirectoryPath, constr
+
+# Project
+from hyperglass.configuration.models._utils import HyperglassModel
+
+
+class Syslog(HyperglassModel):
+    """Validation model for syslog configuration."""
+
+    enable: StrictBool = True
+    host: StrictStr
+    port: StrictInt = 514
+
+
+class Logging(HyperglassModel):
+    """Validation model for logging configuration."""
+
+    directory: DirectoryPath = Path("/tmp")  # noqa: S108
+    format: constr(regex=r"(text|json)") = "text"
+    syslog: Optional[Syslog]
+    max_size: ByteSize = "50MB"

hyperglass/configuration/models/messages.py

@@ -1,11 +1,9 @@
 """Validate error message configuration variables."""
 
 # Third Party
-# Third Party Imports
 from pydantic import Field, StrictStr
 
 # Project
-# Project Imports
 from hyperglass.configuration.models._utils import HyperglassModel
 
 

hyperglass/configuration/models/params.py

@@ -2,17 +2,14 @@
 
 # Standard Library
 from typing import List, Union, Optional
-from pathlib import Path
 from ipaddress import ip_address
 
 # Third Party
 from pydantic import (
     Field,
-    ByteSize,
     StrictInt,
     StrictStr,
     StrictBool,
-    DirectoryPath,
     IPvAnyAddress,
     constr,
     validator,
@@ -23,6 +20,7 @@ from hyperglass.configuration.models.web import Web
 from hyperglass.configuration.models.docs import Docs
 from hyperglass.configuration.models.cache import Cache
 from hyperglass.configuration.models._utils import IntFloat, HyperglassModel
+from hyperglass.configuration.models.logging import Logging
 from hyperglass.configuration.models.queries import Queries
 from hyperglass.configuration.models.messages import Messages
 
@@ -98,19 +96,6 @@ class Params(HyperglassModel):
         title="Listen Port",
         description="Local TCP port the hyperglass application listens on to serve web traffic.",
     )
-    log_directory: DirectoryPath = Field(
-        Path("/tmp"),  # noqa: S108
-        title="Log Directory",
-        description="Path to a directory, to which hyperglass can write logs. If none is set, hyperglass will write logs to a file located at `/tmp/`, with a uniquely generated name for each time hyperglass is started.",
-    )
-    log_format: constr(regex=r"(text|json)") = Field(
-        "text", title="Log Format", description="Format for logs written to a file."
-    )
-    log_max_size: ByteSize = Field(
-        "50MB",
-        title="Maximum Log File Size",
-        description="Maximum storage space log file may consume.",
-    )
     cors_origins: List[StrictStr] = Field(
         [],
         title="Cross-Origin Resource Sharing",
@@ -125,6 +110,7 @@ class Params(HyperglassModel):
     # Sub Level Params
     cache: Cache = Cache()
     docs: Docs = Docs()
+    logging: Logging = Logging()
     messages: Messages = Messages()
     queries: Queries = Queries()
     web: Web = Web()

hyperglass/configuration/models/routers.py

@@ -10,7 +10,8 @@ from pathlib import Path
 from pydantic import StrictInt, StrictStr, validator
 
 # Project
-from hyperglass.util import log, clean_name
+from hyperglass.log import log
+from hyperglass.util import clean_name
 from hyperglass.constants import SCRAPE_HELPERS, TRANSPORT_REST, TRANSPORT_SCRAPE
 from hyperglass.exceptions import ConfigError, UnsupportedDevice
 from hyperglass.configuration.models.ssl import Ssl

hyperglass/constants.py

@@ -1,6 +1,6 @@
 """Constant definitions used throughout the application."""
+
 # Standard Library
-import sys
 from datetime import datetime
 
 __name__ = "hyperglass"
@@ -19,23 +19,6 @@ TARGET_FORMAT_SPACE = ("huawei", "huawei_vrpv8")
 
 TARGET_JUNIPER_ASPATH = ("juniper", "juniper_junos")
 
-LOG_FMT = (
-    "<lvl><b>[{level}]</b> {time:YYYYMMDD} {time:HH:mm:ss} <lw>|</lw> {name}<lw>:</lw>"
-    "<b>{line}</b> <lw>|</lw> {function}</lvl> <lvl><b>→</b></lvl> {message}"
-)
-LOG_LEVELS = [
-    {"name": "DEBUG", "no": 10, "color": "<c>"},
-    {"name": "INFO", "no": 20, "color": "<le>"},
-    {"name": "SUCCESS", "no": 25, "color": "<g>"},
-    {"name": "WARNING", "no": 30, "color": "<y>"},
-    {"name": "ERROR", "no": 40, "color": "<y>"},
-    {"name": "CRITICAL", "no": 50, "color": "<r>"},
-]
-
-LOG_HANDLER = {"sink": sys.stdout, "format": LOG_FMT, "level": "INFO"}
-
-LOG_HANDLER_FILE = {"format": LOG_FMT, "level": "INFO"}
-
 STATUS_CODE_MAP = {"warning": 400, "error": 400, "danger": 500}
 
 DNS_OVER_HTTPS = {

hyperglass/exceptions.py

@@ -4,7 +4,7 @@
 import json as _json
 
 # Project
-from hyperglass.util import log
+from hyperglass.log import log
 from hyperglass.constants import STATUS_CODE_MAP
 
 

hyperglass/execution/construct.py

@@ -11,7 +11,7 @@ import json as _json
 import operator
 
 # Project
-from hyperglass.util import log
+from hyperglass.log import log
 from hyperglass.constants import (
     TRANSPORT_REST,
     TARGET_FORMAT_SPACE,

hyperglass/execution/execute.py

@@ -22,7 +22,8 @@ from netmiko import (
 )
 
 # Project
-from hyperglass.util import log, parse_exception
+from hyperglass.log import log
+from hyperglass.util import parse_exception
 from hyperglass.compat import _sshtunnel as sshtunnel
 from hyperglass.encode import jwt_decode, jwt_encode
 from hyperglass.constants import Supported

hyperglass/log.py

@@ -0,0 +1,100 @@
+"""Logging instance setup & configuration."""
+
+# Standard Library
+import os
+import sys
+from datetime import datetime
+
+# Third Party
+from loguru import logger as _loguru_logger
+
+_LOG_FMT = (
+    "<lvl><b>[{level}]</b> {time:YYYYMMDD} {time:HH:mm:ss} <lw>|</lw> {name}<lw>:</lw>"
+    "<b>{line}</b> <lw>|</lw> {function}</lvl> <lvl><b>→</b></lvl> {message}"
+)
+_LOG_LEVELS = [
+    {"name": "TRACE", "no": 5, "color": "<m>"},
+    {"name": "DEBUG", "no": 10, "color": "<c>"},
+    {"name": "INFO", "no": 20, "color": "<le>"},
+    {"name": "SUCCESS", "no": 25, "color": "<g>"},
+    {"name": "WARNING", "no": 30, "color": "<y>"},
+    {"name": "ERROR", "no": 40, "color": "<y>"},
+    {"name": "CRITICAL", "no": 50, "color": "<r>"},
+]
+
+
+def base_logger():
+    """Initialize hyperglass logging instance."""
+    _loguru_logger.remove()
+    _loguru_logger.add(sys.stdout, format=_LOG_FMT, level="INFO")
+    _loguru_logger.configure(levels=_LOG_LEVELS)
+    return _loguru_logger
+
+
+log = base_logger()
+
+
+def set_log_level(logger, debug):
+    """Set log level based on debug state."""
+    if debug:
+        os.environ["HYPERGLASS_LOG_LEVEL"] = "DEBUG"
+        logger.remove()
+        logger.add(sys.stdout, format=_LOG_FMT, level="DEBUG")
+        logger.configure(levels=_LOG_LEVELS)
+
+    if debug:
+        logger.debug("Debugging enabled")
+    return True
+
+
+def enable_file_logging(logger, log_directory, log_format, log_max_size):
+    """Set up file-based logging from configuration parameters."""
+
+    if log_format == "json":
+        log_file_name = "hyperglass_log.json"
+        structured = True
+    else:
+        log_file_name = "hyperglass_log.log"
+        structured = False
+
+    log_file = log_directory / log_file_name
+
+    if log_format == "text":
+        now_str = "hyperglass logs for " + datetime.utcnow().strftime(
+            "%B %d, %Y beginning at %H:%M:%S UTC"
+        )
+        now_str_y = len(now_str) + 6
+        now_str_x = len(now_str) + 4
+        log_break = (
+            "#" * now_str_y,
+            "\n#" + " " * now_str_x + "#\n",
+            "#  ",
+            now_str,
+            "  #",
+            "\n#" + " " * now_str_x + "#\n",
+            "#" * now_str_y,
+        )
+
+        with log_file.open("a+") as lf:
+            lf.write(f'\n\n{"".join(log_break)}\n\n')
+
+    logger.add(log_file, rotation=log_max_size, serialize=structured)
+
+    logger.debug("Logging to file enabled")
+
+    return True
+
+
+def enable_syslog_logging(logger, syslog_host, syslog_port):
+    """Set up syslog logging from configuration parameters."""
+    from logging.handlers import SysLogHandler
+
+    logger.add(
+        SysLogHandler(address=(str(syslog_host), syslog_port)), format="{message}"
+    )
+    logger.debug(
+        "Logging to syslog target {h}:{p} enabled",
+        h=str(syslog_host),
+        p=str(syslog_port),
+    )
+    return True

hyperglass/main.py

@@ -11,6 +11,7 @@ from gunicorn.arbiter import Arbiter
 from gunicorn.app.base import BaseApplication
 
 # Project
+from hyperglass.log import log
 from hyperglass.constants import MIN_PYTHON_VERSION, __version__
 
 pretty_version = ".".join(tuple(str(v) for v in MIN_PYTHON_VERSION))
@@ -27,7 +28,6 @@ from hyperglass.configuration import (  # isort:skip
     frontend_params,
 )
 from hyperglass.util import (  # isort:skip
-    log,
     cpu_count,
     check_redis,
     build_frontend,

hyperglass/util.py

@@ -1,17 +1,6 @@
 """Utility functions."""
 
-
-def _logger():
-    from loguru import logger as _loguru_logger
-    from hyperglass.constants import LOG_HANDLER
-    from hyperglass.constants import LOG_LEVELS
-
-    _loguru_logger.remove()
-    _loguru_logger.configure(handlers=[LOG_HANDLER], levels=LOG_LEVELS)
-    return _loguru_logger
-
-
-log = _logger()
+from hyperglass.log import log
 
 
 def cpu_count(multiplier: int = 0):