improve config import & logging initialization

hyperglass/configuration/__init__.py

@@ -4,6 +4,7 @@
 import os
 import copy
 import json
+from typing import Dict, List, Union, Callable
 from pathlib import Path
 
 # Third Party
@@ -17,7 +18,8 @@ from hyperglass.log import (
     enable_file_logging,
     enable_syslog_logging,
 )
-from hyperglass.util import check_path, set_app_path, set_cache_env
+from hyperglass.util import check_path, set_app_path, set_cache_env, current_log_level
+from hyperglass.models import HyperglassModel
 from hyperglass.constants import (
     CREDIT,
     DEFAULT_HELP,
@@ -87,32 +89,34 @@ STATIC_PATH = CONFIG_PATH / "static"
 CONFIG_MAIN, CONFIG_DEVICES, CONFIG_COMMANDS = _check_config_files(CONFIG_PATH)
 
 
-def _config_required(config_path: Path) -> dict:
+def _config_required(config_path: Path) -> Dict:
     try:
         with config_path.open("r") as cf:
             config = yaml.safe_load(cf)
-            log.debug(
-                "Unvalidated data from file '{f}': {c}", f=str(config_path), c=config
-            )
+
     except (yaml.YAMLError, yaml.MarkedYAMLError) as yaml_error:
         raise ConfigError(str(yaml_error))
+
+    if config is None:
+        log.critical("{} appears to be empty", str(config_path))
+        raise ConfigMissing(missing_item=config_path.name)
+
     return config
 
 
-def _config_optional(config_path: Path) -> dict:
+def _config_optional(config_path: Path) -> Dict:
+
     if config_path is None:
         config = {}
+
     else:
         try:
             with config_path.open("r") as cf:
                 config = yaml.safe_load(cf) or {}
-                log.debug(
-                    "Unvalidated data from file '{f}': {c}",
-                    f=str(config_path),
-                    c=config,
-                )
+
         except (yaml.YAMLError, yaml.MarkedYAMLError) as yaml_error:
             raise ConfigError(error_msg=str(yaml_error))
+
     return config
 
 
@@ -138,34 +142,53 @@ def _validate_nos_commands(all_nos, commands):
     return True
 
 
+def _validate_config(config: Union[Dict, List], importer: Callable) -> HyperglassModel:
+    validated = None
+    try:
+        if isinstance(config, Dict):
+            validated = importer(**config)
+        elif isinstance(config, List):
+            validated = importer(config)
+    except ValidationError as err:
+        log.error(str(err))
+        raise ConfigInvalid(err.errors()) from None
+
+    return validated
+
+
 user_config = _config_optional(CONFIG_MAIN)
 
 # Read raw debug value from config to enable debugging quickly.
 set_log_level(logger=log, debug=user_config.get("debug", True))
 
-_user_commands = _config_optional(CONFIG_COMMANDS)
-_user_devices = _config_required(CONFIG_DEVICES)
-
-# Map imported user config files to expected schema:
-try:
-    params = _params.Params(**user_config)
-    commands = _commands.Commands.import_params(_user_commands)
-    devices = _routers.Routers._import(_user_devices.get("routers", {}))
-except ValidationError as validation_errors:
-    errors = validation_errors.errors()
-    log.error(errors)
-    for error in errors:
-        raise ConfigInvalid(
-            field=": ".join([str(item) for item in error["loc"]]),
-            error_msg=error["msg"],
-        )
-
-_validate_nos_commands(devices.all_nos, commands)
-
-set_cache_env(db=params.cache.database, host=params.cache.host, port=params.cache.port)
+# Map imported user configuration to expected schema.
+log.debug("Unvalidated configuration from {}: {}", CONFIG_MAIN, user_config)
+params = _validate_config(config=user_config, importer=_params.Params)
 
 # Re-evaluate debug state after config is validated
-set_log_level(logger=log, debug=params.debug)
+if params.debug and current_log_level(log) != "debug":
+    set_log_level(logger=log, debug=True)
+
+# Map imported user commands to expected schema.
+_user_commands = _config_optional(CONFIG_COMMANDS)
+log.debug("Unvalidated commands from {}: {}", CONFIG_COMMANDS, _user_commands)
+commands = _validate_config(
+    config=_user_commands, importer=_commands.Commands.import_params
+)
+
+# Map imported user devices to expected schema.
+_user_devices = _config_required(CONFIG_DEVICES)
+log.debug("Unvalidated devices from {}: {}", CONFIG_DEVICES, _user_devices)
+devices = _validate_config(
+    config=_user_devices.get("routers", []), importer=_routers.Routers._import,
+)
+
+# Validate commands are both supported and properly mapped.
+_validate_nos_commands(devices.all_nos, commands)
+
+# Set cache configurations to environment variables, so they can be
+# used without importing this module (Gunicorn, etc).
+set_cache_env(db=params.cache.database, host=params.cache.host, port=params.cache.port)
 
 # Set up file logging once configuration parameters are initialized.
 enable_file_logging(

hyperglass/configuration/models/commands/__init__.py

@@ -34,7 +34,7 @@ class Commands(HyperglassModelExtra):
     vyos: CommandGroup = VyosCommands()
 
     @classmethod
-    def import_params(cls, input_params):
+    def import_params(cls, **input_params):
         """Import loaded YAML, initialize per-command definitions.
 
         Dynamically set attributes for the command class.

hyperglass/exceptions.py

@@ -2,9 +2,11 @@
 
 # Standard Library
 import json as _json
+from typing import List
 
 # Project
 from hyperglass.log import log
+from hyperglass.util import validation_error_message
 from hyperglass.constants import STATUS_CODE_MAP
 
 
@@ -136,14 +138,17 @@ class _PredefinedHyperglassError(HyperglassError):
         )
 
 
-class ConfigError(_UnformattedHyperglassError):
-    """Raised for generic user-config issues."""
-
-
-class ConfigInvalid(_PredefinedHyperglassError):
+class ConfigInvalid(HyperglassError):
     """Raised when a config item fails type or option validation."""
 
-    _message = 'The value field "{field}" is invalid: {error_msg}'
+    def __init__(self, errors: List) -> None:
+        """Parse Pydantic ValidationError."""
+
+        super().__init__(message=validation_error_message(*errors))
+
+
+class ConfigError(_UnformattedHyperglassError):
+    """Raised for generic user-config issues."""
 
 
 class ConfigMissing(_PredefinedHyperglassError):

hyperglass/util.py

@@ -1,13 +1,17 @@
 """Utility functions."""
 
 # Standard Library
+import re
 import math
 import shutil
 from queue import Queue
-from typing import Iterable
+from typing import Dict, Iterable
 from pathlib import Path
 from threading import Thread
 
+# Third Party
+from loguru._logger import Logger as LoguruLogger
+
 # Project
 from hyperglass.log import log
 
@@ -38,8 +42,6 @@ def clean_name(_name):
     Returns:
         {str} -- Cleaned field name
     """
-    import re
-
     _replaced = re.sub(r"[\-|\.|\@|\~|\:\/|\s]", "_", _name)
     _scrubbed = "".join(re.findall(r"([a-zA-Z]\w+|\_+)", _replaced))
     return _scrubbed.lower()
@@ -751,8 +753,6 @@ def import_public_key(app_path, device_name, keystring):
     Returns:
         {bool} -- True if file was written
     """
-    import re
-
     if not isinstance(app_path, Path):
         app_path = Path(app_path)
 
@@ -945,3 +945,30 @@ def validate_nos(nos):
         result = (True, "scrape")
 
     return result
+
+
+def current_log_level(logger: LoguruLogger) -> str:
+    """Get the current log level of a logger instance."""
+
+    try:
+        handler = list(logger._core.handlers.values())[0]
+        levels = {v.no: k for k, v in logger._core.levels.items()}
+        current_level = levels[handler.levelno].lower()
+
+    except Exception as err:
+        logger.error(err)
+        current_level = "info"
+
+    return current_level
+
+
+def validation_error_message(*errors: Dict) -> str:
+    """Parse errors return from pydantic.ValidationError.errors()."""
+
+    errs = ("\n",)
+
+    for err in errors:
+        loc = " → ".join(str(loc) for loc in err["loc"])
+        errs += (f'Field: {loc}\n  Error: {err["msg"]}\n',)
+
+    return "\n".join(errs)