From fe84d72c622da741c438bb47fca76095a6b3f9cb Mon Sep 17 00:00:00 2001 From: checktheroads Date: Tue, 14 Jul 2020 00:15:41 -0700 Subject: [PATCH] improve config import & logging initialization --- hyperglass/configuration/__init__.py | 87 ++++++++++++------- .../configuration/models/commands/__init__.py | 2 +- hyperglass/exceptions.py | 17 ++-- hyperglass/util.py | 37 ++++++-- validate_examples.py | 2 +- 5 files changed, 100 insertions(+), 45 deletions(-) diff --git a/hyperglass/configuration/__init__.py b/hyperglass/configuration/__init__.py index 2c3d4d1..dd852b5 100644 --- a/hyperglass/configuration/__init__.py +++ b/hyperglass/configuration/__init__.py @@ -4,6 +4,7 @@ import os import copy import json +from typing import Dict, List, Union, Callable from pathlib import Path # Third Party @@ -17,7 +18,8 @@ from hyperglass.log import ( enable_file_logging, enable_syslog_logging, ) -from hyperglass.util import check_path, set_app_path, set_cache_env +from hyperglass.util import check_path, set_app_path, set_cache_env, current_log_level +from hyperglass.models import HyperglassModel from hyperglass.constants import ( CREDIT, DEFAULT_HELP, @@ -87,32 +89,34 @@ STATIC_PATH = CONFIG_PATH / "static" CONFIG_MAIN, CONFIG_DEVICES, CONFIG_COMMANDS = _check_config_files(CONFIG_PATH) -def _config_required(config_path: Path) -> dict: +def _config_required(config_path: Path) -> Dict: try: with config_path.open("r") as cf: config = yaml.safe_load(cf) - log.debug( - "Unvalidated data from file '{f}': {c}", f=str(config_path), c=config - ) + except (yaml.YAMLError, yaml.MarkedYAMLError) as yaml_error: raise ConfigError(str(yaml_error)) + + if config is None: + log.critical("{} appears to be empty", str(config_path)) + raise ConfigMissing(missing_item=config_path.name) + return config -def _config_optional(config_path: Path) -> dict: +def _config_optional(config_path: Path) -> Dict: + if config_path is None: config = {} + else: try: with config_path.open("r") as cf: config = yaml.safe_load(cf) or {} - log.debug( - "Unvalidated data from file '{f}': {c}", - f=str(config_path), - c=config, - ) + except (yaml.YAMLError, yaml.MarkedYAMLError) as yaml_error: raise ConfigError(error_msg=str(yaml_error)) + return config @@ -138,34 +142,53 @@ def _validate_nos_commands(all_nos, commands): return True +def _validate_config(config: Union[Dict, List], importer: Callable) -> HyperglassModel: + validated = None + try: + if isinstance(config, Dict): + validated = importer(**config) + elif isinstance(config, List): + validated = importer(config) + except ValidationError as err: + log.error(str(err)) + raise ConfigInvalid(err.errors()) from None + + return validated + + user_config = _config_optional(CONFIG_MAIN) # Read raw debug value from config to enable debugging quickly. set_log_level(logger=log, debug=user_config.get("debug", True)) -_user_commands = _config_optional(CONFIG_COMMANDS) -_user_devices = _config_required(CONFIG_DEVICES) - -# Map imported user config files to expected schema: -try: - params = _params.Params(**user_config) - commands = _commands.Commands.import_params(_user_commands) - devices = _routers.Routers._import(_user_devices.get("routers", {})) -except ValidationError as validation_errors: - errors = validation_errors.errors() - log.error(errors) - for error in errors: - raise ConfigInvalid( - field=": ".join([str(item) for item in error["loc"]]), - error_msg=error["msg"], - ) - -_validate_nos_commands(devices.all_nos, commands) - -set_cache_env(db=params.cache.database, host=params.cache.host, port=params.cache.port) +# Map imported user configuration to expected schema. +log.debug("Unvalidated configuration from {}: {}", CONFIG_MAIN, user_config) +params = _validate_config(config=user_config, importer=_params.Params) # Re-evaluate debug state after config is validated -set_log_level(logger=log, debug=params.debug) +if params.debug and current_log_level(log) != "debug": + set_log_level(logger=log, debug=True) + +# Map imported user commands to expected schema. +_user_commands = _config_optional(CONFIG_COMMANDS) +log.debug("Unvalidated commands from {}: {}", CONFIG_COMMANDS, _user_commands) +commands = _validate_config( + config=_user_commands, importer=_commands.Commands.import_params +) + +# Map imported user devices to expected schema. +_user_devices = _config_required(CONFIG_DEVICES) +log.debug("Unvalidated devices from {}: {}", CONFIG_DEVICES, _user_devices) +devices = _validate_config( + config=_user_devices.get("routers", []), importer=_routers.Routers._import, +) + +# Validate commands are both supported and properly mapped. +_validate_nos_commands(devices.all_nos, commands) + +# Set cache configurations to environment variables, so they can be +# used without importing this module (Gunicorn, etc). +set_cache_env(db=params.cache.database, host=params.cache.host, port=params.cache.port) # Set up file logging once configuration parameters are initialized. enable_file_logging( diff --git a/hyperglass/configuration/models/commands/__init__.py b/hyperglass/configuration/models/commands/__init__.py index d14dca4..827e556 100644 --- a/hyperglass/configuration/models/commands/__init__.py +++ b/hyperglass/configuration/models/commands/__init__.py @@ -34,7 +34,7 @@ class Commands(HyperglassModelExtra): vyos: CommandGroup = VyosCommands() @classmethod - def import_params(cls, input_params): + def import_params(cls, **input_params): """Import loaded YAML, initialize per-command definitions. Dynamically set attributes for the command class. diff --git a/hyperglass/exceptions.py b/hyperglass/exceptions.py index 0b4dc60..850a5d1 100644 --- a/hyperglass/exceptions.py +++ b/hyperglass/exceptions.py @@ -2,9 +2,11 @@ # Standard Library import json as _json +from typing import List # Project from hyperglass.log import log +from hyperglass.util import validation_error_message from hyperglass.constants import STATUS_CODE_MAP @@ -136,14 +138,17 @@ class _PredefinedHyperglassError(HyperglassError): ) -class ConfigError(_UnformattedHyperglassError): - """Raised for generic user-config issues.""" - - -class ConfigInvalid(_PredefinedHyperglassError): +class ConfigInvalid(HyperglassError): """Raised when a config item fails type or option validation.""" - _message = 'The value field "{field}" is invalid: {error_msg}' + def __init__(self, errors: List) -> None: + """Parse Pydantic ValidationError.""" + + super().__init__(message=validation_error_message(*errors)) + + +class ConfigError(_UnformattedHyperglassError): + """Raised for generic user-config issues.""" class ConfigMissing(_PredefinedHyperglassError): diff --git a/hyperglass/util.py b/hyperglass/util.py index ba10906..a69c30e 100644 --- a/hyperglass/util.py +++ b/hyperglass/util.py @@ -1,13 +1,17 @@ """Utility functions.""" # Standard Library +import re import math import shutil from queue import Queue -from typing import Iterable +from typing import Dict, Iterable from pathlib import Path from threading import Thread +# Third Party +from loguru._logger import Logger as LoguruLogger + # Project from hyperglass.log import log @@ -38,8 +42,6 @@ def clean_name(_name): Returns: {str} -- Cleaned field name """ - import re - _replaced = re.sub(r"[\-|\.|\@|\~|\:\/|\s]", "_", _name) _scrubbed = "".join(re.findall(r"([a-zA-Z]\w+|\_+)", _replaced)) return _scrubbed.lower() @@ -751,8 +753,6 @@ def import_public_key(app_path, device_name, keystring): Returns: {bool} -- True if file was written """ - import re - if not isinstance(app_path, Path): app_path = Path(app_path) @@ -945,3 +945,30 @@ def validate_nos(nos): result = (True, "scrape") return result + + +def current_log_level(logger: LoguruLogger) -> str: + """Get the current log level of a logger instance.""" + + try: + handler = list(logger._core.handlers.values())[0] + levels = {v.no: k for k, v in logger._core.levels.items()} + current_level = levels[handler.levelno].lower() + + except Exception as err: + logger.error(err) + current_level = "info" + + return current_level + + +def validation_error_message(*errors: Dict) -> str: + """Parse errors return from pydantic.ValidationError.errors().""" + + errs = ("\n",) + + for err in errors: + loc = " → ".join(str(loc) for loc in err["loc"]) + errs += (f'Field: {loc}\n Error: {err["msg"]}\n',) + + return "\n".join(errs) diff --git a/validate_examples.py b/validate_examples.py index a0001ed..0c7c8ee 100644 --- a/validate_examples.py +++ b/validate_examples.py @@ -67,7 +67,7 @@ def _validate_commands(): with COMMANDS.open() as raw: commands_dict = yaml.safe_load(raw.read()) or {} try: - Commands.import_params(commands_dict) + Commands.import_params(**commands_dict) except Exception as e: raise ValueError(str(e)) return True