Complete new config file implementation

hyperglass/configuration/.gitignore

@@ -1,5 +0,0 @@
-.DS_Store
-*.toml
-*.yaml
-*.test
-configuration_old

hyperglass/configuration/__init__.py

@@ -6,7 +6,7 @@ from hyperglass.state import use_state
 from hyperglass.defaults.directives import init_builtin_directives
 
 # Local
-from .main import init_params, init_devices, init_ui_params, init_directives
+from .validate import init_params, init_devices, init_ui_params, init_directives
 
 __all__ = ("init_user_config",)
 

hyperglass/configuration/load.py

@@ -5,6 +5,7 @@ import typing as t
 from pathlib import Path
 
 # Project
+from hyperglass.log import log
 from hyperglass.util import run_coroutine_in_new_thread
 from hyperglass.settings import Settings
 from hyperglass.constants import CONFIG_EXTENSIONS
@@ -66,6 +67,7 @@ def load_dsl(path: Path, *, empty_allowed: bool) -> LoadedConfig:
             raise ConfigError(
                 "'{!s}' exists, but it is empty and is required to start hyperglass.".format(path),
             )
+    log.debug("Loaded configuration from {!s}", path)
     return data or {}
 
 
@@ -102,6 +104,8 @@ def load_python(path: Path, *, empty_allowed: bool) -> LoadedConfig:
 
     if data is None and empty_allowed is False:
         raise ConfigError(f"'{path!s} exists', but variable or function 'main' is an invalid type")
+
+    log.debug("Loaded configuration from {!s}", path)
     return data or {}
 
 

hyperglass/configuration/markdown.py

@@ -1,61 +1,26 @@
 """Markdown processing utility functions."""
 
-# Project
+# Standard Library
-from hyperglass.log import log
+import typing as t
+from pathlib import Path
+
+if t.TYPE_CHECKING:
+    # Project
+    from hyperglass.models import HyperglassModel
 
 
-def _get_file(path_obj):
+def get_markdown(config: "HyperglassModel", default: str, params: t.Dict[str, t.Any]) -> str:
-    """Read a file.
+    """Get markdown file if specified, or use default."""
 
-    Arguments:
+    if config.enable and config.file is not None:
-        path_obj {Path} -- Path to file.
+        # with config_path.file
-
+        if hasattr(config, "file") and isinstance(config.file, Path):
-    Returns:
+            with config.file.open("r") as config_file:
-        {str} -- File contents
+                md = config_file.read()
-    """
-    with path_obj.open("r") as raw_file:
-        return raw_file.read()
-
-
-def format_markdown(content, params):
-    """Format content with config parameters.
-
-    Arguments:
-        content {str} -- Unformatted content
-
-    Returns:
-        {str} -- Formatted content
-    """
-    try:
-        fmt = content.format(**params)
-    except KeyError:
-        fmt = content
-    return fmt
-
-
-def get_markdown(config_path, default, params):
-    """Get markdown file if specified, or use default.
-
-    Format the content with config parameters.
-
-    Arguments:
-        config_path {object} -- content config
-        default {str} -- default content
-
-    Returns:
-        {str} -- Formatted content
-    """
-    log.trace(f"Getting Markdown content for '{params['title']}'")
-
-    if config_path.enable and config_path.file is not None:
-        md = _get_file(config_path.file)
     else:
         md = default
 
-    log.trace(f"Unformatted Content for '{params['title']}':\n{md}")
+    try:
-
+        return md.format(**params)
-    md_fmt = format_markdown(md, params)
+    except KeyError:
-
+        return md
-    log.trace(f"Formatted Content for '{params['title']}':\n{md_fmt}")
-
-    return md_fmt

hyperglass/configuration/validate.py

@@ -1,26 +1,20 @@
-"""Import configuration files and returns default values if undefined."""
+"""Import configuration files and run validation."""
 
-# Standard Library
-import typing as t
-from pathlib import Path
 
 # Third Party
-import yaml
 from pydantic import ValidationError
 
 # Project
 from hyperglass.log import log, enable_file_logging, enable_syslog_logging
-from hyperglass.settings import Settings
 from hyperglass.models.ui import UIParameters
-from hyperglass.util.files import check_path
 from hyperglass.models.directive import Directive, Directives
-from hyperglass.exceptions.private import ConfigError, ConfigMissing
+from hyperglass.exceptions.private import ConfigError, ConfigInvalid
 from hyperglass.models.config.params import Params
 from hyperglass.models.config.devices import Devices
 
 # Local
+from .load import load_config
 from .markdown import get_markdown
-from .validation import validate_config
 
 __all__ = (
     "init_params",
@@ -29,93 +23,12 @@ __all__ = (
     "init_ui_params",
 )
 
-# Project Directories
-CONFIG_PATH = Settings.app_path
-CONFIG_FILES = (
-    ("hyperglass.yaml", False),
-    ("devices.yaml", True),
-    ("directives.yaml", False),
-)
-
-
-def _check_config_files(directory: Path):
-    """Verify config files exist and are readable."""
-
-    files = ()
-
-    for file in CONFIG_FILES:
-        file_name, required = file
-        file_path = directory / file_name
-
-        checked = check_path(file_path)
-
-        if checked is None and required:
-            raise ConfigMissing(missing_item=str(file_path))
-
-        if checked is None and not required:
-            log.warning(
-                "'{f}' was not found, but is not required to run hyperglass. "
-                + "Defaults will be used.",
-                f=str(file_path),
-            )
-        files += (checked,)
-
-    return files
-
-
-CONFIG_MAIN, CONFIG_DEVICES, CONFIG_DIRECTIVES = _check_config_files(CONFIG_PATH)
-
-
-def _config_required(config_path: Path) -> t.Dict[str, t.Any]:
-    try:
-        with config_path.open("r") as cf:
-            config = yaml.safe_load(cf)
-
-    except (yaml.YAMLError, yaml.MarkedYAMLError) as yaml_error:
-        raise ConfigError(message="Error reading YAML file: '{e}'", e=yaml_error)
-
-    if config is None:
-        raise ConfigMissing(missing_item=config_path.name)
-
-    return config
-
-
-def _config_optional(config_path: Path) -> t.Dict[str, t.Any]:
-
-    config = {}
-
-    if config_path is None:
-        return config
-
-    else:
-        try:
-            with config_path.open("r") as cf:
-                config = yaml.safe_load(cf) or {}
-
-        except (yaml.YAMLError, yaml.MarkedYAMLError) as yaml_error:
-            raise ConfigError(message="Error reading YAML file: '{e}'", e=yaml_error)
-
-    return config
-
-
-def _get_directives(data: t.Dict[str, t.Any]) -> "Directives":
-    directives = ()
-    for name, directive in data.items():
-        try:
-            directives += (Directive(id=name, **directive),)
-        except ValidationError as err:
-            raise ConfigError(
-                message="Validation error in directive '{d}': '{e}'", d=name, e=err
-            ) from err
-    return Directives(*directives)
-
 
 def init_params() -> "Params":
     """Validate & initialize configuration parameters."""
-    user_config = _config_optional(CONFIG_MAIN)
+    user_config = load_config("config", required=False)
     # Map imported user configuration to expected schema.
-    log.debug("Unvalidated configuration from {}: {}", CONFIG_MAIN, user_config)
+    params = Params(**user_config)
-    params = validate_config(config=user_config, importer=Params)
 
     # Set up file logging once configuration parameters are initialized.
     enable_file_logging(
@@ -160,27 +73,35 @@ def init_params() -> "Params":
 def init_directives() -> "Directives":
     """Validate & initialize directives."""
     # Map imported user directives to expected schema.
-    _user_directives = _config_optional(CONFIG_DIRECTIVES)
+    directives = load_config("directives", required=False)
-    log.debug("Unvalidated directives from {!s}: {}", CONFIG_DIRECTIVES, _user_directives)
+    try:
-    return _get_directives(_user_directives)
+        directives = (
+            Directive(id=name, **directive)
+            for name, directive in load_config("directives", required=False).items()
+        )
+
+    except ValidationError as err:
+        raise ConfigInvalid(errors=err.errors()) from err
+
+    return Directives(*directives)
 
 
 def init_devices() -> "Devices":
     """Validate & initialize devices."""
-    devices_config = _config_required(CONFIG_DEVICES)
+    devices_config = load_config("devices", required=True)
-    log.debug("Unvalidated devices from {!s}: {!r}", CONFIG_DEVICES, devices_config)
     items = []
 
+    # Support first matching main key name.
     for key in ("main", "devices", "routers"):
         if key in devices_config:
             items = devices_config[key]
             break
 
     if len(items) < 1:
-        raise ConfigError("No devices are defined in devices.yaml")
+        raise ConfigError("No devices are defined in devices file")
 
     devices = Devices(*items)
-    log.info("Initialized devices {!r}", devices)
+    log.debug("Initialized devices {!r}", devices)
 
     return devices
 

hyperglass/configuration/validation.py

@@ -1,28 +0,0 @@
-"""Post-Validation Validation.
-
-Some validations need to occur across multiple config files.
-"""
-# Standard Library
-from typing import Any, Dict, List, Union, TypeVar
-
-# Third Party
-from pydantic import ValidationError
-
-# Project
-from hyperglass.exceptions.private import ConfigInvalid
-
-Importer = TypeVar("Importer")
-
-
-def validate_config(config: Union[Dict[str, Any], List[Any]], importer: Importer) -> Importer:
-    """Validate a config dict against a model."""
-    validated = None
-    try:
-        if isinstance(config, Dict):
-            validated = importer(**config)
-        elif isinstance(config, List):
-            validated = importer(config)
-    except ValidationError as err:
-        raise ConfigInvalid(errors=err.errors()) from None
-
-    return validated