From 97484e16c22e85f14eef6fbfa27317d6230db387 Mon Sep 17 00:00:00 2001 From: checktheroads Date: Mon, 13 Apr 2020 01:04:28 -0700 Subject: [PATCH] remove async config init --- hyperglass/configuration/__init__.py | 71 +++++++++++++++++----------- hyperglass/configuration/markdown.py | 18 +++---- hyperglass/util.py | 12 ++--- 3 files changed, 57 insertions(+), 44 deletions(-) diff --git a/hyperglass/configuration/__init__.py b/hyperglass/configuration/__init__.py index fe58188..fd2edd7 100644 --- a/hyperglass/configuration/__init__.py +++ b/hyperglass/configuration/__init__.py @@ -26,7 +26,6 @@ from hyperglass.constants import ( __version__, ) from hyperglass.exceptions import ConfigError, ConfigInvalid, ConfigMissing -from hyperglass.compat._asyncio import aiorun from hyperglass.configuration.models import params as _params from hyperglass.configuration.models import routers as _routers from hyperglass.configuration.models import commands as _commands @@ -46,7 +45,7 @@ CONFIG_FILES = ( ) -async def _check_config_files(directory): +def _check_config_files(directory): """Verify config files exist and are readable. Arguments: @@ -63,7 +62,7 @@ async def _check_config_files(directory): file_name, required = file file_path = directory / file_name - checked = await check_path(file_path) + checked = check_path(file_path) if checked is None and required: raise ConfigMissing(missing_item=str(file_path)) @@ -81,7 +80,7 @@ async def _check_config_files(directory): STATIC_PATH = CONFIG_PATH / "static" -CONFIG_MAIN, CONFIG_DEVICES, CONFIG_COMMANDS = aiorun(_check_config_files(CONFIG_PATH)) +CONFIG_MAIN, CONFIG_DEVICES, CONFIG_COMMANDS = _check_config_files(CONFIG_PATH) def _set_log_level(debug, log_file=None): @@ -115,6 +114,35 @@ def _set_log_level(debug, log_file=None): return True +def _config_required(config_path: Path) -> dict: + try: + with config_path.open("r") as cf: + config = yaml.safe_load(cf) + log.debug( + "Unvalidated data from file '{f}': {c}", f=str(config_path), c=config + ) + except (yaml.YAMLError, yaml.MarkedYAMLError) as yaml_error: + raise ConfigError(error_msg=str(yaml_error)) + return config + + +def _config_optional(config_path: Path) -> dict: + if config_path is None: + config = {} + else: + try: + with config_path.open("r") as cf: + config = yaml.safe_load(cf) or {} + log.debug( + "Unvalidated data from file '{f}': {c}", + f=str(config_path), + c=config, + ) + except (yaml.YAMLError, yaml.MarkedYAMLError) as yaml_error: + raise ConfigError(error_msg=str(yaml_error)) + return config + + async def _config_main(): """Open main config file and load YAML to dict. @@ -166,19 +194,16 @@ async def _config_devices(): return config -user_config = aiorun(_config_main()) +user_config = _config_optional(CONFIG_MAIN) # Logging Config -try: - _debug = user_config["debug"] -except KeyError: - _debug = True +_debug = user_config.get("debug", True) # Read raw debug value from config to enable debugging quickly. _set_log_level(_debug) -_user_commands = aiorun(_config_commands()) -_user_devices = aiorun(_config_devices()) +_user_commands = _config_optional(CONFIG_COMMANDS) +_user_devices = _config_required(CONFIG_DEVICES) # Map imported user config files to expected schema: try: @@ -412,12 +437,10 @@ def _build_vrf_help(): "title" ] = f"{vrf.display_name}: {command_params.display_name}" - md = aiorun( - get_markdown( - config_path=cmd, - default=DEFAULT_DETAILS[command], - params=help_params, - ) + md = get_markdown( + config_path=cmd, + default=DEFAULT_DETAILS[command], + params=help_params, ) vrf_help.update( @@ -439,20 +462,14 @@ content_vrf = _build_vrf_help() content_help_params = copy.copy(content_params) content_help_params["title"] = params.web.help_menu.title -content_help = aiorun( - get_markdown( - config_path=params.web.help_menu, - default=DEFAULT_HELP, - params=content_help_params, - ) +content_help = get_markdown( + config_path=params.web.help_menu, default=DEFAULT_HELP, params=content_help_params ) content_terms_params = copy.copy(content_params) content_terms_params["title"] = params.web.terms.title -content_terms = aiorun( - get_markdown( - config_path=params.web.terms, default=DEFAULT_TERMS, params=content_terms_params - ) +content_terms = get_markdown( + config_path=params.web.terms, default=DEFAULT_TERMS, params=content_terms_params ) content_credit = CREDIT.format(version=__version__) diff --git a/hyperglass/configuration/markdown.py b/hyperglass/configuration/markdown.py index d92ad8e..36d5433 100644 --- a/hyperglass/configuration/markdown.py +++ b/hyperglass/configuration/markdown.py @@ -1,13 +1,10 @@ """Markdown processing utility functions.""" -# Third Party -from aiofile import AIOFile - # Project from hyperglass.util import log -async def _get_file(path_obj): +def _get_file(path_obj): """Read a file. Arguments: @@ -16,12 +13,11 @@ async def _get_file(path_obj): Returns: {str} -- File contents """ - async with AIOFile(path_obj, "r") as raw_file: - file = await raw_file.read() - return file + with path_obj.open("r") as raw_file: + return raw_file.read() -async def format_markdown(content, params): +def format_markdown(content, params): """Format content with config parameters. Arguments: @@ -37,7 +33,7 @@ async def format_markdown(content, params): return fmt -async def get_markdown(config_path, default, params): +def get_markdown(config_path, default, params): """Get markdown file if specified, or use default. Format the content with config parameters. @@ -52,13 +48,13 @@ async def get_markdown(config_path, default, params): log.trace(f"Getting Markdown content for '{params['title']}'") if config_path.enable and config_path.file is not None: - md = await _get_file(config_path.file) + md = _get_file(config_path.file) else: md = default log.trace(f"Unformatted Content for '{params['title']}':\n{md}") - md_fmt = await format_markdown(md, params) + md_fmt = format_markdown(md, params) log.trace(f"Formatted Content for '{params['title']}':\n{md_fmt}") diff --git a/hyperglass/util.py b/hyperglass/util.py index 8bac32b..db62f7a 100644 --- a/hyperglass/util.py +++ b/hyperglass/util.py @@ -14,7 +14,7 @@ def _logger(): log = _logger() -def cpu_count(): +def cpu_count(multiplier: int = 0): """Get server's CPU core count. Used for number of web server workers. @@ -24,7 +24,7 @@ def cpu_count(): """ import multiprocessing - return multiprocessing.cpu_count() + return multiprocessing.cpu_count() * multiplier def clean_name(_name): @@ -47,7 +47,7 @@ def clean_name(_name): return _scrubbed.lower() -async def check_path(path, mode="r"): +def check_path(path, mode="r"): """Verify if a path exists and is accessible. Arguments: @@ -61,7 +61,6 @@ async def check_path(path, mode="r"): {Path|None} -- Path object if checks pass, None if not. """ from pathlib import Path - from aiofile import AIOFile try: if not isinstance(path, Path): @@ -70,7 +69,7 @@ async def check_path(path, mode="r"): if not path.exists(): raise FileNotFoundError(f"{str(path)} does not exist.") - async with AIOFile(path, mode): + with path.open(mode): result = path except Exception: @@ -497,6 +496,7 @@ async def build_frontend( # noqa: C901 temp_file = tempfile.NamedTemporaryFile( mode="w+", prefix="hyperglass_", suffix=".json", delete=not dev_mode ) + log.info("Starting UI build...") log.debug( f"Created temporary UI config file: '{temp_file.name}' for build {build_id}" ) @@ -523,7 +523,7 @@ async def build_frontend( # noqa: C901 log.debug("Re-initialized node_modules") if build_result: - log.debug("Completed UI build") + log.success("Completed UI build") elif dev_mode and not force: log.debug("Running in developer mode, did not build new UI files")