From 81c65ce71e12f89a03160f6d01fbcb34a4bfdabf Mon Sep 17 00:00:00 2001 From: checktheroads Date: Thu, 28 Jan 2021 23:02:25 -0700 Subject: [PATCH] restructure util module --- hyperglass/configuration/main.py | 3 +- hyperglass/main.py | 31 +- hyperglass/util/__init__.py | 602 +------------------------------ hyperglass/util/files.py | 163 +++++++++ hyperglass/util/frontend.py | 385 ++++++++++++++++++++ 5 files changed, 569 insertions(+), 615 deletions(-) create mode 100644 hyperglass/util/files.py diff --git a/hyperglass/configuration/main.py b/hyperglass/configuration/main.py index 050e1c9..61a32e5 100644 --- a/hyperglass/configuration/main.py +++ b/hyperglass/configuration/main.py @@ -17,13 +17,14 @@ from hyperglass.log import ( enable_file_logging, enable_syslog_logging, ) -from hyperglass.util import check_path, set_app_path, set_cache_env, current_log_level +from hyperglass.util import set_app_path, set_cache_env, current_log_level from hyperglass.constants import ( SUPPORTED_QUERY_TYPES, PARSED_RESPONSE_FIELDS, __version__, ) from hyperglass.exceptions import ConfigError, ConfigMissing +from hyperglass.util.files import check_path from hyperglass.models.commands import Commands from hyperglass.models.config.params import Params from hyperglass.models.config.devices import Devices diff --git a/hyperglass/main.py b/hyperglass/main.py index 020fcdd..7edf673 100644 --- a/hyperglass/main.py +++ b/hyperglass/main.py @@ -29,26 +29,21 @@ if node_version != MIN_NODE_VERSION: raise RuntimeError(f"NodeJS {MIN_NODE_VERSION}+ is required.") -# Local -from .cache import SyncCache +# Project +from hyperglass.compat._asyncio import aiorun -from .configuration import ( # isort:skip - params, +# Local +from .util import cpu_count, clear_redis_cache, format_listen_address +from .cache import SyncCache +from .configuration import ( URL_DEV, URL_PROD, CONFIG_PATH, REDIS_CONFIG, + params, frontend_params, ) -from .util import ( # isort:skip - cpu_count, - build_frontend, - clear_redis_cache, - format_listen_address, -) - - -from hyperglass.compat._asyncio import aiorun # isort:skip +from .util.frontend import build_frontend if params.debug: workers = 1 @@ -84,12 +79,8 @@ def check_redis_instance() -> bool: return True -async def build_ui(): - """Perform a UI build prior to starting the application. - - Returns: - {bool} -- True if successful. - """ +async def build_ui() -> bool: + """Perform a UI build prior to starting the application.""" await build_frontend( dev_mode=params.developer_mode, dev_url=URL_DEV, @@ -109,7 +100,7 @@ async def clear_cache(): pass -def cache_config(): +def cache_config() -> bool: """Add configuration to Redis cache as a pickled object.""" # Standard Library import pickle diff --git a/hyperglass/util/__init__.py b/hyperglass/util/__init__.py index b928e76..82924e8 100644 --- a/hyperglass/util/__init__.py +++ b/hyperglass/util/__init__.py @@ -2,33 +2,26 @@ # Standard Library import os -import re +import sys import json -import math -import shutil -import asyncio +import platform from queue import Queue -from typing import Dict, Union, Iterable, Optional, Generator +from typing import Dict, Union, Generator +from asyncio import iscoroutine from pathlib import Path from ipaddress import IPv4Address, IPv6Address, ip_address -from threading import Thread # Third Party from loguru._logger import Logger as LoguruLogger # Project from hyperglass.log import log -from hyperglass.cache import AsyncCache -from hyperglass.models import HyperglassModel -def cpu_count(multiplier: int = 0): +def cpu_count(multiplier: int = 0) -> int: """Get server's CPU core count. - Used for number of web server workers. - - Returns: - {int} -- CPU Cores + Used to determine the number of web server workers. """ # Standard Library import multiprocessing @@ -36,58 +29,8 @@ def cpu_count(multiplier: int = 0): return multiprocessing.cpu_count() * multiplier -def check_path( - path: Union[Path, str], mode: str = "r", create: bool = False -) -> Optional[Path]: - """Verify if a path exists and is accessible. - - Arguments: - path {Path|str} -- Path object or string of path - mode {str} -- File mode, r or w - - Raises: - RuntimeError: Raised if file does not exist or is not accessible - - Returns: - {Path|None} -- Path object if checks pass, None if not. - """ - - try: - if not isinstance(path, Path): - path = Path(path) - - if not path.exists(): - if create: - if path.is_file(): - path.parent.mkdir(parents=True) - else: - path.mkdir(parents=True) - else: - raise FileNotFoundError(f"{str(path)} does not exist.") - - if path.exists(): - with path.open(mode): - result = path - - except Exception: - result = None - - return result - - def check_python() -> str: - """Verify Python Version. - - Raises: - RuntimeError: Raised if running Python version is invalid. - - Returns: - {str} -- Python version - """ - # Standard Library - import sys - import platform - + """Verify Python Version.""" # Project from hyperglass.constants import MIN_PYTHON_VERSION @@ -97,67 +40,6 @@ def check_python() -> str: return platform.python_version() -def get_ui_build_timeout() -> int: - """Read the UI build timeout from environment variables or set a default.""" - - timeout = 90 - - if "HYPERGLASS_UI_BUILD_TIMEOUT" in os.environ: - timeout = int(os.environ["HYPERGLASS_UI_BUILD_TIMEOUT"]) - log.info("Found UI build timeout environment variable: {}", timeout) - - elif "POETRY_HYPERGLASS_UI_BUILD_TIMEOUT" in os.environ: - timeout = int(os.environ["POETRY_HYPERGLASS_UI_BUILD_TIMEOUT"]) - log.info("Found UI build timeout environment variable: {}", timeout) - - return timeout - - -async def build_ui(app_path): - """Execute `next build` & `next export` from UI directory. - - Raises: - RuntimeError: Raised if exit code is not 0. - RuntimeError: Raised when any other error occurs. - """ - timeout = get_ui_build_timeout() - - ui_dir = Path(__file__).parent.parent / "ui" - build_dir = app_path / "static" / "ui" - - build_command = "node_modules/.bin/next build" - export_command = "node_modules/.bin/next export -o {f}".format(f=build_dir) - - all_messages = [] - for command in (build_command, export_command): - try: - proc = await asyncio.create_subprocess_shell( - cmd=command, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - cwd=ui_dir, - ) - - stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=timeout) - messages = stdout.decode("utf-8").strip() - errors = stderr.decode("utf-8").strip() - - if proc.returncode != 0: - raise RuntimeError(f"\nMessages:\n{messages}\nErrors:\n{errors}") - - await proc.wait() - all_messages.append(messages) - - except asyncio.TimeoutError: - raise RuntimeError(f"{timeout} second timeout exceeded while building UI") - - except Exception as err: - log.error(err) - raise RuntimeError(str(err)) - - return "\n".join(all_messages) - - async def write_env(variables: Dict) -> str: """Write environment variables to temporary JSON file.""" env_file = Path("/tmp/hyperglass.env.json") # noqa: S108 @@ -198,459 +80,6 @@ def sync_clear_redis_cache() -> None: raise RuntimeError from err -async def move_files(src, dst, files): # noqa: C901 - """Move iterable of files from source to destination. - - Arguments: - src {Path} -- Current directory of files - dst {Path} -- Target destination directory - files {Iterable} -- Iterable of files - """ - - # Standard Library - from typing import Iterable - - def error(*args, **kwargs): - msg = ", ".join(args) - kwargs = {k: str(v) for k, v in kwargs.items()} - error_msg = msg.format(**kwargs) - log.error(error_msg) - return RuntimeError(error_msg) - - if not isinstance(src, Path): - try: - src = Path(src) - except TypeError: - raise error("{p} is not a valid path", p=src) - - if not isinstance(dst, Path): - try: - dst = Path(dst) - except TypeError: - raise error("{p} is not a valid path", p=dst) - - if not isinstance(files, Iterable): - raise error( - "{fa} must be an iterable (list, tuple, or generator). Received {f}", - fa="Files argument", - f=files, - ) - - for path in (src, dst): - if not path.exists(): - raise error("{p} does not exist", p=path) - - migrated = () - - for file in files: - dst_file = dst / file.name - - if not file.exists(): - raise error("{f} does not exist", f=file) - - try: - if not dst_file.exists(): - shutil.copyfile(file, dst_file) - migrated += (str(dst_file),) - except Exception as e: - raise error("Failed to migrate {f}: {e}", f=dst_file, e=e) - - return migrated - - -async def check_node_modules(): - """Check if node_modules exists and has contents. - - Returns: - {bool} -- True if exists and has contents. - """ - - ui_path = Path(__file__).parent.parent / "ui" - node_modules = ui_path / "node_modules" - - exists = node_modules.exists() - valid = exists - - if exists and not tuple(node_modules.iterdir()): - valid = False - - return valid - - -async def node_initial(dev_mode=False): - """Initialize node_modules. - - Raises: - RuntimeError: Raised if exit code is not 0 - RuntimeError: Raised if other exceptions occur - - Returns: - {str} -- Command output - """ - - ui_path = Path(__file__).parent.parent / "ui" - - timeout = get_ui_build_timeout() - - all_messages = [] - try: - proc = await asyncio.create_subprocess_shell( - cmd="yarn --silent --emoji false", - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - cwd=ui_path, - ) - - stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=timeout) - messages = stdout.decode("utf-8").strip() - errors = stderr.decode("utf-8").strip() - - if proc.returncode != 0: - raise RuntimeError(f"\nMessages:\n{messages}\nErrors:\n{errors}") - - await proc.wait() - all_messages.append(messages) - - except Exception as e: - raise RuntimeError(str(e)) - - return "\n".join(all_messages) - - -async def read_package_json(): - """Import package.json as a python dict. - - Raises: - RuntimeError: Raised if unable to read package.json - - Returns: - {dict} -- NPM package.json as dict - """ - - # Standard Library - import json - - package_json_file = Path(__file__).parent.parent / "ui" / "package.json" - - try: - - with package_json_file.open("r") as file: - package_json = json.load(file) - - except Exception as e: - raise RuntimeError(f"Error reading package.json: {str(e)}") - - log.debug("package.json:\n{p}", p=package_json) - - return package_json - - -def generate_opengraph( - image_path: Path, - max_width: int, - max_height: int, - target_path: Path, - background_color: str, -): - """Generate an OpenGraph compliant image.""" - # Third Party - from PIL import Image - - def center_point(background: Image, foreground: Image): - """Generate a tuple of center points for PIL.""" - bg_x, bg_y = background.size[0:2] - fg_x, fg_y = foreground.size[0:2] - x1 = math.floor((bg_x / 2) - (fg_x / 2)) - y1 = math.floor((bg_y / 2) - (fg_y / 2)) - x2 = math.floor((bg_x / 2) + (fg_x / 2)) - y2 = math.floor((bg_y / 2) + (fg_y / 2)) - return (x1, y1, x2, y2) - - # Convert image to JPEG format with static name "opengraph.jpg" - dst_path = target_path / "opengraph.jpg" - - # Copy the original image to the target path - copied = shutil.copy2(image_path, target_path) - log.debug("Copied {} to {}", str(image_path), str(target_path)) - - with Image.open(copied) as src: - - # Only resize the image if it needs to be resized - if src.size[0] != max_width or src.size[1] != max_height: - - # Resize image while maintaining aspect ratio - log.debug("Opengraph image is not 1200x630, resizing...") - src.thumbnail((max_width, max_height)) - - # Only impose a background image if the original image has - # alpha/transparency channels - if src.mode in ("RGBA", "LA"): - log.debug("Opengraph image has transparency, converting...") - background = Image.new("RGB", (max_width, max_height), background_color) - background.paste(src, box=center_point(background, src)) - dst = background - else: - dst = src - - # Save new image to derived target path - dst.save(dst_path) - - # Delete the copied image - Path(copied).unlink() - - if not dst_path.exists(): - raise RuntimeError(f"Unable to save resized image to {str(dst_path)}") - - log.debug("Opengraph image ready at {}", str(dst_path)) - - return True - - -class FileCopy(Thread): - """Custom thread for copyfiles() function.""" - - def __init__(self, src: Path, dst: Path, queue: Queue): - """Initialize custom thread.""" - super().__init__() - - if not src.exists(): - raise ValueError("{} does not exist", str(src)) - - self.src = src - self.dst = dst - self.queue = queue - - def run(self): - """Put one object into the queue for each file.""" - try: - try: - shutil.copy(self.src, self.dst) - except IOError as err: - self.queue.put(err) - else: - self.queue.put(self.src) - finally: - pass - - -def copyfiles(src_files: Iterable[Path], dst_files: Iterable[Path]): - """Copy iterable of files from source to destination with threading.""" - queue = Queue() - threads = () - src_files_len = len(src_files) - dst_files_len = len(dst_files) - - if src_files_len != dst_files_len: - raise ValueError( - "The number of source files " - + "({}) must match the number of destination files ({}).".format( - src_files_len, dst_files_len - ) - ) - - for i, file in enumerate(src_files): - file_thread = FileCopy(src=file, dst=dst_files[i], queue=queue) - threads += (file_thread,) - - for thread in threads: - thread.start() - - for _file in src_files: - copied = queue.get() - log.debug("Copied {}", str(copied)) - - for thread in threads: - thread.join() - - for i, file in enumerate(dst_files): - if not file.exists(): - raise RuntimeError("{} was not copied to {}", str(src_files[i]), str(file)) - - return True - - -def migrate_images(app_path: Path, params: dict): - """Migrate images from source code to install directory.""" - images_dir = app_path / "static" / "images" - favicon_dir = images_dir / "favicons" - check_path(favicon_dir, create=True) - src_files = () - dst_files = () - - for image in ("light", "dark", "favicon"): - src = Path(params["web"]["logo"][image]) - dst = images_dir / f"{image + src.suffix}" - src_files += (src,) - dst_files += (dst,) - return copyfiles(src_files, dst_files) - - -async def build_frontend( # noqa: C901 - dev_mode: bool, - dev_url: str, - prod_url: str, - params: Dict, - app_path: Path, - force: bool = False, -) -> bool: - """Perform full frontend UI build process. - - Securely creates temporary file, writes frontend configuration - parameters to file as JSON. Then writes the name of the temporary - file to /tmp/hyperglass.env.json as {"configFile": }. - - Webpack reads /tmp/hyperglass.env.json, loads the temporary file, - and sets its contents to Node environment variables during the build - process. - - After the build is successful, the temporary file is automatically - closed during garbage collection. - - Arguments: - dev_mode {bool} -- Development Mode - dev_url {str} -- Development Mode URL - prod_url {str} -- Production Mode URL - params {dict} -- Frontend Config paramters - - Raises: - RuntimeError: Raised if errors occur during build process. - - Returns: - {bool} -- True if successful - """ - # Standard Library - import hashlib - import tempfile - - # Third Party - from favicons import Favicons - - # Project - from hyperglass.constants import __version__ - - env_file = Path("/tmp/hyperglass.env.json") # noqa: S108 - - package_json = await read_package_json() - - env_vars = { - "_HYPERGLASS_CONFIG_": params, - "_HYPERGLASS_VERSION_": __version__, - "_HYPERGLASS_PACKAGE_JSON_": package_json, - "_HYPERGLASS_APP_PATH_": str(app_path), - } - - # Set NextJS production/development mode and base URL based on - # developer_mode setting. - if dev_mode: - env_vars.update({"NODE_ENV": "development", "_HYPERGLASS_URL_": dev_url}) - - else: - env_vars.update({"NODE_ENV": "production", "_HYPERGLASS_URL_": prod_url}) - - # Check if hyperglass/ui/node_modules has been initialized. If not, - # initialize it. - initialized = await check_node_modules() - - if initialized: - log.debug("node_modules is already initialized") - - elif not initialized: - log.debug("node_modules has not been initialized. Starting initialization...") - - node_setup = await node_initial(dev_mode) - - if node_setup == "": - log.debug("Re-initialized node_modules") - - images_dir = app_path / "static" / "images" - favicon_dir = images_dir / "favicons" - - try: - if not favicon_dir.exists(): - favicon_dir.mkdir() - async with Favicons( - source=params["web"]["logo"]["favicon"], - output_directory=favicon_dir, - base_url="/images/favicons/", - ) as favicons: - await favicons.generate() - log.debug("Generated {} favicons", favicons.completed) - env_vars.update({"_HYPERGLASS_FAVICONS_": favicons.formats()}) - - env_json = json.dumps(env_vars, default=str) - - # Create SHA256 hash from all parameters passed to UI, use as - # build identifier. - build_id = hashlib.sha256(env_json.encode()).hexdigest() - - # Read hard-coded environment file from last build. If build ID - # matches this build's ID, don't run a new build. - if env_file.exists() and not force: - - with env_file.open("r") as ef: - ef_id = json.load(ef).get("buildId", "empty") - - log.debug("Previous Build ID: {id}", id=ef_id) - - if ef_id == build_id: - - log.debug( - "UI parameters unchanged since last build, skipping UI build..." - ) - - return True - - # Create temporary file. json file extension is added for easy - # webpack JSON parsing. - temp_file = tempfile.NamedTemporaryFile( - mode="w+", prefix="hyperglass_", suffix=".json", delete=not dev_mode - ) - - log.info("Starting UI build...") - log.debug( - f"Created temporary UI config file: '{temp_file.name}' for build {build_id}" - ) - - with Path(temp_file.name).open("w+") as temp: - temp.write(env_json) - - # Write "permanent" file (hard-coded named) for Node to read. - env_file.write_text( - json.dumps({"configFile": temp_file.name, "buildId": build_id}) - ) - - # While temporary file is still open, initiate UI build process. - if not dev_mode or force: - initialize_result = await node_initial(dev_mode) - build_result = await build_ui(app_path=app_path) - - if initialize_result: - log.debug(initialize_result) - elif initialize_result == "": - log.debug("Re-initialized node_modules") - - if build_result: - log.success("Completed UI build") - elif dev_mode and not force: - log.debug("Running in developer mode, did not build new UI files") - - migrate_images(app_path, params) - - generate_opengraph( - Path(params["web"]["opengraph"]["image"]), - 1200, - 630, - images_dir, - params["web"]["theme"]["colors"]["black"], - ) - - except Exception as err: - log.error(err) - raise RuntimeError(str(err)) from None - - return True - - def set_app_path(required: bool = False) -> Path: """Find app directory and set value to environment variable.""" @@ -699,22 +128,10 @@ to access the following directories: def format_listen_address(listen_address: Union[IPv4Address, IPv6Address, str]) -> str: - """Format a listen_address. - - Wraps IPv6 address in brackets. - - Arguments: - listen_address {str} -- Preformatted listen_address - - Returns: - {str} -- Formatted listen_address - """ + """Format a listen_address. Wraps IPv6 address in brackets.""" fmt = str(listen_address) if isinstance(listen_address, str): - # Standard Library - from ipaddress import ip_address - try: listen_address = ip_address(listen_address) except ValueError as err: @@ -734,7 +151,6 @@ def split_on_uppercase(s): """Split characters by uppercase letters. From: https://stackoverflow.com/a/40382663 - """ string_length = len(s) is_lower_around = ( @@ -811,8 +227,6 @@ def get_cache_env(): def make_repr(_class): """Create a user-friendly represention of an object.""" - # Standard Library - from asyncio import iscoroutine def _process_attrs(_dir): for attr in _dir: diff --git a/hyperglass/util/files.py b/hyperglass/util/files.py new file mode 100644 index 0000000..6356157 --- /dev/null +++ b/hyperglass/util/files.py @@ -0,0 +1,163 @@ +"""Utilities for working with files.""" + +# Standard Library +import shutil +from queue import Queue +from typing import List, Tuple, Union, Iterable, Optional, Generator +from pathlib import Path +from threading import Thread + +# Project +from hyperglass.log import log + + +async def move_files( # noqa: C901 + src: Path, dst: Path, files: Iterable[Path] +) -> Tuple[str]: + """Move iterable of files from source to destination. + + Arguments: + src {Path} -- Current directory of files + dst {Path} -- Target destination directory + files {Iterable} -- Iterable of files + """ + + def error(*args, **kwargs): + msg = ", ".join(args) + kwargs = {k: str(v) for k, v in kwargs.items()} + error_msg = msg.format(**kwargs) + log.error(error_msg) + return RuntimeError(error_msg) + + if not isinstance(src, Path): + try: + src = Path(src) + except TypeError: + raise error("{p} is not a valid path", p=src) + + if not isinstance(dst, Path): + try: + dst = Path(dst) + except TypeError: + raise error("{p} is not a valid path", p=dst) + + if not isinstance(files, (List, Tuple, Generator)): + raise error( + "{fa} must be an iterable (list, tuple, or generator). Received {f}", + fa="Files argument", + f=files, + ) + + for path in (src, dst): + if not path.exists(): + raise error("{p} does not exist", p=path) + + migrated = () + + for file in files: + dst_file = dst / file.name + + if not file.exists(): + raise error("{f} does not exist", f=file) + + try: + if not dst_file.exists(): + shutil.copyfile(file, dst_file) + migrated += (str(dst_file),) + except Exception as e: + raise error("Failed to migrate {f}: {e}", f=dst_file, e=e) + + return migrated + + +class FileCopy(Thread): + """Custom thread for copyfiles() function.""" + + def __init__(self, src: Path, dst: Path, queue: Queue): + """Initialize custom thread.""" + super().__init__() + + if not src.exists(): + raise ValueError("{} does not exist", str(src)) + + self.src = src + self.dst = dst + self.queue = queue + + def run(self): + """Put one object into the queue for each file.""" + try: + try: + shutil.copy(self.src, self.dst) + except IOError as err: + self.queue.put(err) + else: + self.queue.put(self.src) + finally: + pass + + +def copyfiles(src_files: Iterable[Path], dst_files: Iterable[Path]): + """Copy iterable of files from source to destination with threading.""" + queue = Queue() + threads = () + src_files_len = len(src_files) + dst_files_len = len(dst_files) + + if src_files_len != dst_files_len: + raise ValueError( + "The number of source files " + + "({}) must match the number of destination files ({}).".format( + src_files_len, dst_files_len + ) + ) + + for i, file in enumerate(src_files): + file_thread = FileCopy(src=file, dst=dst_files[i], queue=queue) + threads += (file_thread,) + + for thread in threads: + thread.start() + + for _file in src_files: + copied = queue.get() + log.debug("Copied {}", str(copied)) + + for thread in threads: + thread.join() + + for i, file in enumerate(dst_files): + if not file.exists(): + raise RuntimeError("{} was not copied to {}", str(src_files[i]), str(file)) + + return True + + +def check_path( + path: Union[Path, str], mode: str = "r", create: bool = False +) -> Optional[Path]: + """Verify if a path exists and is accessible.""" + + result = None + + try: + if not isinstance(path, Path): + path = Path(path) + + if not path.exists(): + if create: + if path.is_file(): + path.parent.mkdir(parents=True) + else: + path.mkdir(parents=True) + else: + raise FileNotFoundError(f"{str(path)} does not exist.") + + if path.exists(): + with path.open(mode): + result = path + + except Exception: # noqa: S110 + pass + + return result diff --git a/hyperglass/util/frontend.py b/hyperglass/util/frontend.py index 2ce792c..130bb6f 100644 --- a/hyperglass/util/frontend.py +++ b/hyperglass/util/frontend.py @@ -1,8 +1,20 @@ """Utility functions for frontend-related tasks.""" # Standard Library +import os +import json +import math import shutil +import asyncio import subprocess +from typing import Dict, Optional +from pathlib import Path + +# Project +from hyperglass.log import log + +# Local +from .files import copyfiles, check_path def get_node_version() -> int: @@ -19,3 +31,376 @@ def get_node_version() -> int: major, minor, patch = version.split(".") return int(major) + + +def get_ui_build_timeout() -> Optional[int]: + """Read the UI build timeout from environment variables or set a default.""" + timeout = None + + if "HYPERGLASS_UI_BUILD_TIMEOUT" in os.environ: + timeout = int(os.environ["HYPERGLASS_UI_BUILD_TIMEOUT"]) + log.info("Found UI build timeout environment variable: {}", timeout) + + elif "POETRY_HYPERGLASS_UI_BUILD_TIMEOUT" in os.environ: + timeout = int(os.environ["POETRY_HYPERGLASS_UI_BUILD_TIMEOUT"]) + log.info("Found UI build timeout environment variable: {}", timeout) + + return timeout + + +async def check_node_modules() -> bool: + """Check if node_modules exists and has contents.""" + + ui_path = Path(__file__).parent.parent / "ui" + node_modules = ui_path / "node_modules" + + exists = node_modules.exists() + valid = exists + + if exists and not tuple(node_modules.iterdir()): + valid = False + + return valid + + +async def read_package_json() -> Dict: + """Import package.json as a python dict.""" + + package_json_file = Path(__file__).parent.parent / "ui" / "package.json" + + try: + + with package_json_file.open("r") as file: + package_json = json.load(file) + + except Exception as e: + raise RuntimeError(f"Error reading package.json: {str(e)}") + + log.debug("package.json:\n{p}", p=package_json) + + return package_json + + +async def node_initial(timeout: int = 180, dev_mode: bool = False) -> str: + """Initialize node_modules.""" + + ui_path = Path(__file__).parent.parent / "ui" + + env_timeout = get_ui_build_timeout() + + if env_timeout is not None and env_timeout > timeout: + timeout = env_timeout + + all_messages = () + + try: + proc = await asyncio.create_subprocess_shell( + cmd="yarn --silent --emoji false", + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + cwd=ui_path, + ) + + stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=timeout) + messages = stdout.decode("utf-8").strip() + errors = stderr.decode("utf-8").strip() + + if proc.returncode != 0: + raise RuntimeError(f"\nMessages:\n{messages}\nErrors:\n{errors}") + + await proc.wait() + all_messages += (messages,) + + except Exception as e: + raise RuntimeError(str(e)) + + return "\n".join(all_messages) + + +async def build_ui(app_path): + """Execute `next build` & `next export` from UI directory. + + Raises: + RuntimeError: Raised if exit code is not 0. + RuntimeError: Raised when any other error occurs. + """ + timeout = get_ui_build_timeout() + + ui_dir = Path(__file__).parent.parent / "ui" + build_dir = app_path / "static" / "ui" + + build_command = "node_modules/.bin/next build" + export_command = "node_modules/.bin/next export -o {f}".format(f=build_dir) + + all_messages = [] + for command in (build_command, export_command): + try: + proc = await asyncio.create_subprocess_shell( + cmd=command, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + cwd=ui_dir, + ) + + stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=timeout) + messages = stdout.decode("utf-8").strip() + errors = stderr.decode("utf-8").strip() + + if proc.returncode != 0: + raise RuntimeError(f"\nMessages:\n{messages}\nErrors:\n{errors}") + + await proc.wait() + all_messages.append(messages) + + except asyncio.TimeoutError: + raise RuntimeError(f"{timeout} second timeout exceeded while building UI") + + except Exception as err: + log.error(err) + raise RuntimeError(str(err)) + + return "\n".join(all_messages) + + +def generate_opengraph( + image_path: Path, + max_width: int, + max_height: int, + target_path: Path, + background_color: str, +): + """Generate an OpenGraph compliant image.""" + # Third Party + from PIL import Image + + def center_point(background: Image, foreground: Image): + """Generate a tuple of center points for PIL.""" + bg_x, bg_y = background.size[0:2] + fg_x, fg_y = foreground.size[0:2] + x1 = math.floor((bg_x / 2) - (fg_x / 2)) + y1 = math.floor((bg_y / 2) - (fg_y / 2)) + x2 = math.floor((bg_x / 2) + (fg_x / 2)) + y2 = math.floor((bg_y / 2) + (fg_y / 2)) + return (x1, y1, x2, y2) + + # Convert image to JPEG format with static name "opengraph.jpg" + dst_path = target_path / "opengraph.jpg" + + # Copy the original image to the target path + copied = shutil.copy2(image_path, target_path) + log.debug("Copied {} to {}", str(image_path), str(target_path)) + + with Image.open(copied) as src: + + # Only resize the image if it needs to be resized + if src.size[0] != max_width or src.size[1] != max_height: + + # Resize image while maintaining aspect ratio + log.debug("Opengraph image is not 1200x630, resizing...") + src.thumbnail((max_width, max_height)) + + # Only impose a background image if the original image has + # alpha/transparency channels + if src.mode in ("RGBA", "LA"): + log.debug("Opengraph image has transparency, converting...") + background = Image.new("RGB", (max_width, max_height), background_color) + background.paste(src, box=center_point(background, src)) + dst = background + else: + dst = src + + # Save new image to derived target path + dst.save(dst_path) + + # Delete the copied image + Path(copied).unlink() + + if not dst_path.exists(): + raise RuntimeError(f"Unable to save resized image to {str(dst_path)}") + + log.debug("Opengraph image ready at {}", str(dst_path)) + + return True + + +def migrate_images(app_path: Path, params: dict): + """Migrate images from source code to install directory.""" + images_dir = app_path / "static" / "images" + favicon_dir = images_dir / "favicons" + check_path(favicon_dir, create=True) + src_files = () + dst_files = () + + for image in ("light", "dark", "favicon"): + src = Path(params["web"]["logo"][image]) + dst = images_dir / f"{image + src.suffix}" + src_files += (src,) + dst_files += (dst,) + return copyfiles(src_files, dst_files) + + +async def build_frontend( # noqa: C901 + dev_mode: bool, + dev_url: str, + prod_url: str, + params: Dict, + app_path: Path, + force: bool = False, + timeout: int = 180, +) -> bool: + """Perform full frontend UI build process. + + Securely creates temporary file, writes frontend configuration + parameters to file as JSON. Then writes the name of the temporary + file to /tmp/hyperglass.env.json as {"configFile": }. + + Webpack reads /tmp/hyperglass.env.json, loads the temporary file, + and sets its contents to Node environment variables during the build + process. + + After the build is successful, the temporary file is automatically + closed during garbage collection. + + Arguments: + dev_mode {bool} -- Development Mode + dev_url {str} -- Development Mode URL + prod_url {str} -- Production Mode URL + params {dict} -- Frontend Config paramters + + Raises: + RuntimeError: Raised if errors occur during build process. + + Returns: + {bool} -- True if successful + """ + # Standard Library + import hashlib + import tempfile + + # Third Party + from favicons import Favicons + + # Project + from hyperglass.constants import __version__ + + env_file = Path("/tmp/hyperglass.env.json") # noqa: S108 + + package_json = await read_package_json() + + env_vars = { + "_HYPERGLASS_CONFIG_": params, + "_HYPERGLASS_VERSION_": __version__, + "_HYPERGLASS_PACKAGE_JSON_": package_json, + "_HYPERGLASS_APP_PATH_": str(app_path), + } + + # Set NextJS production/development mode and base URL based on + # developer_mode setting. + if dev_mode: + env_vars.update({"NODE_ENV": "development", "_HYPERGLASS_URL_": dev_url}) + + else: + env_vars.update({"NODE_ENV": "production", "_HYPERGLASS_URL_": prod_url}) + + # Check if hyperglass/ui/node_modules has been initialized. If not, + # initialize it. + initialized = await check_node_modules() + + if initialized: + log.debug("node_modules is already initialized") + + elif not initialized: + log.debug("node_modules has not been initialized. Starting initialization...") + + node_setup = await node_initial(timeout, dev_mode) + + if node_setup == "": + log.debug("Re-initialized node_modules") + + images_dir = app_path / "static" / "images" + favicon_dir = images_dir / "favicons" + + try: + if not favicon_dir.exists(): + favicon_dir.mkdir() + async with Favicons( + source=params["web"]["logo"]["favicon"], + output_directory=favicon_dir, + base_url="/images/favicons/", + ) as favicons: + await favicons.generate() + log.debug("Generated {} favicons", favicons.completed) + env_vars.update({"_HYPERGLASS_FAVICONS_": favicons.formats()}) + + env_json = json.dumps(env_vars, default=str) + + # Create SHA256 hash from all parameters passed to UI, use as + # build identifier. + build_id = hashlib.sha256(env_json.encode()).hexdigest() + + # Read hard-coded environment file from last build. If build ID + # matches this build's ID, don't run a new build. + if env_file.exists() and not force: + + with env_file.open("r") as ef: + ef_id = json.load(ef).get("buildId", "empty") + + log.debug("Previous Build ID: {id}", id=ef_id) + + if ef_id == build_id: + + log.debug( + "UI parameters unchanged since last build, skipping UI build..." + ) + + return True + + # Create temporary file. json file extension is added for easy + # webpack JSON parsing. + temp_file = tempfile.NamedTemporaryFile( + mode="w+", prefix="hyperglass_", suffix=".json", delete=not dev_mode + ) + + log.info("Starting UI build...") + log.debug( + f"Created temporary UI config file: '{temp_file.name}' for build {build_id}" + ) + + with Path(temp_file.name).open("w+") as temp: + temp.write(env_json) + + # Write "permanent" file (hard-coded named) for Node to read. + env_file.write_text( + json.dumps({"configFile": temp_file.name, "buildId": build_id}) + ) + + # While temporary file is still open, initiate UI build process. + if not dev_mode or force: + initialize_result = await node_initial(timeout, dev_mode) + build_result = await build_ui(app_path=app_path) + + if initialize_result: + log.debug(initialize_result) + elif initialize_result == "": + log.debug("Re-initialized node_modules") + + if build_result: + log.success("Completed UI build") + elif dev_mode and not force: + log.debug("Running in developer mode, did not build new UI files") + + migrate_images(app_path, params) + + generate_opengraph( + Path(params["web"]["opengraph"]["image"]), + 1200, + 630, + images_dir, + params["web"]["theme"]["colors"]["black"], + ) + + except Exception as err: + log.error(err) + raise RuntimeError(str(err)) from None + + return True