mirror of
https://github.com/checktheroads/hyperglass
synced 2024-05-11 05:55:08 +00:00
restructure util module
This commit is contained in:
checktheroads
@@ -17,13 +17,14 @@ from hyperglass.log import (
|
||||
enable_file_logging,
|
||||
enable_syslog_logging,
|
||||
)
|
||||
from hyperglass.util import check_path, set_app_path, set_cache_env, current_log_level
|
||||
from hyperglass.util import set_app_path, set_cache_env, current_log_level
|
||||
from hyperglass.constants import (
|
||||
SUPPORTED_QUERY_TYPES,
|
||||
PARSED_RESPONSE_FIELDS,
|
||||
__version__,
|
||||
)
|
||||
from hyperglass.exceptions import ConfigError, ConfigMissing
|
||||
from hyperglass.util.files import check_path
|
||||
from hyperglass.models.commands import Commands
|
||||
from hyperglass.models.config.params import Params
|
||||
from hyperglass.models.config.devices import Devices
|
||||
|
||||
@@ -29,26 +29,21 @@ if node_version != MIN_NODE_VERSION:
|
||||
raise RuntimeError(f"NodeJS {MIN_NODE_VERSION}+ is required.")
|
||||
|
||||
|
||||
# Local
|
||||
from .cache import SyncCache
|
||||
# Project
|
||||
from hyperglass.compat._asyncio import aiorun
|
||||
|
||||
from .configuration import ( # isort:skip
|
||||
params,
|
||||
# Local
|
||||
from .util import cpu_count, clear_redis_cache, format_listen_address
|
||||
from .cache import SyncCache
|
||||
from .configuration import (
|
||||
URL_DEV,
|
||||
URL_PROD,
|
||||
CONFIG_PATH,
|
||||
REDIS_CONFIG,
|
||||
params,
|
||||
frontend_params,
|
||||
)
|
||||
from .util import ( # isort:skip
|
||||
cpu_count,
|
||||
build_frontend,
|
||||
clear_redis_cache,
|
||||
format_listen_address,
|
||||
)
|
||||
|
||||
|
||||
from hyperglass.compat._asyncio import aiorun # isort:skip
|
||||
from .util.frontend import build_frontend
|
||||
|
||||
if params.debug:
|
||||
workers = 1
|
||||
@@ -84,12 +79,8 @@ def check_redis_instance() -> bool:
|
||||
return True
|
||||
|
||||
|
||||
async def build_ui():
|
||||
"""Perform a UI build prior to starting the application.
|
||||
|
||||
Returns:
|
||||
{bool} -- True if successful.
|
||||
"""
|
||||
async def build_ui() -> bool:
|
||||
"""Perform a UI build prior to starting the application."""
|
||||
await build_frontend(
|
||||
dev_mode=params.developer_mode,
|
||||
dev_url=URL_DEV,
|
||||
@@ -109,7 +100,7 @@ async def clear_cache():
|
||||
pass
|
||||
|
||||
|
||||
def cache_config():
|
||||
def cache_config() -> bool:
|
||||
"""Add configuration to Redis cache as a pickled object."""
|
||||
# Standard Library
|
||||
import pickle
|
||||
|
||||
@@ -2,33 +2,26 @@
|
||||
|
||||
# Standard Library
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import json
|
||||
import math
|
||||
import shutil
|
||||
import asyncio
|
||||
import platform
|
||||
from queue import Queue
|
||||
from typing import Dict, Union, Iterable, Optional, Generator
|
||||
from typing import Dict, Union, Generator
|
||||
from asyncio import iscoroutine
|
||||
from pathlib import Path
|
||||
from ipaddress import IPv4Address, IPv6Address, ip_address
|
||||
from threading import Thread
|
||||
|
||||
# Third Party
|
||||
from loguru._logger import Logger as LoguruLogger
|
||||
|
||||
# Project
|
||||
from hyperglass.log import log
|
||||
from hyperglass.cache import AsyncCache
|
||||
from hyperglass.models import HyperglassModel
|
||||
|
||||
|
||||
def cpu_count(multiplier: int = 0):
|
||||
def cpu_count(multiplier: int = 0) -> int:
|
||||
"""Get server's CPU core count.
|
||||
|
||||
Used for number of web server workers.
|
||||
|
||||
Returns:
|
||||
{int} -- CPU Cores
|
||||
Used to determine the number of web server workers.
|
||||
"""
|
||||
# Standard Library
|
||||
import multiprocessing
|
||||
@@ -36,58 +29,8 @@ def cpu_count(multiplier: int = 0):
|
||||
return multiprocessing.cpu_count() * multiplier
|
||||
|
||||
|
||||
def check_path(
|
||||
path: Union[Path, str], mode: str = "r", create: bool = False
|
||||
) -> Optional[Path]:
|
||||
"""Verify if a path exists and is accessible.
|
||||
|
||||
Arguments:
|
||||
path {Path|str} -- Path object or string of path
|
||||
mode {str} -- File mode, r or w
|
||||
|
||||
Raises:
|
||||
RuntimeError: Raised if file does not exist or is not accessible
|
||||
|
||||
Returns:
|
||||
{Path|None} -- Path object if checks pass, None if not.
|
||||
"""
|
||||
|
||||
try:
|
||||
if not isinstance(path, Path):
|
||||
path = Path(path)
|
||||
|
||||
if not path.exists():
|
||||
if create:
|
||||
if path.is_file():
|
||||
path.parent.mkdir(parents=True)
|
||||
else:
|
||||
path.mkdir(parents=True)
|
||||
else:
|
||||
raise FileNotFoundError(f"{str(path)} does not exist.")
|
||||
|
||||
if path.exists():
|
||||
with path.open(mode):
|
||||
result = path
|
||||
|
||||
except Exception:
|
||||
result = None
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def check_python() -> str:
|
||||
"""Verify Python Version.
|
||||
|
||||
Raises:
|
||||
RuntimeError: Raised if running Python version is invalid.
|
||||
|
||||
Returns:
|
||||
{str} -- Python version
|
||||
"""
|
||||
# Standard Library
|
||||
import sys
|
||||
import platform
|
||||
|
||||
"""Verify Python Version."""
|
||||
# Project
|
||||
from hyperglass.constants import MIN_PYTHON_VERSION
|
||||
|
||||
@@ -97,67 +40,6 @@ def check_python() -> str:
|
||||
return platform.python_version()
|
||||
|
||||
|
||||
def get_ui_build_timeout() -> int:
|
||||
"""Read the UI build timeout from environment variables or set a default."""
|
||||
|
||||
timeout = 90
|
||||
|
||||
if "HYPERGLASS_UI_BUILD_TIMEOUT" in os.environ:
|
||||
timeout = int(os.environ["HYPERGLASS_UI_BUILD_TIMEOUT"])
|
||||
log.info("Found UI build timeout environment variable: {}", timeout)
|
||||
|
||||
elif "POETRY_HYPERGLASS_UI_BUILD_TIMEOUT" in os.environ:
|
||||
timeout = int(os.environ["POETRY_HYPERGLASS_UI_BUILD_TIMEOUT"])
|
||||
log.info("Found UI build timeout environment variable: {}", timeout)
|
||||
|
||||
return timeout
|
||||
|
||||
|
||||
async def build_ui(app_path):
|
||||
"""Execute `next build` & `next export` from UI directory.
|
||||
|
||||
Raises:
|
||||
RuntimeError: Raised if exit code is not 0.
|
||||
RuntimeError: Raised when any other error occurs.
|
||||
"""
|
||||
timeout = get_ui_build_timeout()
|
||||
|
||||
ui_dir = Path(__file__).parent.parent / "ui"
|
||||
build_dir = app_path / "static" / "ui"
|
||||
|
||||
build_command = "node_modules/.bin/next build"
|
||||
export_command = "node_modules/.bin/next export -o {f}".format(f=build_dir)
|
||||
|
||||
all_messages = []
|
||||
for command in (build_command, export_command):
|
||||
try:
|
||||
proc = await asyncio.create_subprocess_shell(
|
||||
cmd=command,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
cwd=ui_dir,
|
||||
)
|
||||
|
||||
stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=timeout)
|
||||
messages = stdout.decode("utf-8").strip()
|
||||
errors = stderr.decode("utf-8").strip()
|
||||
|
||||
if proc.returncode != 0:
|
||||
raise RuntimeError(f"\nMessages:\n{messages}\nErrors:\n{errors}")
|
||||
|
||||
await proc.wait()
|
||||
all_messages.append(messages)
|
||||
|
||||
except asyncio.TimeoutError:
|
||||
raise RuntimeError(f"{timeout} second timeout exceeded while building UI")
|
||||
|
||||
except Exception as err:
|
||||
log.error(err)
|
||||
raise RuntimeError(str(err))
|
||||
|
||||
return "\n".join(all_messages)
|
||||
|
||||
|
||||
async def write_env(variables: Dict) -> str:
|
||||
"""Write environment variables to temporary JSON file."""
|
||||
env_file = Path("/tmp/hyperglass.env.json") # noqa: S108
|
||||
@@ -198,459 +80,6 @@ def sync_clear_redis_cache() -> None:
|
||||
raise RuntimeError from err
|
||||
|
||||
|
||||
async def move_files(src, dst, files): # noqa: C901
|
||||
"""Move iterable of files from source to destination.
|
||||
|
||||
Arguments:
|
||||
src {Path} -- Current directory of files
|
||||
dst {Path} -- Target destination directory
|
||||
files {Iterable} -- Iterable of files
|
||||
"""
|
||||
|
||||
# Standard Library
|
||||
from typing import Iterable
|
||||
|
||||
def error(*args, **kwargs):
|
||||
msg = ", ".join(args)
|
||||
kwargs = {k: str(v) for k, v in kwargs.items()}
|
||||
error_msg = msg.format(**kwargs)
|
||||
log.error(error_msg)
|
||||
return RuntimeError(error_msg)
|
||||
|
||||
if not isinstance(src, Path):
|
||||
try:
|
||||
src = Path(src)
|
||||
except TypeError:
|
||||
raise error("{p} is not a valid path", p=src)
|
||||
|
||||
if not isinstance(dst, Path):
|
||||
try:
|
||||
dst = Path(dst)
|
||||
except TypeError:
|
||||
raise error("{p} is not a valid path", p=dst)
|
||||
|
||||
if not isinstance(files, Iterable):
|
||||
raise error(
|
||||
"{fa} must be an iterable (list, tuple, or generator). Received {f}",
|
||||
fa="Files argument",
|
||||
f=files,
|
||||
)
|
||||
|
||||
for path in (src, dst):
|
||||
if not path.exists():
|
||||
raise error("{p} does not exist", p=path)
|
||||
|
||||
migrated = ()
|
||||
|
||||
for file in files:
|
||||
dst_file = dst / file.name
|
||||
|
||||
if not file.exists():
|
||||
raise error("{f} does not exist", f=file)
|
||||
|
||||
try:
|
||||
if not dst_file.exists():
|
||||
shutil.copyfile(file, dst_file)
|
||||
migrated += (str(dst_file),)
|
||||
except Exception as e:
|
||||
raise error("Failed to migrate {f}: {e}", f=dst_file, e=e)
|
||||
|
||||
return migrated
|
||||
|
||||
|
||||
async def check_node_modules():
|
||||
"""Check if node_modules exists and has contents.
|
||||
|
||||
Returns:
|
||||
{bool} -- True if exists and has contents.
|
||||
"""
|
||||
|
||||
ui_path = Path(__file__).parent.parent / "ui"
|
||||
node_modules = ui_path / "node_modules"
|
||||
|
||||
exists = node_modules.exists()
|
||||
valid = exists
|
||||
|
||||
if exists and not tuple(node_modules.iterdir()):
|
||||
valid = False
|
||||
|
||||
return valid
|
||||
|
||||
|
||||
async def node_initial(dev_mode=False):
|
||||
"""Initialize node_modules.
|
||||
|
||||
Raises:
|
||||
RuntimeError: Raised if exit code is not 0
|
||||
RuntimeError: Raised if other exceptions occur
|
||||
|
||||
Returns:
|
||||
{str} -- Command output
|
||||
"""
|
||||
|
||||
ui_path = Path(__file__).parent.parent / "ui"
|
||||
|
||||
timeout = get_ui_build_timeout()
|
||||
|
||||
all_messages = []
|
||||
try:
|
||||
proc = await asyncio.create_subprocess_shell(
|
||||
cmd="yarn --silent --emoji false",
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
cwd=ui_path,
|
||||
)
|
||||
|
||||
stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=timeout)
|
||||
messages = stdout.decode("utf-8").strip()
|
||||
errors = stderr.decode("utf-8").strip()
|
||||
|
||||
if proc.returncode != 0:
|
||||
raise RuntimeError(f"\nMessages:\n{messages}\nErrors:\n{errors}")
|
||||
|
||||
await proc.wait()
|
||||
all_messages.append(messages)
|
||||
|
||||
except Exception as e:
|
||||
raise RuntimeError(str(e))
|
||||
|
||||
return "\n".join(all_messages)
|
||||
|
||||
|
||||
async def read_package_json():
|
||||
"""Import package.json as a python dict.
|
||||
|
||||
Raises:
|
||||
RuntimeError: Raised if unable to read package.json
|
||||
|
||||
Returns:
|
||||
{dict} -- NPM package.json as dict
|
||||
"""
|
||||
|
||||
# Standard Library
|
||||
import json
|
||||
|
||||
package_json_file = Path(__file__).parent.parent / "ui" / "package.json"
|
||||
|
||||
try:
|
||||
|
||||
with package_json_file.open("r") as file:
|
||||
package_json = json.load(file)
|
||||
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"Error reading package.json: {str(e)}")
|
||||
|
||||
log.debug("package.json:\n{p}", p=package_json)
|
||||
|
||||
return package_json
|
||||
|
||||
|
||||
def generate_opengraph(
|
||||
image_path: Path,
|
||||
max_width: int,
|
||||
max_height: int,
|
||||
target_path: Path,
|
||||
background_color: str,
|
||||
):
|
||||
"""Generate an OpenGraph compliant image."""
|
||||
# Third Party
|
||||
from PIL import Image
|
||||
|
||||
def center_point(background: Image, foreground: Image):
|
||||
"""Generate a tuple of center points for PIL."""
|
||||
bg_x, bg_y = background.size[0:2]
|
||||
fg_x, fg_y = foreground.size[0:2]
|
||||
x1 = math.floor((bg_x / 2) - (fg_x / 2))
|
||||
y1 = math.floor((bg_y / 2) - (fg_y / 2))
|
||||
x2 = math.floor((bg_x / 2) + (fg_x / 2))
|
||||
y2 = math.floor((bg_y / 2) + (fg_y / 2))
|
||||
return (x1, y1, x2, y2)
|
||||
|
||||
# Convert image to JPEG format with static name "opengraph.jpg"
|
||||
dst_path = target_path / "opengraph.jpg"
|
||||
|
||||
# Copy the original image to the target path
|
||||
copied = shutil.copy2(image_path, target_path)
|
||||
log.debug("Copied {} to {}", str(image_path), str(target_path))
|
||||
|
||||
with Image.open(copied) as src:
|
||||
|
||||
# Only resize the image if it needs to be resized
|
||||
if src.size[0] != max_width or src.size[1] != max_height:
|
||||
|
||||
# Resize image while maintaining aspect ratio
|
||||
log.debug("Opengraph image is not 1200x630, resizing...")
|
||||
src.thumbnail((max_width, max_height))
|
||||
|
||||
# Only impose a background image if the original image has
|
||||
# alpha/transparency channels
|
||||
if src.mode in ("RGBA", "LA"):
|
||||
log.debug("Opengraph image has transparency, converting...")
|
||||
background = Image.new("RGB", (max_width, max_height), background_color)
|
||||
background.paste(src, box=center_point(background, src))
|
||||
dst = background
|
||||
else:
|
||||
dst = src
|
||||
|
||||
# Save new image to derived target path
|
||||
dst.save(dst_path)
|
||||
|
||||
# Delete the copied image
|
||||
Path(copied).unlink()
|
||||
|
||||
if not dst_path.exists():
|
||||
raise RuntimeError(f"Unable to save resized image to {str(dst_path)}")
|
||||
|
||||
log.debug("Opengraph image ready at {}", str(dst_path))
|
||||
|
||||
return True
|
||||
|
||||
|
||||
class FileCopy(Thread):
|
||||
"""Custom thread for copyfiles() function."""
|
||||
|
||||
def __init__(self, src: Path, dst: Path, queue: Queue):
|
||||
"""Initialize custom thread."""
|
||||
super().__init__()
|
||||
|
||||
if not src.exists():
|
||||
raise ValueError("{} does not exist", str(src))
|
||||
|
||||
self.src = src
|
||||
self.dst = dst
|
||||
self.queue = queue
|
||||
|
||||
def run(self):
|
||||
"""Put one object into the queue for each file."""
|
||||
try:
|
||||
try:
|
||||
shutil.copy(self.src, self.dst)
|
||||
except IOError as err:
|
||||
self.queue.put(err)
|
||||
else:
|
||||
self.queue.put(self.src)
|
||||
finally:
|
||||
pass
|
||||
|
||||
|
||||
def copyfiles(src_files: Iterable[Path], dst_files: Iterable[Path]):
|
||||
"""Copy iterable of files from source to destination with threading."""
|
||||
queue = Queue()
|
||||
threads = ()
|
||||
src_files_len = len(src_files)
|
||||
dst_files_len = len(dst_files)
|
||||
|
||||
if src_files_len != dst_files_len:
|
||||
raise ValueError(
|
||||
"The number of source files "
|
||||
+ "({}) must match the number of destination files ({}).".format(
|
||||
src_files_len, dst_files_len
|
||||
)
|
||||
)
|
||||
|
||||
for i, file in enumerate(src_files):
|
||||
file_thread = FileCopy(src=file, dst=dst_files[i], queue=queue)
|
||||
threads += (file_thread,)
|
||||
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
|
||||
for _file in src_files:
|
||||
copied = queue.get()
|
||||
log.debug("Copied {}", str(copied))
|
||||
|
||||
for thread in threads:
|
||||
thread.join()
|
||||
|
||||
for i, file in enumerate(dst_files):
|
||||
if not file.exists():
|
||||
raise RuntimeError("{} was not copied to {}", str(src_files[i]), str(file))
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def migrate_images(app_path: Path, params: dict):
|
||||
"""Migrate images from source code to install directory."""
|
||||
images_dir = app_path / "static" / "images"
|
||||
favicon_dir = images_dir / "favicons"
|
||||
check_path(favicon_dir, create=True)
|
||||
src_files = ()
|
||||
dst_files = ()
|
||||
|
||||
for image in ("light", "dark", "favicon"):
|
||||
src = Path(params["web"]["logo"][image])
|
||||
dst = images_dir / f"{image + src.suffix}"
|
||||
src_files += (src,)
|
||||
dst_files += (dst,)
|
||||
return copyfiles(src_files, dst_files)
|
||||
|
||||
|
||||
async def build_frontend( # noqa: C901
|
||||
dev_mode: bool,
|
||||
dev_url: str,
|
||||
prod_url: str,
|
||||
params: Dict,
|
||||
app_path: Path,
|
||||
force: bool = False,
|
||||
) -> bool:
|
||||
"""Perform full frontend UI build process.
|
||||
|
||||
Securely creates temporary file, writes frontend configuration
|
||||
parameters to file as JSON. Then writes the name of the temporary
|
||||
file to /tmp/hyperglass.env.json as {"configFile": <file_name> }.
|
||||
|
||||
Webpack reads /tmp/hyperglass.env.json, loads the temporary file,
|
||||
and sets its contents to Node environment variables during the build
|
||||
process.
|
||||
|
||||
After the build is successful, the temporary file is automatically
|
||||
closed during garbage collection.
|
||||
|
||||
Arguments:
|
||||
dev_mode {bool} -- Development Mode
|
||||
dev_url {str} -- Development Mode URL
|
||||
prod_url {str} -- Production Mode URL
|
||||
params {dict} -- Frontend Config paramters
|
||||
|
||||
Raises:
|
||||
RuntimeError: Raised if errors occur during build process.
|
||||
|
||||
Returns:
|
||||
{bool} -- True if successful
|
||||
"""
|
||||
# Standard Library
|
||||
import hashlib
|
||||
import tempfile
|
||||
|
||||
# Third Party
|
||||
from favicons import Favicons
|
||||
|
||||
# Project
|
||||
from hyperglass.constants import __version__
|
||||
|
||||
env_file = Path("/tmp/hyperglass.env.json") # noqa: S108
|
||||
|
||||
package_json = await read_package_json()
|
||||
|
||||
env_vars = {
|
||||
"_HYPERGLASS_CONFIG_": params,
|
||||
"_HYPERGLASS_VERSION_": __version__,
|
||||
"_HYPERGLASS_PACKAGE_JSON_": package_json,
|
||||
"_HYPERGLASS_APP_PATH_": str(app_path),
|
||||
}
|
||||
|
||||
# Set NextJS production/development mode and base URL based on
|
||||
# developer_mode setting.
|
||||
if dev_mode:
|
||||
env_vars.update({"NODE_ENV": "development", "_HYPERGLASS_URL_": dev_url})
|
||||
|
||||
else:
|
||||
env_vars.update({"NODE_ENV": "production", "_HYPERGLASS_URL_": prod_url})
|
||||
|
||||
# Check if hyperglass/ui/node_modules has been initialized. If not,
|
||||
# initialize it.
|
||||
initialized = await check_node_modules()
|
||||
|
||||
if initialized:
|
||||
log.debug("node_modules is already initialized")
|
||||
|
||||
elif not initialized:
|
||||
log.debug("node_modules has not been initialized. Starting initialization...")
|
||||
|
||||
node_setup = await node_initial(dev_mode)
|
||||
|
||||
if node_setup == "":
|
||||
log.debug("Re-initialized node_modules")
|
||||
|
||||
images_dir = app_path / "static" / "images"
|
||||
favicon_dir = images_dir / "favicons"
|
||||
|
||||
try:
|
||||
if not favicon_dir.exists():
|
||||
favicon_dir.mkdir()
|
||||
async with Favicons(
|
||||
source=params["web"]["logo"]["favicon"],
|
||||
output_directory=favicon_dir,
|
||||
base_url="/images/favicons/",
|
||||
) as favicons:
|
||||
await favicons.generate()
|
||||
log.debug("Generated {} favicons", favicons.completed)
|
||||
env_vars.update({"_HYPERGLASS_FAVICONS_": favicons.formats()})
|
||||
|
||||
env_json = json.dumps(env_vars, default=str)
|
||||
|
||||
# Create SHA256 hash from all parameters passed to UI, use as
|
||||
# build identifier.
|
||||
build_id = hashlib.sha256(env_json.encode()).hexdigest()
|
||||
|
||||
# Read hard-coded environment file from last build. If build ID
|
||||
# matches this build's ID, don't run a new build.
|
||||
if env_file.exists() and not force:
|
||||
|
||||
with env_file.open("r") as ef:
|
||||
ef_id = json.load(ef).get("buildId", "empty")
|
||||
|
||||
log.debug("Previous Build ID: {id}", id=ef_id)
|
||||
|
||||
if ef_id == build_id:
|
||||
|
||||
log.debug(
|
||||
"UI parameters unchanged since last build, skipping UI build..."
|
||||
)
|
||||
|
||||
return True
|
||||
|
||||
# Create temporary file. json file extension is added for easy
|
||||
# webpack JSON parsing.
|
||||
temp_file = tempfile.NamedTemporaryFile(
|
||||
mode="w+", prefix="hyperglass_", suffix=".json", delete=not dev_mode
|
||||
)
|
||||
|
||||
log.info("Starting UI build...")
|
||||
log.debug(
|
||||
f"Created temporary UI config file: '{temp_file.name}' for build {build_id}"
|
||||
)
|
||||
|
||||
with Path(temp_file.name).open("w+") as temp:
|
||||
temp.write(env_json)
|
||||
|
||||
# Write "permanent" file (hard-coded named) for Node to read.
|
||||
env_file.write_text(
|
||||
json.dumps({"configFile": temp_file.name, "buildId": build_id})
|
||||
)
|
||||
|
||||
# While temporary file is still open, initiate UI build process.
|
||||
if not dev_mode or force:
|
||||
initialize_result = await node_initial(dev_mode)
|
||||
build_result = await build_ui(app_path=app_path)
|
||||
|
||||
if initialize_result:
|
||||
log.debug(initialize_result)
|
||||
elif initialize_result == "":
|
||||
log.debug("Re-initialized node_modules")
|
||||
|
||||
if build_result:
|
||||
log.success("Completed UI build")
|
||||
elif dev_mode and not force:
|
||||
log.debug("Running in developer mode, did not build new UI files")
|
||||
|
||||
migrate_images(app_path, params)
|
||||
|
||||
generate_opengraph(
|
||||
Path(params["web"]["opengraph"]["image"]),
|
||||
1200,
|
||||
630,
|
||||
images_dir,
|
||||
params["web"]["theme"]["colors"]["black"],
|
||||
)
|
||||
|
||||
except Exception as err:
|
||||
log.error(err)
|
||||
raise RuntimeError(str(err)) from None
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def set_app_path(required: bool = False) -> Path:
|
||||
"""Find app directory and set value to environment variable."""
|
||||
|
||||
@@ -699,22 +128,10 @@ to access the following directories:
|
||||
|
||||
|
||||
def format_listen_address(listen_address: Union[IPv4Address, IPv6Address, str]) -> str:
|
||||
"""Format a listen_address.
|
||||
|
||||
Wraps IPv6 address in brackets.
|
||||
|
||||
Arguments:
|
||||
listen_address {str} -- Preformatted listen_address
|
||||
|
||||
Returns:
|
||||
{str} -- Formatted listen_address
|
||||
"""
|
||||
"""Format a listen_address. Wraps IPv6 address in brackets."""
|
||||
fmt = str(listen_address)
|
||||
|
||||
if isinstance(listen_address, str):
|
||||
# Standard Library
|
||||
from ipaddress import ip_address
|
||||
|
||||
try:
|
||||
listen_address = ip_address(listen_address)
|
||||
except ValueError as err:
|
||||
@@ -734,7 +151,6 @@ def split_on_uppercase(s):
|
||||
"""Split characters by uppercase letters.
|
||||
|
||||
From: https://stackoverflow.com/a/40382663
|
||||
|
||||
"""
|
||||
string_length = len(s)
|
||||
is_lower_around = (
|
||||
@@ -811,8 +227,6 @@ def get_cache_env():
|
||||
|
||||
def make_repr(_class):
|
||||
"""Create a user-friendly represention of an object."""
|
||||
# Standard Library
|
||||
from asyncio import iscoroutine
|
||||
|
||||
def _process_attrs(_dir):
|
||||
for attr in _dir:
|
||||
|
||||
163
hyperglass/util/files.py
Normal file
163
hyperglass/util/files.py
Normal file
@@ -0,0 +1,163 @@
|
||||
"""Utilities for working with files."""
|
||||
|
||||
# Standard Library
|
||||
import shutil
|
||||
from queue import Queue
|
||||
from typing import List, Tuple, Union, Iterable, Optional, Generator
|
||||
from pathlib import Path
|
||||
from threading import Thread
|
||||
|
||||
# Project
|
||||
from hyperglass.log import log
|
||||
|
||||
|
||||
async def move_files( # noqa: C901
|
||||
src: Path, dst: Path, files: Iterable[Path]
|
||||
) -> Tuple[str]:
|
||||
"""Move iterable of files from source to destination.
|
||||
|
||||
Arguments:
|
||||
src {Path} -- Current directory of files
|
||||
dst {Path} -- Target destination directory
|
||||
files {Iterable} -- Iterable of files
|
||||
"""
|
||||
|
||||
def error(*args, **kwargs):
|
||||
msg = ", ".join(args)
|
||||
kwargs = {k: str(v) for k, v in kwargs.items()}
|
||||
error_msg = msg.format(**kwargs)
|
||||
log.error(error_msg)
|
||||
return RuntimeError(error_msg)
|
||||
|
||||
if not isinstance(src, Path):
|
||||
try:
|
||||
src = Path(src)
|
||||
except TypeError:
|
||||
raise error("{p} is not a valid path", p=src)
|
||||
|
||||
if not isinstance(dst, Path):
|
||||
try:
|
||||
dst = Path(dst)
|
||||
except TypeError:
|
||||
raise error("{p} is not a valid path", p=dst)
|
||||
|
||||
if not isinstance(files, (List, Tuple, Generator)):
|
||||
raise error(
|
||||
"{fa} must be an iterable (list, tuple, or generator). Received {f}",
|
||||
fa="Files argument",
|
||||
f=files,
|
||||
)
|
||||
|
||||
for path in (src, dst):
|
||||
if not path.exists():
|
||||
raise error("{p} does not exist", p=path)
|
||||
|
||||
migrated = ()
|
||||
|
||||
for file in files:
|
||||
dst_file = dst / file.name
|
||||
|
||||
if not file.exists():
|
||||
raise error("{f} does not exist", f=file)
|
||||
|
||||
try:
|
||||
if not dst_file.exists():
|
||||
shutil.copyfile(file, dst_file)
|
||||
migrated += (str(dst_file),)
|
||||
except Exception as e:
|
||||
raise error("Failed to migrate {f}: {e}", f=dst_file, e=e)
|
||||
|
||||
return migrated
|
||||
|
||||
|
||||
class FileCopy(Thread):
|
||||
"""Custom thread for copyfiles() function."""
|
||||
|
||||
def __init__(self, src: Path, dst: Path, queue: Queue):
|
||||
"""Initialize custom thread."""
|
||||
super().__init__()
|
||||
|
||||
if not src.exists():
|
||||
raise ValueError("{} does not exist", str(src))
|
||||
|
||||
self.src = src
|
||||
self.dst = dst
|
||||
self.queue = queue
|
||||
|
||||
def run(self):
|
||||
"""Put one object into the queue for each file."""
|
||||
try:
|
||||
try:
|
||||
shutil.copy(self.src, self.dst)
|
||||
except IOError as err:
|
||||
self.queue.put(err)
|
||||
else:
|
||||
self.queue.put(self.src)
|
||||
finally:
|
||||
pass
|
||||
|
||||
|
||||
def copyfiles(src_files: Iterable[Path], dst_files: Iterable[Path]):
|
||||
"""Copy iterable of files from source to destination with threading."""
|
||||
queue = Queue()
|
||||
threads = ()
|
||||
src_files_len = len(src_files)
|
||||
dst_files_len = len(dst_files)
|
||||
|
||||
if src_files_len != dst_files_len:
|
||||
raise ValueError(
|
||||
"The number of source files "
|
||||
+ "({}) must match the number of destination files ({}).".format(
|
||||
src_files_len, dst_files_len
|
||||
)
|
||||
)
|
||||
|
||||
for i, file in enumerate(src_files):
|
||||
file_thread = FileCopy(src=file, dst=dst_files[i], queue=queue)
|
||||
threads += (file_thread,)
|
||||
|
||||
for thread in threads:
|
||||
thread.start()
|
||||
|
||||
for _file in src_files:
|
||||
copied = queue.get()
|
||||
log.debug("Copied {}", str(copied))
|
||||
|
||||
for thread in threads:
|
||||
thread.join()
|
||||
|
||||
for i, file in enumerate(dst_files):
|
||||
if not file.exists():
|
||||
raise RuntimeError("{} was not copied to {}", str(src_files[i]), str(file))
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def check_path(
|
||||
path: Union[Path, str], mode: str = "r", create: bool = False
|
||||
) -> Optional[Path]:
|
||||
"""Verify if a path exists and is accessible."""
|
||||
|
||||
result = None
|
||||
|
||||
try:
|
||||
if not isinstance(path, Path):
|
||||
path = Path(path)
|
||||
|
||||
if not path.exists():
|
||||
if create:
|
||||
if path.is_file():
|
||||
path.parent.mkdir(parents=True)
|
||||
else:
|
||||
path.mkdir(parents=True)
|
||||
else:
|
||||
raise FileNotFoundError(f"{str(path)} does not exist.")
|
||||
|
||||
if path.exists():
|
||||
with path.open(mode):
|
||||
result = path
|
||||
|
||||
except Exception: # noqa: S110
|
||||
pass
|
||||
|
||||
return result
|
||||
@@ -1,8 +1,20 @@
|
||||
"""Utility functions for frontend-related tasks."""
|
||||
|
||||
# Standard Library
|
||||
import os
|
||||
import json
|
||||
import math
|
||||
import shutil
|
||||
import asyncio
|
||||
import subprocess
|
||||
from typing import Dict, Optional
|
||||
from pathlib import Path
|
||||
|
||||
# Project
|
||||
from hyperglass.log import log
|
||||
|
||||
# Local
|
||||
from .files import copyfiles, check_path
|
||||
|
||||
|
||||
def get_node_version() -> int:
|
||||
@@ -19,3 +31,376 @@ def get_node_version() -> int:
|
||||
major, minor, patch = version.split(".")
|
||||
|
||||
return int(major)
|
||||
|
||||
|
||||
def get_ui_build_timeout() -> Optional[int]:
|
||||
"""Read the UI build timeout from environment variables or set a default."""
|
||||
timeout = None
|
||||
|
||||
if "HYPERGLASS_UI_BUILD_TIMEOUT" in os.environ:
|
||||
timeout = int(os.environ["HYPERGLASS_UI_BUILD_TIMEOUT"])
|
||||
log.info("Found UI build timeout environment variable: {}", timeout)
|
||||
|
||||
elif "POETRY_HYPERGLASS_UI_BUILD_TIMEOUT" in os.environ:
|
||||
timeout = int(os.environ["POETRY_HYPERGLASS_UI_BUILD_TIMEOUT"])
|
||||
log.info("Found UI build timeout environment variable: {}", timeout)
|
||||
|
||||
return timeout
|
||||
|
||||
|
||||
async def check_node_modules() -> bool:
|
||||
"""Check if node_modules exists and has contents."""
|
||||
|
||||
ui_path = Path(__file__).parent.parent / "ui"
|
||||
node_modules = ui_path / "node_modules"
|
||||
|
||||
exists = node_modules.exists()
|
||||
valid = exists
|
||||
|
||||
if exists and not tuple(node_modules.iterdir()):
|
||||
valid = False
|
||||
|
||||
return valid
|
||||
|
||||
|
||||
async def read_package_json() -> Dict:
|
||||
"""Import package.json as a python dict."""
|
||||
|
||||
package_json_file = Path(__file__).parent.parent / "ui" / "package.json"
|
||||
|
||||
try:
|
||||
|
||||
with package_json_file.open("r") as file:
|
||||
package_json = json.load(file)
|
||||
|
||||
except Exception as e:
|
||||
raise RuntimeError(f"Error reading package.json: {str(e)}")
|
||||
|
||||
log.debug("package.json:\n{p}", p=package_json)
|
||||
|
||||
return package_json
|
||||
|
||||
|
||||
async def node_initial(timeout: int = 180, dev_mode: bool = False) -> str:
|
||||
"""Initialize node_modules."""
|
||||
|
||||
ui_path = Path(__file__).parent.parent / "ui"
|
||||
|
||||
env_timeout = get_ui_build_timeout()
|
||||
|
||||
if env_timeout is not None and env_timeout > timeout:
|
||||
timeout = env_timeout
|
||||
|
||||
all_messages = ()
|
||||
|
||||
try:
|
||||
proc = await asyncio.create_subprocess_shell(
|
||||
cmd="yarn --silent --emoji false",
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
cwd=ui_path,
|
||||
)
|
||||
|
||||
stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=timeout)
|
||||
messages = stdout.decode("utf-8").strip()
|
||||
errors = stderr.decode("utf-8").strip()
|
||||
|
||||
if proc.returncode != 0:
|
||||
raise RuntimeError(f"\nMessages:\n{messages}\nErrors:\n{errors}")
|
||||
|
||||
await proc.wait()
|
||||
all_messages += (messages,)
|
||||
|
||||
except Exception as e:
|
||||
raise RuntimeError(str(e))
|
||||
|
||||
return "\n".join(all_messages)
|
||||
|
||||
|
||||
async def build_ui(app_path):
|
||||
"""Execute `next build` & `next export` from UI directory.
|
||||
|
||||
Raises:
|
||||
RuntimeError: Raised if exit code is not 0.
|
||||
RuntimeError: Raised when any other error occurs.
|
||||
"""
|
||||
timeout = get_ui_build_timeout()
|
||||
|
||||
ui_dir = Path(__file__).parent.parent / "ui"
|
||||
build_dir = app_path / "static" / "ui"
|
||||
|
||||
build_command = "node_modules/.bin/next build"
|
||||
export_command = "node_modules/.bin/next export -o {f}".format(f=build_dir)
|
||||
|
||||
all_messages = []
|
||||
for command in (build_command, export_command):
|
||||
try:
|
||||
proc = await asyncio.create_subprocess_shell(
|
||||
cmd=command,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
cwd=ui_dir,
|
||||
)
|
||||
|
||||
stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=timeout)
|
||||
messages = stdout.decode("utf-8").strip()
|
||||
errors = stderr.decode("utf-8").strip()
|
||||
|
||||
if proc.returncode != 0:
|
||||
raise RuntimeError(f"\nMessages:\n{messages}\nErrors:\n{errors}")
|
||||
|
||||
await proc.wait()
|
||||
all_messages.append(messages)
|
||||
|
||||
except asyncio.TimeoutError:
|
||||
raise RuntimeError(f"{timeout} second timeout exceeded while building UI")
|
||||
|
||||
except Exception as err:
|
||||
log.error(err)
|
||||
raise RuntimeError(str(err))
|
||||
|
||||
return "\n".join(all_messages)
|
||||
|
||||
|
||||
def generate_opengraph(
|
||||
image_path: Path,
|
||||
max_width: int,
|
||||
max_height: int,
|
||||
target_path: Path,
|
||||
background_color: str,
|
||||
):
|
||||
"""Generate an OpenGraph compliant image."""
|
||||
# Third Party
|
||||
from PIL import Image
|
||||
|
||||
def center_point(background: Image, foreground: Image):
|
||||
"""Generate a tuple of center points for PIL."""
|
||||
bg_x, bg_y = background.size[0:2]
|
||||
fg_x, fg_y = foreground.size[0:2]
|
||||
x1 = math.floor((bg_x / 2) - (fg_x / 2))
|
||||
y1 = math.floor((bg_y / 2) - (fg_y / 2))
|
||||
x2 = math.floor((bg_x / 2) + (fg_x / 2))
|
||||
y2 = math.floor((bg_y / 2) + (fg_y / 2))
|
||||
return (x1, y1, x2, y2)
|
||||
|
||||
# Convert image to JPEG format with static name "opengraph.jpg"
|
||||
dst_path = target_path / "opengraph.jpg"
|
||||
|
||||
# Copy the original image to the target path
|
||||
copied = shutil.copy2(image_path, target_path)
|
||||
log.debug("Copied {} to {}", str(image_path), str(target_path))
|
||||
|
||||
with Image.open(copied) as src:
|
||||
|
||||
# Only resize the image if it needs to be resized
|
||||
if src.size[0] != max_width or src.size[1] != max_height:
|
||||
|
||||
# Resize image while maintaining aspect ratio
|
||||
log.debug("Opengraph image is not 1200x630, resizing...")
|
||||
src.thumbnail((max_width, max_height))
|
||||
|
||||
# Only impose a background image if the original image has
|
||||
# alpha/transparency channels
|
||||
if src.mode in ("RGBA", "LA"):
|
||||
log.debug("Opengraph image has transparency, converting...")
|
||||
background = Image.new("RGB", (max_width, max_height), background_color)
|
||||
background.paste(src, box=center_point(background, src))
|
||||
dst = background
|
||||
else:
|
||||
dst = src
|
||||
|
||||
# Save new image to derived target path
|
||||
dst.save(dst_path)
|
||||
|
||||
# Delete the copied image
|
||||
Path(copied).unlink()
|
||||
|
||||
if not dst_path.exists():
|
||||
raise RuntimeError(f"Unable to save resized image to {str(dst_path)}")
|
||||
|
||||
log.debug("Opengraph image ready at {}", str(dst_path))
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def migrate_images(app_path: Path, params: dict):
|
||||
"""Migrate images from source code to install directory."""
|
||||
images_dir = app_path / "static" / "images"
|
||||
favicon_dir = images_dir / "favicons"
|
||||
check_path(favicon_dir, create=True)
|
||||
src_files = ()
|
||||
dst_files = ()
|
||||
|
||||
for image in ("light", "dark", "favicon"):
|
||||
src = Path(params["web"]["logo"][image])
|
||||
dst = images_dir / f"{image + src.suffix}"
|
||||
src_files += (src,)
|
||||
dst_files += (dst,)
|
||||
return copyfiles(src_files, dst_files)
|
||||
|
||||
|
||||
async def build_frontend( # noqa: C901
|
||||
dev_mode: bool,
|
||||
dev_url: str,
|
||||
prod_url: str,
|
||||
params: Dict,
|
||||
app_path: Path,
|
||||
force: bool = False,
|
||||
timeout: int = 180,
|
||||
) -> bool:
|
||||
"""Perform full frontend UI build process.
|
||||
|
||||
Securely creates temporary file, writes frontend configuration
|
||||
parameters to file as JSON. Then writes the name of the temporary
|
||||
file to /tmp/hyperglass.env.json as {"configFile": <file_name> }.
|
||||
|
||||
Webpack reads /tmp/hyperglass.env.json, loads the temporary file,
|
||||
and sets its contents to Node environment variables during the build
|
||||
process.
|
||||
|
||||
After the build is successful, the temporary file is automatically
|
||||
closed during garbage collection.
|
||||
|
||||
Arguments:
|
||||
dev_mode {bool} -- Development Mode
|
||||
dev_url {str} -- Development Mode URL
|
||||
prod_url {str} -- Production Mode URL
|
||||
params {dict} -- Frontend Config paramters
|
||||
|
||||
Raises:
|
||||
RuntimeError: Raised if errors occur during build process.
|
||||
|
||||
Returns:
|
||||
{bool} -- True if successful
|
||||
"""
|
||||
# Standard Library
|
||||
import hashlib
|
||||
import tempfile
|
||||
|
||||
# Third Party
|
||||
from favicons import Favicons
|
||||
|
||||
# Project
|
||||
from hyperglass.constants import __version__
|
||||
|
||||
env_file = Path("/tmp/hyperglass.env.json") # noqa: S108
|
||||
|
||||
package_json = await read_package_json()
|
||||
|
||||
env_vars = {
|
||||
"_HYPERGLASS_CONFIG_": params,
|
||||
"_HYPERGLASS_VERSION_": __version__,
|
||||
"_HYPERGLASS_PACKAGE_JSON_": package_json,
|
||||
"_HYPERGLASS_APP_PATH_": str(app_path),
|
||||
}
|
||||
|
||||
# Set NextJS production/development mode and base URL based on
|
||||
# developer_mode setting.
|
||||
if dev_mode:
|
||||
env_vars.update({"NODE_ENV": "development", "_HYPERGLASS_URL_": dev_url})
|
||||
|
||||
else:
|
||||
env_vars.update({"NODE_ENV": "production", "_HYPERGLASS_URL_": prod_url})
|
||||
|
||||
# Check if hyperglass/ui/node_modules has been initialized. If not,
|
||||
# initialize it.
|
||||
initialized = await check_node_modules()
|
||||
|
||||
if initialized:
|
||||
log.debug("node_modules is already initialized")
|
||||
|
||||
elif not initialized:
|
||||
log.debug("node_modules has not been initialized. Starting initialization...")
|
||||
|
||||
node_setup = await node_initial(timeout, dev_mode)
|
||||
|
||||
if node_setup == "":
|
||||
log.debug("Re-initialized node_modules")
|
||||
|
||||
images_dir = app_path / "static" / "images"
|
||||
favicon_dir = images_dir / "favicons"
|
||||
|
||||
try:
|
||||
if not favicon_dir.exists():
|
||||
favicon_dir.mkdir()
|
||||
async with Favicons(
|
||||
source=params["web"]["logo"]["favicon"],
|
||||
output_directory=favicon_dir,
|
||||
base_url="/images/favicons/",
|
||||
) as favicons:
|
||||
await favicons.generate()
|
||||
log.debug("Generated {} favicons", favicons.completed)
|
||||
env_vars.update({"_HYPERGLASS_FAVICONS_": favicons.formats()})
|
||||
|
||||
env_json = json.dumps(env_vars, default=str)
|
||||
|
||||
# Create SHA256 hash from all parameters passed to UI, use as
|
||||
# build identifier.
|
||||
build_id = hashlib.sha256(env_json.encode()).hexdigest()
|
||||
|
||||
# Read hard-coded environment file from last build. If build ID
|
||||
# matches this build's ID, don't run a new build.
|
||||
if env_file.exists() and not force:
|
||||
|
||||
with env_file.open("r") as ef:
|
||||
ef_id = json.load(ef).get("buildId", "empty")
|
||||
|
||||
log.debug("Previous Build ID: {id}", id=ef_id)
|
||||
|
||||
if ef_id == build_id:
|
||||
|
||||
log.debug(
|
||||
"UI parameters unchanged since last build, skipping UI build..."
|
||||
)
|
||||
|
||||
return True
|
||||
|
||||
# Create temporary file. json file extension is added for easy
|
||||
# webpack JSON parsing.
|
||||
temp_file = tempfile.NamedTemporaryFile(
|
||||
mode="w+", prefix="hyperglass_", suffix=".json", delete=not dev_mode
|
||||
)
|
||||
|
||||
log.info("Starting UI build...")
|
||||
log.debug(
|
||||
f"Created temporary UI config file: '{temp_file.name}' for build {build_id}"
|
||||
)
|
||||
|
||||
with Path(temp_file.name).open("w+") as temp:
|
||||
temp.write(env_json)
|
||||
|
||||
# Write "permanent" file (hard-coded named) for Node to read.
|
||||
env_file.write_text(
|
||||
json.dumps({"configFile": temp_file.name, "buildId": build_id})
|
||||
)
|
||||
|
||||
# While temporary file is still open, initiate UI build process.
|
||||
if not dev_mode or force:
|
||||
initialize_result = await node_initial(timeout, dev_mode)
|
||||
build_result = await build_ui(app_path=app_path)
|
||||
|
||||
if initialize_result:
|
||||
log.debug(initialize_result)
|
||||
elif initialize_result == "":
|
||||
log.debug("Re-initialized node_modules")
|
||||
|
||||
if build_result:
|
||||
log.success("Completed UI build")
|
||||
elif dev_mode and not force:
|
||||
log.debug("Running in developer mode, did not build new UI files")
|
||||
|
||||
migrate_images(app_path, params)
|
||||
|
||||
generate_opengraph(
|
||||
Path(params["web"]["opengraph"]["image"]),
|
||||
1200,
|
||||
630,
|
||||
images_dir,
|
||||
params["web"]["theme"]["colors"]["black"],
|
||||
)
|
||||
|
||||
except Exception as err:
|
||||
log.error(err)
|
||||
raise RuntimeError(str(err)) from None
|
||||
|
||||
return True
|
||||
|
||||
Reference in New Issue
Block a user