diff --git a/Pipfile b/Pipfile index bd16f8a..96aab07 100644 --- a/Pipfile +++ b/Pipfile @@ -48,6 +48,7 @@ sanic = "==19.6.2" sshtunnel = "==0.1.5" stackprinter = "==0.2.3" uvloop = "==0.13.0" +aiofiles = "*" [requires] python_version = "3.7" diff --git a/Pipfile.lock b/Pipfile.lock index 161416e..b7ee994 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "3e7ddad4bedaf6f02025640840f112de7f92a7a8afffafe46651ef710b2477d6" + "sha256": "a91bc44cbe2dea50ecd9d2804ead28506ddb307036529b292184d4458d06565e" }, "pipfile-spec": 6, "requires": { @@ -21,6 +21,7 @@ "sha256:021ea0ba314a86027c166ecc4b4c07f2d40fc0f4b3a950d1868a0f2571c2bbee", "sha256:1e644c2573f953664368de28d2aa4c89dfd64550429d0c27c4680ccd3aa4985d" ], + "index": "pypi", "version": "==0.4.0" }, "aredis": { diff --git a/hyperglass/render/webassets.py b/hyperglass/render/webassets.py index 179be4c..a0b8790 100644 --- a/hyperglass/render/webassets.py +++ b/hyperglass/render/webassets.py @@ -1,11 +1,13 @@ """Renders Jinja2 & Sass templates for use by the front end application.""" # Standard Library Imports +import asyncio import json -import subprocess +import time from pathlib import Path # Third Party Imports +import aiofiles import jinja2 # Project Imports @@ -21,16 +23,19 @@ working_directory = Path(__file__).resolve().parent hyperglass_root = working_directory.parent file_loader = jinja2.FileSystemLoader(str(working_directory)) env = jinja2.Environment( - loader=file_loader, autoescape=True, extensions=["jinja2.ext.autoescape"] + loader=file_loader, + autoescape=True, + extensions=["jinja2.ext.autoescape"], + enable_async=True, ) -def render_frontend_config(): +async def render_frontend_config(): """Render user config to JSON for use by frontend.""" rendered_frontend_file = hyperglass_root.joinpath("static/src/js/frontend.json") try: - with rendered_frontend_file.open(mode="w") as frontend_file: - frontend_file.write( + async with aiofiles.open(rendered_frontend_file, mode="w") as frontend_file: + await frontend_file.write( json.dumps( { "config": frontend_params, @@ -39,12 +44,11 @@ def render_frontend_config(): } ) ) - except jinja2.exceptions as frontend_error: - log.error(f"Error rendering front end config: {frontend_error}") - raise HyperglassError(frontend_error) + except Exception as frontend_error: + raise HyperglassError(f"Error rendering front end config: {frontend_error}") -def get_fonts(): +async def get_fonts(): """Download Google fonts.""" font_dir = hyperglass_root.joinpath("static/src/sass/fonts") font_bin = str( @@ -54,90 +58,93 @@ def get_fonts(): font_primary = "+".join(params.branding.font.primary.split(" ")).strip() font_mono = "+".join(params.branding.font.mono.split(" ")).strip() font_url = font_base.format(p=font_primary + ":300,400,700", m=font_mono + ":400") - proc = subprocess.Popen( - ["node", font_bin, "-w", "-i", font_url, "-o", font_dir], - cwd=hyperglass_root.joinpath("static"), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) + command = f"node {str(font_bin)} -w -i '{font_url}' -o {str(font_dir)}" try: - stdout, stderr = proc.communicate(timeout=60) - except subprocess.TimeoutExpired: - proc.kill() - stdout, stderr = proc.communicate() - if proc.returncode != 0: - output_error = stderr.decode("utf-8") - log.error(output_error) - raise HyperglassError(f"Error downloading font from URL {font_url}") - else: - proc.kill() + proc = await asyncio.create_subprocess_shell( + command, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE + ) + stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=60) + for line in stdout.decode().strip().split("\n"): + log.debug(line) + if proc.returncode != 0: + output_error = stderr.decode("utf-8") + log.error(output_error) + raise RuntimeError(f"Error downloading font from URL {font_url}") + await proc.wait() + except Exception as e: + raise HyperglassError(str(e)) -def render_theme(): +async def render_theme(): """Render Jinja2 template to Sass file.""" rendered_theme_file = hyperglass_root.joinpath("static/src/sass/theme.sass") try: template = env.get_template("templates/theme.sass.j2") + rendered_theme = await template.render_async(params.branding) + log.debug(f"Branding variables:\n{params.branding.json(indent=4)}") - rendered_theme = template.render(params.branding) log.debug(f"Rendered theme:\n{str(rendered_theme)}") - with rendered_theme_file.open(mode="w") as theme_file: - theme_file.write(rendered_theme) + async with aiofiles.open(rendered_theme_file, mode="w") as theme_file: + await theme_file.write(rendered_theme) except jinja2.exceptions as theme_error: - log.error(f"Error rendering theme: {theme_error}") - raise HyperglassError(theme_error) + raise HyperglassError(f"Error rendering theme: {theme_error}") -def build_assets(): +async def build_assets(): """Build, bundle, and minify Sass/CSS/JS web assets.""" - proc = subprocess.Popen( - ["yarn", "--silent", "--emoji", "false", "--json", "--no-progress", "build"], - cwd=hyperglass_root.joinpath("static/src"), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) + command = "yarn --silent --emoji false --json --no-progress build" + static_dir = hyperglass_root.joinpath("static/src") try: - stdout, stderr = proc.communicate(timeout=60) - except subprocess.TimeoutExpired: - proc.kill() - stdout, stderr = proc.communicate() - output_out = json.loads(stdout.decode("utf-8").split("\n")[0]) - if proc.returncode != 0: - output_error = json.loads(stderr.decode("utf-8").strip("\n")) - log.error(output_error["data"]) - raise HyperglassError( - f'Error building web assets with script {output_out["data"]}:' - f'{output_error["data"]}' + proc = await asyncio.create_subprocess_shell( + command, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + cwd=static_dir, ) + stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=60) + output_out = json.loads(stdout.decode("utf-8").split("\n")[0]) + if proc.returncode != 0: + output_error = json.loads(stderr.decode("utf-8").strip("\n")) + raise RuntimeError( + f'Error building web assets with script {output_out["data"]}:' + f'{output_error["data"]}' + ) + await proc.wait() + except Exception as e: + raise HyperglassError(str(e)) log.debug(f'Built web assets with script {output_out["data"]}') def render_assets(): """Run web asset rendering functions.""" + start = time.time() try: log.debug("Rendering front end config...") - render_frontend_config() + asyncio.run(render_frontend_config()) log.debug("Rendered front end config") except HyperglassError as frontend_error: - raise HyperglassError(frontend_error) from None + raise HyperglassError(str(frontend_error)) try: log.debug("Downloading theme fonts...") - get_fonts() + asyncio.run(get_fonts()) log.debug("Downloaded theme fonts") except HyperglassError as theme_error: - raise HyperglassError(theme_error) from None + raise HyperglassError(str(theme_error)) try: log.debug("Rendering theme elements...") - render_theme() + asyncio.run(render_theme()) log.debug("Rendered theme elements") except HyperglassError as theme_error: - raise HyperglassError(theme_error) from None + raise HyperglassError(str(theme_error)) try: log.debug("Building web assets...") - build_assets() + asyncio.run(build_assets()) log.debug("Built web assets") except HyperglassError as assets_error: - raise HyperglassError(str(assets_error)) from None + raise HyperglassError(str(assets_error)) + end = time.time() + elapsed = round(end - start, 2) + log.debug(f"Rendered assets in {elapsed} seconds.") diff --git a/manage.py b/manage.py index 533f920..0945e49 100755 --- a/manage.py +++ b/manage.py @@ -685,15 +685,11 @@ def dev_server(host, port, assets): @hg.command("render-assets", help="Render theme & build web assets") def render_assets(): """Render theme template to Sass file and build web assets""" - try: - assets_rendered = render_hyperglass_assets() - except Exception as e: - raise click.ClickException( - click.style("✗ Error rendering assets: ", fg="red", bold=True) - + click.style(e, fg="blue") - ) + assets_rendered = render_hyperglass_assets() if not assets_rendered: raise click.ClickException("✗ Error rendering assets") + elif assets_rendered: + click.secho("✓ Rendered assets", fg="green", bold=True) @hg.command("migrate-configs", help="Copy YAML examples to usable config files")