From 7fb6ab7c92cb5168a83fb372bb2e4e45f59c3aa6 Mon Sep 17 00:00:00 2001 From: checktheroads Date: Tue, 31 Dec 2019 11:35:39 -0700 Subject: [PATCH] refactor aiofiles to aiofile lib --- Pipfile | 1 + Pipfile.lock | 16 +++- hyperglass/render/webassets.py | 134 ++++++++++++++++++--------------- 3 files changed, 91 insertions(+), 60 deletions(-) diff --git a/Pipfile b/Pipfile index 96aab07..12bcf89 100644 --- a/Pipfile +++ b/Pipfile @@ -49,6 +49,7 @@ sshtunnel = "==0.1.5" stackprinter = "==0.2.3" uvloop = "==0.13.0" aiofiles = "*" +aiofile = "*" [requires] python_version = "3.7" diff --git a/Pipfile.lock b/Pipfile.lock index b7ee994..5e8c4b1 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "a91bc44cbe2dea50ecd9d2804ead28506ddb307036529b292184d4458d06565e" + "sha256": "28db1e4943f6cc77cd519dd14feb859f1370a91c7102880cf34ef4f11dd6e524" }, "pipfile-spec": 6, "requires": { @@ -16,6 +16,19 @@ ] }, "default": { + "aiofile": { + "hashes": [ + "sha256:229078abbaab87adfcaad0fa7766b9b8251d42d0242deac6166da433b027ef1f", + "sha256:312d50ed7e646a40ab2a5457fdf382870aca926f956921ab8c7ab72c3922f372", + "sha256:8c50fcb42ee2bad2ae811edb972724e7f6bf3b0a6565a498f2432862b548b92d", + "sha256:a9a457654e561c396b88f70a0d5fa00e40a337853aa180bc805d9d5efb82317c", + "sha256:cef9e7bdf93db6a4c7ffe9ef0c354e2887695ec2a3a9dda8ed285005ec835616", + "sha256:d1da2fc9aa7509d29ea09617bf533bd1045f79cfdfb10ee83da90ba2212720a2", + "sha256:e43cb5e3181a8dfb73afbb4749b024e9a35a52e60ecf97d1d3db2731212cb0a0" + ], + "index": "pypi", + "version": "==1.5.2" + }, "aiofiles": { "hashes": [ "sha256:021ea0ba314a86027c166ecc4b4c07f2d40fc0f4b3a950d1868a0f2571c2bbee", @@ -318,6 +331,7 @@ "sha256:78bed18e7f1eb21f3d10ff3acde900b4d630098648fe1d65bb4abfb3e22c4900", "sha256:a02fade7b5476c4f88efe9593ff2f3286698d8c6d715ba4f426954f73f382026", "sha256:aacbde3a8875352a640efa2d1b96e5244a29b0f8df79cbf1ec6470e86fd84697", + "sha256:be813fb9e5ce41a5a99a29cdb857144a1bd6670883586f995b940a4878dc5238", "sha256:bfcad6da0b8839f01a819602aaa5c5a5b4c85ecbfae9b261a31df3d9262fb31e", "sha256:c2bfc0db3166e68515bc4a2b9164f4f75ae9c793e9635f8651f2c9ffc65c8dad", "sha256:c66d11870ae066499a3541963e6ce18512ca827c2aaeaa2f4e37501cee39ac5d", diff --git a/hyperglass/render/webassets.py b/hyperglass/render/webassets.py index a0b8790..c28c8b6 100644 --- a/hyperglass/render/webassets.py +++ b/hyperglass/render/webassets.py @@ -7,8 +7,8 @@ import time from pathlib import Path # Third Party Imports -import aiofiles import jinja2 +from aiofile import AIOFile # Project Imports from hyperglass.configuration import frontend_devices @@ -18,24 +18,32 @@ from hyperglass.configuration import params from hyperglass.exceptions import HyperglassError from hyperglass.util import log -# Module Directories -working_directory = Path(__file__).resolve().parent -hyperglass_root = working_directory.parent -file_loader = jinja2.FileSystemLoader(str(working_directory)) -env = jinja2.Environment( - loader=file_loader, +# File & Path Definitions +CWD = Path(__file__).parent +ROOT_DIR = CWD.parent +FRONTEND_CONFIG = ROOT_DIR / "static/src/js/frontend.json" +FONT_DIR = ROOT_DIR / "static/src/sass/fonts" +FONT_CMD = ROOT_DIR / "static/src/node_modules/get-google-fonts/cli.js" +THEME_FILE = ROOT_DIR / "static/src/sass/theme.sass" +STATIC_DIR = ROOT_DIR / "static/src" + +# Jinja2 Config +JINJA_FILE_LOADER = jinja2.FileSystemLoader(str(CWD)) +JINJA_ENV = jinja2.Environment( + loader=JINJA_FILE_LOADER, autoescape=True, extensions=["jinja2.ext.autoescape"], enable_async=True, ) -async def render_frontend_config(): +async def _render_frontend_config(): """Render user config to JSON for use by frontend.""" - rendered_frontend_file = hyperglass_root.joinpath("static/src/js/frontend.json") + log.debug("Rendering front end config...") + try: - async with aiofiles.open(rendered_frontend_file, mode="w") as frontend_file: - await frontend_file.write( + async with AIOFile(FRONTEND_CONFIG, "w+") as file: + await file.write( json.dumps( { "config": frontend_params, @@ -44,107 +52,115 @@ async def render_frontend_config(): } ) ) - except Exception as frontend_error: - raise HyperglassError(f"Error rendering front end config: {frontend_error}") + await file.fsync() + log.debug("Rendered front end config") + except Exception as e: + raise HyperglassError(f"Error rendering front end config: {str(e)}") -async def get_fonts(): +async def _get_fonts(): """Download Google fonts.""" - font_dir = hyperglass_root.joinpath("static/src/sass/fonts") - font_bin = str( - hyperglass_root.joinpath("static/src/node_modules/get-google-fonts/cli.js") - ) + log.debug("Downloading theme fonts...") + font_base = "https://fonts.googleapis.com/css?family={p}|{m}&display=swap" font_primary = "+".join(params.branding.font.primary.split(" ")).strip() font_mono = "+".join(params.branding.font.mono.split(" ")).strip() font_url = font_base.format(p=font_primary + ":300,400,700", m=font_mono + ":400") - command = f"node {str(font_bin)} -w -i '{font_url}' -o {str(font_dir)}" + + font_command = f"node {str(FONT_CMD)} -w -i '{font_url}' -o {str(FONT_DIR)}" + try: proc = await asyncio.create_subprocess_shell( - command, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE + cmd=font_command, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + cwd=STATIC_DIR, ) stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=60) + for line in stdout.decode().strip().split("\n"): log.debug(line) + if proc.returncode != 0: output_error = stderr.decode("utf-8") log.error(output_error) raise RuntimeError(f"Error downloading font from URL {font_url}") + await proc.wait() + log.debug("Downloaded theme fonts") + except Exception as e: raise HyperglassError(str(e)) -async def render_theme(): +async def _render_theme(): """Render Jinja2 template to Sass file.""" - rendered_theme_file = hyperglass_root.joinpath("static/src/sass/theme.sass") + log.debug("Rendering theme elements...") try: - template = env.get_template("templates/theme.sass.j2") + template = JINJA_ENV.get_template("templates/theme.sass.j2") rendered_theme = await template.render_async(params.branding) log.debug(f"Branding variables:\n{params.branding.json(indent=4)}") log.debug(f"Rendered theme:\n{str(rendered_theme)}") - async with aiofiles.open(rendered_theme_file, mode="w") as theme_file: - await theme_file.write(rendered_theme) - except jinja2.exceptions as theme_error: - raise HyperglassError(f"Error rendering theme: {theme_error}") + + async with AIOFile(THEME_FILE, "w+") as file: + await file.write(rendered_theme) + await file.fsync() + log.debug("Rendered theme elements") + except Exception as e: + raise HyperglassError(f"Error rendering theme: {e}") -async def build_assets(): +async def _build_assets(): """Build, bundle, and minify Sass/CSS/JS web assets.""" - command = "yarn --silent --emoji false --json --no-progress build" - static_dir = hyperglass_root.joinpath("static/src") + log.debug("Building web assets...") + + yarn_command = "yarn --silent --emoji false --json --no-progress build" try: proc = await asyncio.create_subprocess_shell( - command, + cmd=yarn_command, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, - cwd=static_dir, + cwd=STATIC_DIR, ) + stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=60) output_out = json.loads(stdout.decode("utf-8").split("\n")[0]) + if proc.returncode != 0: output_error = json.loads(stderr.decode("utf-8").strip("\n")) raise RuntimeError( f'Error building web assets with script {output_out["data"]}:' f'{output_error["data"]}' ) + await proc.wait() + log.debug(f'Built web assets with script {output_out["data"]}') except Exception as e: raise HyperglassError(str(e)) - log.debug(f'Built web assets with script {output_out["data"]}') + + +async def _render_all(): + """Run all asset rendering/building functions. + + Raises: + HyperglassError: Raised if any downstream errors occur. + """ + try: + await _render_frontend_config() + await _get_fonts() + await _render_theme() + await _build_assets() + except HyperglassError as e: + raise HyperglassError(str(e)) from None def render_assets(): - """Run web asset rendering functions.""" + """Render assets.""" start = time.time() - try: - log.debug("Rendering front end config...") - asyncio.run(render_frontend_config()) - log.debug("Rendered front end config") - except HyperglassError as frontend_error: - raise HyperglassError(str(frontend_error)) - try: - log.debug("Downloading theme fonts...") - asyncio.run(get_fonts()) - log.debug("Downloaded theme fonts") - except HyperglassError as theme_error: - raise HyperglassError(str(theme_error)) + asyncio.run(_render_all()) - try: - log.debug("Rendering theme elements...") - asyncio.run(render_theme()) - log.debug("Rendered theme elements") - except HyperglassError as theme_error: - raise HyperglassError(str(theme_error)) - - try: - log.debug("Building web assets...") - asyncio.run(build_assets()) - log.debug("Built web assets") - except HyperglassError as assets_error: - raise HyperglassError(str(assets_error)) end = time.time() elapsed = round(end - start, 2) log.debug(f"Rendered assets in {elapsed} seconds.")