mirror/checktheroads-hyperglass
1
0
Fork 0
mirror of https://github.com/checktheroads/hyperglass synced 2024-05-11 05:55:08 +00:00
Code Releases Activity

add redis password authentication support, closes #82

Browse Source
This commit is contained in:
checktheroads
2020-10-10 21:13:28 -07:00
parent 14ec5da505
commit ba1a91c93f
11 changed files with 157 additions and 136 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
hyperglass/api/events.py
View File

@@ -1,27 +1,16 @@
"""API Events."""
# Project
from hyperglass.util import check_redis
from hyperglass.exceptions import HyperglassError
from hyperglass.cache import AsyncCache
from hyperglass.configuration import REDIS_CONFIG, params
async def _check_redis():
"""Ensure Redis is running before starting server.
Raises:
HyperglassError: Raised if Redis is not running.
Returns:
{bool} -- True if Redis is running.
"""
try:
await check_redis(db=params.cache.database, config=REDIS_CONFIG)
except RuntimeError as e:
raise HyperglassError(str(e), level="danger") from None
async def check_redis() -> bool:
"""Ensure Redis is running before starting server."""
cache = AsyncCache(db=params.cache.database, **REDIS_CONFIG)
await cache.test()
return True
on_startup = (_check_redis,)
on_startup = (check_redis,)
on_shutdown = ()

46
hyperglass/cache/aio.py vendored
View File

@@ -23,16 +23,46 @@ class AsyncCache(BaseCache):
def __init__(self, *args, **kwargs):
"""Initialize Redis connection."""
super().__init__(*args, **kwargs)
password = self.password
if password is not None:
password = password.get_secret_value()
self.instance: AsyncRedis = AsyncRedis(
db=self.db,
host=self.host,
port=self.port,
password=password,
decode_responses=self.decode_responses,
**self.redis_args,
)
async def test(self):
"""Send an echo to Redis to ensure it can be reached."""
try:
self.instance: AsyncRedis = AsyncRedis(
db=self.db,
host=self.host,
port=self.port,
decode_responses=self.decode_responses,
**self.redis_args,
)
await self.instance.echo("hyperglass test")
except RedisError as err:
raise HyperglassError(str(err), level="danger")
err_msg = str(err)
if not err_msg and hasattr(err, "__context__"):
# Some Redis exceptions are raised without a message
# even if they are raised from another exception that
# does have a message.
err_msg = str(err.__context__)
if "auth" in err_msg.lower():
raise HyperglassError(
"Authentication to Redis server {server} failed.".format(
server=repr(self)
),
level="danger",
) from None
else:
raise HyperglassError(
"Unable to connect to Redis server {server}".format(
server=repr(self)
),
level="danger",
) from None
async def get(self, *args: str) -> Any:
"""Get item(s) from cache."""

11
hyperglass/cache/base.py vendored
View File

@@ -3,7 +3,10 @@
# Standard Library
import re
import json
from typing import Any
from typing import Any, Optional
# Third Party
from pydantic import SecretStr
class BaseCache:
@@ -14,6 +17,7 @@ class BaseCache:
db: int,
host: str = "localhost",
port: int = 6379,
password: Optional[SecretStr] = None,
decode_responses: bool = True,
**kwargs: Any,
) -> None:
@@ -21,12 +25,15 @@ class BaseCache:
self.db: int = db
self.host: str = str(host)
self.port: int = port
self.password: Optional[SecretStr] = password
self.decode_responses: bool = decode_responses
self.redis_args: dict = kwargs
def __repr__(self) -> str:
"""Represent class state."""
return f"HyperglassCache(db={self.db}, host={self.host}, port={self.port})"
return "HyperglassCache(db={}, host={}, port={}, password={})".format(
self.db, self.host, self.port, self.password
)
def parse_types(self, value: str) -> Any:
"""Parse a string to standard python types."""

46
hyperglass/cache/sync.py vendored
View File

@@ -22,16 +22,46 @@ class SyncCache(BaseCache):
def __init__(self, *args, **kwargs):
"""Initialize Redis connection."""
super().__init__(*args, **kwargs)
password = self.password
if password is not None:
password = password.get_secret_value()
self.instance: SyncRedis = SyncRedis(
db=self.db,
host=self.host,
port=self.port,
password=password,
decode_responses=self.decode_responses,
**self.redis_args,
)
def test(self):
"""Send an echo to Redis to ensure it can be reached."""
try:
self.instance: SyncRedis = SyncRedis(
db=self.db,
host=self.host,
port=self.port,
decode_responses=self.decode_responses,
**self.redis_args,
)
self.instance.echo("hyperglass test")
except RedisError as err:
raise HyperglassError(str(err), level="danger")
err_msg = str(err)
if not err_msg and hasattr(err, "__context__"):
# Some Redis exceptions are raised without a message
# even if they are raised from another exception that
# does have a message.
err_msg = str(err.__context__)
if "auth" in err_msg.lower():
raise HyperglassError(
"Authentication to Redis server {server} failed.".format(
server=repr(self)
),
level="danger",
) from None
else:
raise HyperglassError(
"Unable to connect to Redis server {server}".format(
server=repr(self)
),
level="danger",
) from None
def get(self, *args: str) -> Any:
"""Get item(s) from cache."""

2
hyperglass/cli/commands.py
View File

@@ -123,7 +123,7 @@ def start(build, direct, workers):
elif not build and direct:
uvicorn_start(**kwargs)
except Exception as err:
except BaseException as err:
error(str(err))

1
hyperglass/configuration/__init__.py
View File

@@ -502,4 +502,5 @@ REDIS_CONFIG = {
"host": str(params.cache.host),
"port": params.cache.port,
"decode_responses": True,
"password": params.cache.password,
}

37
hyperglass/configuration/models/cache.py
View File

@@ -1,10 +1,10 @@
"""Validation model for Redis cache config."""
# Standard Library
from typing import Union
from typing import Union, Optional
# Third Party
from pydantic import Field, StrictInt, StrictStr, StrictBool, IPvAnyAddress
from pydantic import SecretStr, StrictInt, StrictStr, StrictBool, IPvAnyAddress
# Project
from hyperglass.models import HyperglassModel
@@ -13,26 +13,25 @@ from hyperglass.models import HyperglassModel
class Cache(HyperglassModel):
"""Validation model for params.cache."""
host: Union[IPvAnyAddress, StrictStr] = Field(
"localhost", title="Host", description="Redis server IP address or hostname."
)
port: StrictInt = Field(6379, title="Port", description="Redis server TCP port.")
database: StrictInt = Field(
1, title="Database ID", description="Redis server database ID."
)
timeout: StrictInt = Field(
120,
title="Timeout",
description="Time in seconds query output will be kept in the Redis cache.",
)
show_text: StrictBool = Field(
True,
title="Show Text",
description="Show the cache text in the hyperglass UI.",
)
host: Union[IPvAnyAddress, StrictStr] = "localhost"
port: StrictInt = 6379
database: StrictInt = 1
password: Optional[SecretStr]
timeout: StrictInt = 120
show_text: StrictBool = True
class Config:
"""Pydantic model configuration."""
title = "Cache"
description = "Redis server & cache timeout configuration."
fields = {
"host": {"description": "Redis server IP address or hostname."},
"port": {"description": "Redis server TCP port."},
"database": {"description": "Redis server database ID."},
"password": {"description": "Redis authentication password."},
"timeout": {
"description": "Time in seconds query output will be kept in the Redis cache."
},
"show_test": {description: "Show the cache text in the hyperglass UI."},
}

32
hyperglass/exceptions.py
View File

@@ -2,14 +2,25 @@
# Standard Library
import json as _json
from typing import List
from typing import Dict, List, Union, Sequence
# Project
from hyperglass.log import log
from hyperglass.util import validation_error_message
from hyperglass.constants import STATUS_CODE_MAP
def validation_error_message(*errors: Dict) -> str:
"""Parse errors return from pydantic.ValidationError.errors()."""
errs = ("\n",)
for err in errors:
loc = " → ".join(str(loc) for loc in err["loc"])
errs += (f'Field: {loc}\n Error: {err["msg"]}\n',)
return "\n".join(errs)
class HyperglassError(Exception):
"""hyperglass base exception."""
@@ -203,4 +214,19 @@ class UnsupportedDevice(_UnformattedHyperglassError):
class ParsingError(_UnformattedHyperglassError):
"""Raised when there is a problem parsing a structured response."""
_level = "danger"
def __init__(
self,
unformatted_msg: Union[Sequence[Dict], str],
level: str = "danger",
**kwargs,
):
"""Format error message with keyword arguments."""
if isinstance(unformatted_msg, Sequence):
self._message = validation_error_message(*unformatted_msg)
else:
self._message = unformatted_msg.format(**kwargs)
self._level = level or self._level
self._keywords = list(kwargs.values())
super().__init__(
message=self._message, level=self._level, keywords=self._keywords
)

29
hyperglass/main.py
View File

@@ -12,13 +12,15 @@ from gunicorn.app.base import BaseApplication
# Project
from hyperglass.log import log
from hyperglass.cache import AsyncCache
from hyperglass.constants import MIN_PYTHON_VERSION, __version__
pretty_version = ".".join(tuple(str(v) for v in MIN_PYTHON_VERSION))
if sys.version_info < MIN_PYTHON_VERSION:
raise RuntimeError(f"Python {pretty_version}+ is required.")
# Project
from hyperglass.cache import SyncCache
from hyperglass.configuration import ( # isort:skip
params,
URL_DEV,
@@ -29,7 +31,6 @@ from hyperglass.configuration import ( # isort:skip
)
from hyperglass.util import ( # isort:skip
cpu_count,
check_redis,
build_frontend,
clear_redis_cache,
format_listen_address,
@@ -44,14 +45,11 @@ else:
loglevel = "WARNING"
async def check_redis_instance():
"""Ensure Redis is running before starting server.
Returns:
{bool} -- True if Redis is running.
"""
await check_redis(db=params.cache.database, config=REDIS_CONFIG)
def check_redis_instance() -> bool:
"""Ensure Redis is running before starting server."""
cache = SyncCache(db=params.cache.database, **REDIS_CONFIG)
cache.test()
log.debug("Redis is running at: {}:{}", REDIS_CONFIG["host"], REDIS_CONFIG["port"])
return True
@@ -81,15 +79,13 @@ async def clear_cache():
pass
async def cache_config():
def cache_config():
"""Add configuration to Redis cache as a pickled object."""
# Standard Library
import pickle
cache = AsyncCache(
db=params.cache.database, host=params.cache.host, port=params.cache.port
)
await cache.set("HYPERGLASS_CONFIG", pickle.dumps(params))
cache = SyncCache(db=params.cache.database, **REDIS_CONFIG)
cache.set("HYPERGLASS_CONFIG", pickle.dumps(params))
return True
@@ -107,8 +103,9 @@ def on_starting(server: Arbiter):
await gather(build_ui(), cache_config())
aiorun(check_redis_instance())
aiorun(runner())
check_redis_instance()
aiorun(build_ui())
cache_config()
log.success(
"Started hyperglass {v} on http://{h}:{p} with {w} workers",

3
hyperglass/parsing/juniper.py
View File

@@ -9,7 +9,6 @@ from pydantic import ValidationError
# Project
from hyperglass.log import log
from hyperglass.util import validation_error_message
from hyperglass.exceptions import ParsingError, ResponseEmpty
from hyperglass.configuration import params
from hyperglass.parsing.models.juniper import JuniperRoute
@@ -58,6 +57,6 @@ def parse_juniper(output: Iterable) -> Dict: # noqa: C901
except ValidationError as err:
log.critical(str(err))
raise ParsingError(validation_error_message(*err.errors()))
raise ParsingError(err.errors())
return data

63
hyperglass/util/__init__.py
View File

@@ -18,6 +18,7 @@ from loguru._logger import Logger as LoguruLogger
# Project
from hyperglass.log import log
from hyperglass.cache import AsyncCache
from hyperglass.models import HyperglassModel
@@ -147,14 +148,7 @@ async def build_ui(app_path):
async def write_env(variables: Dict) -> str:
"""Write environment variables to temporary JSON file.
Arguments:
variables {dict} -- Environment variables to write.
Raises:
RuntimeError: Raised on any errors.
"""
"""Write environment variables to temporary JSON file."""
env_file = Path("/tmp/hyperglass.env.json") # noqa: S108
env_vars = json.dumps(variables)
@@ -167,47 +161,8 @@ async def write_env(variables: Dict) -> str:
return f"Wrote {env_vars} to {str(env_file)}"
async def check_redis(db: int, config: Dict) -> bool:
"""Ensure Redis is running before starting server.
Arguments:
db {int} -- Redis database ID
config {dict} -- Redis configuration parameters
Raises:
RuntimeError: Raised if Redis is not running.
Returns:
{bool} -- True if redis is running.
"""
# Third Party
import aredis
redis_instance = aredis.StrictRedis(db=db, **config)
redis_host = config["host"]
redis_port = config["port"]
try:
await redis_instance.echo("hyperglass test")
except Exception:
raise RuntimeError(
f"Redis isn't running at: {redis_host}:{redis_port}"
) from None
return True
async def clear_redis_cache(db: int, config: Dict) -> bool:
"""Clear the Redis cache.
Arguments:
db {int} -- Redis database ID
config {dict} -- Redis configuration parameters
Raises:
RuntimeError: Raised if clearing the cache produces an error.
Returns:
{bool} -- True if cache was cleared.
"""
"""Clear the Redis cache."""
# Third Party
import aredis
@@ -936,18 +891,6 @@ def current_log_level(logger: LoguruLogger) -> str:
return current_level
def validation_error_message(*errors: Dict) -> str:
"""Parse errors return from pydantic.ValidationError.errors()."""
errs = ("\n",)
for err in errors:
loc = " → ".join(str(loc) for loc in err["loc"])
errs += (f'Field: {loc}\n Error: {err["msg"]}\n',)
return "\n".join(errs)
def resolve_hostname(hostname: str) -> Generator:
"""Resolve a hostname via DNS/hostfile."""
# Standard Library