diff --git a/.flake8 b/.flake8 index efb97a6..81f0153 100644 --- a/.flake8 +++ b/.flake8 @@ -18,13 +18,15 @@ per-file-ignores= hyperglass/*/__init__.py:F401 hyperglass/models/*/__init__.py:F401 # Disable assertion and docstring checks on tests. - hyperglass/**/test_*.py:S101,D103 + hyperglass/**/test_*.py:S101,D103,D100,D104 + hyperglass/**/tests/*.py:S101,D103,D100,D104 + hyperglass/**/tests/__init__.py:D103,D100,D104 hyperglass/state/hooks.py:F811 # Ignore whitespace in docstrings hyperglass/cli/static.py:W293 # Ignore docstring standards hyperglass/cli/main.py:D400,D403 -ignore=W503,C0330,R504,D202,S403,S301,S404,E731,D402,IF100,B008 +ignore=W503,R504,D202,S403,S301,S404,E731,D402,IF100,B008 select=B, BLK, C, D, E, F, I, II, N, P, PIE, S, R, W disable-noqa=False hang-closing=False diff --git a/.gitignore b/.gitignore index e3f7a41..c6fd39f 100644 --- a/.gitignore +++ b/.gitignore @@ -13,10 +13,17 @@ old_*.py # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] +.ipynb* # Pyenv .python-version +# MyPy +.mypy_cache + +# Pytest +.pytest_cache + # C extensions *.so diff --git a/hyperglass/api/error_handlers.py b/hyperglass/api/error_handlers.py index 01886a2..d937969 100644 --- a/hyperglass/api/error_handlers.py +++ b/hyperglass/api/error_handlers.py @@ -1,13 +1,14 @@ """API Error Handlers.""" # Third Party +from fastapi import Request from starlette.responses import JSONResponse # Project from hyperglass.state import use_state -async def default_handler(request, exc): +async def default_handler(request: Request, exc: BaseException) -> JSONResponse: """Handle uncaught errors.""" state = use_state() return JSONResponse( @@ -16,7 +17,7 @@ async def default_handler(request, exc): ) -async def http_handler(request, exc): +async def http_handler(request: Request, exc: BaseException) -> JSONResponse: """Handle web server errors.""" return JSONResponse( @@ -25,7 +26,7 @@ async def http_handler(request, exc): ) -async def app_handler(request, exc): +async def app_handler(request: Request, exc: BaseException) -> JSONResponse: """Handle application errors.""" return JSONResponse( {"output": exc.message, "level": exc.level, "keywords": exc.keywords}, @@ -33,7 +34,7 @@ async def app_handler(request, exc): ) -async def validation_handler(request, exc): +async def validation_handler(request: Request, exc: BaseException) -> JSONResponse: """Handle Pydantic validation errors raised by FastAPI.""" error = exc.errors()[0] return JSONResponse( diff --git a/hyperglass/api/routes.py b/hyperglass/api/routes.py index 41dec05..3b9ce9c 100644 --- a/hyperglass/api/routes.py +++ b/hyperglass/api/routes.py @@ -6,8 +6,7 @@ import typing as t from datetime import datetime # Third Party -from fastapi import Depends, HTTPException, BackgroundTasks -from starlette.requests import Request +from fastapi import Depends, Request, HTTPException, BackgroundTasks from fastapi.openapi.docs import get_redoc_html, get_swagger_ui_html # Project diff --git a/hyperglass/configuration/__init__.py b/hyperglass/configuration/__init__.py index 0d52baa..c0334ba 100644 --- a/hyperglass/configuration/__init__.py +++ b/hyperglass/configuration/__init__.py @@ -1,4 +1,6 @@ """hyperglass Configuration.""" +# Standard Library +import typing as t # Project from hyperglass.state import use_state @@ -7,26 +9,36 @@ from hyperglass.defaults.directives import init_builtin_directives # Local from .validate import init_files, init_params, init_devices, init_ui_params, init_directives +if t.TYPE_CHECKING: + # Project + from hyperglass.models.directive import Directives + from hyperglass.models.config.params import Params + from hyperglass.models.config.devices import Devices + __all__ = ("init_user_config",) -def init_user_config() -> None: +def init_user_config( + params: t.Optional["Params"] = None, + directives: t.Optional["Directives"] = None, + devices: t.Optional["Devices"] = None, +) -> None: """Initialize all user configurations and add them to global state.""" state = use_state() init_files() - params = init_params() + _params = params or init_params() builtins = init_builtin_directives() - custom = init_directives() - directives = builtins + custom + _custom = directives or init_directives() + _directives = builtins + _custom with state.cache.pipeline() as pipeline: # Write params and directives to the cache first to avoid a race condition where ui_params # or devices try to access params or directives before they're available. - pipeline.set("params", params) - pipeline.set("directives", directives) + pipeline.set("params", _params) + pipeline.set("directives", _directives) - devices = init_devices() - ui_params = init_ui_params(params=params, devices=devices) + _devices = devices or init_devices() + ui_params = init_ui_params(params=_params, devices=_devices) with state.cache.pipeline() as pipeline: - pipeline.set("devices", devices) + pipeline.set("devices", _devices) pipeline.set("ui_params", ui_params) diff --git a/hyperglass/execution/drivers/_construct.py b/hyperglass/execution/drivers/_construct.py index 2c075da..999feb7 100644 --- a/hyperglass/execution/drivers/_construct.py +++ b/hyperglass/execution/drivers/_construct.py @@ -19,10 +19,12 @@ from hyperglass.exceptions.private import ConfigError if t.TYPE_CHECKING: # Project - from hyperglass.models.api.query import Query + from hyperglass.models.api.query import Query, QueryTarget from hyperglass.models.directive import Directive from hyperglass.models.config.devices import Device +FormatterCallback = t.Callable[[str], "QueryTarget"] + class Construct: """Construct SSH commands/REST API parameters from validated query data.""" @@ -54,8 +56,21 @@ class Construct: if self.device.platform in TARGET_FORMAT_SPACE: self.target = re.sub(r"\/", r" ", str(self.query.query_target)) - with Formatter(self.device.platform, self.query.query_type) as formatter: - self.target = formatter(self.query.query_target) + with Formatter(self.query) as formatter: + self.target = formatter(self.prepare_target()) + + def prepare_target(self) -> "QueryTarget": + """Format the query target based on directive parameters.""" + if isinstance(self.query.query_target, t.List): + # Directive can accept multiple values in a single command. + if self.directive.multiple: + return self.directive.multiple_separator.join(self.query.query_target) + # Target is an array of one, return single item. + if len(self.query.query_target) == 1: + return self.query.query_target[0] + # Directive commands should be run once for each item in the target. + + return self.query.query_target def json(self, afi): """Return JSON version of validated query for REST devices.""" @@ -107,10 +122,11 @@ class Construct: class Formatter: """Modify query target based on the device's NOS requirements and the query type.""" - def __init__(self, platform: str, query_type: str) -> None: + def __init__(self, query: "Query") -> None: """Initialize target formatting.""" - self.platform = platform - self.query_type = query_type + self.query = query + self.platform = query.device.platform + self.query_type = query.query_type def __enter__(self): """Get the relevant formatter.""" @@ -125,18 +141,25 @@ class Formatter: def _get_formatter(self): if self.platform in ("juniper", "juniper_junos"): if self.query_type == "bgp_aspath": - return self._juniper_bgp_aspath + return self._with_formatter(self._juniper_bgp_aspath) if self.platform in ("bird", "bird_ssh"): if self.query_type == "bgp_aspath": - return self._bird_bgp_aspath + return self._with_formatter(self._bird_bgp_aspath) elif self.query_type == "bgp_community": - return self._bird_bgp_community - return self._default + return self._with_formatter(self._bird_bgp_community) + return self._with_formatter(self._default) def _default(self, target: str) -> str: """Don't format targets by default.""" return target + def _with_formatter(self, formatter: t.Callable[[str], str]) -> FormatterCallback: + result: FormatterCallback + if isinstance(self.query.query_target, t.List): + result = lambda s: [formatter(i) for i in s] + result = lambda s: formatter(s) + return result + def _juniper_bgp_aspath(self, target: str) -> str: """Convert from Cisco AS_PATH format to Juniper format.""" query = str(target) diff --git a/hyperglass/execution/drivers/tests/__init__.py b/hyperglass/execution/drivers/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/hyperglass/execution/drivers/tests/test_construct.py b/hyperglass/execution/drivers/tests/test_construct.py new file mode 100644 index 0000000..a45c332 --- /dev/null +++ b/hyperglass/execution/drivers/tests/test_construct.py @@ -0,0 +1,33 @@ +# Project +from hyperglass.models.api import Query +from hyperglass.configuration import init_user_config +from hyperglass.models.directive import Directives +from hyperglass.models.config.devices import Devices + +# Local +from .._construct import Construct + + +def test_construct(): + + devices = Devices( + { + "name": "test1", + "address": "127.0.0.1", + "credential": {"username": "", "password": ""}, + "platform": "juniper", + "attrs": {"source4": "192.0.2.1", "source6": "2001:db8::1"}, + "directives": ["juniper_bgp_route"], + } + ) + directives = Directives( + {"juniper_bgp_route": {"name": "BGP Route", "plugins": [], "rules": [], "groups": []}} + ) + init_user_config(devices=devices, directives=directives) + query = Query( + queryLocation="test1", + queryTarget="192.0.2.0/24", + queryType="juniper_bgp_route", + ) + constructor = Construct(device=devices["test1"], query=query) + assert constructor.target == "192.0.2.0/24" diff --git a/hyperglass/models/api/query.py b/hyperglass/models/api/query.py index eb33913..fd32d54 100644 --- a/hyperglass/models/api/query.py +++ b/hyperglass/models/api/query.py @@ -7,7 +7,7 @@ import secrets from datetime import datetime # Third Party -from pydantic import BaseModel, StrictStr, constr, validator +from pydantic import BaseModel, constr, validator # Project from hyperglass.log import log @@ -22,56 +22,46 @@ from ..config.devices import Device (TEXT := use_state("params").web.text) +QueryLocation = constr(strip_whitespace=True, strict=True, min_length=1) QueryTarget = constr(strip_whitespace=True, min_length=1) +QueryType = constr(strip_whitespace=True, strict=True, min_length=1) class Query(BaseModel): """Validation model for input query parameters.""" - query_location: StrictStr # Device `name` field - query_type: StrictStr # Directive `id` field + query_location: QueryLocation # Device `name` field query_target: t.Union[t.List[QueryTarget], QueryTarget] + query_type: QueryType # Directive `id` field class Config: """Pydantic model configuration.""" extra = "allow" alias_generator = snake_to_camel - fields = { - "query_location": { - "title": TEXT.query_location, - "description": "Router/Location Name", - "example": "router01", - }, - "query_type": { - "title": TEXT.query_type, - "description": "Type of Query to Execute", - "example": "bgp_route", - }, - "query_target": { - "title": TEXT.query_target, - "description": "IP Address, Community, or AS Path", - "example": "1.1.1.0/24", - }, - } - schema_extra = {"x-code-samples": [{"lang": "Python", "source": "print('stuff')"}]} + allow_population_by_field_name = True - def __init__(self, **kwargs): + def __init__(self, **data) -> None: """Initialize the query with a UTC timestamp at initialization time.""" - super().__init__(**kwargs) + super().__init__(**data) self.timestamp = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S") + state = use_state() self._state = state + query_directives = self.device.directives.matching(self.query_type) + if len(query_directives) < 1: raise QueryTypeNotFound(query_type=self.query_type) + self.directive = query_directives[0] + try: self.validate_query_target() except InputValidationError as err: raise InputInvalid(**err.kwargs) - def __repr__(self): + def __repr__(self) -> str: """Represent only the query fields.""" return repr_from_attrs(self, self.__config__.fields.keys()) @@ -79,17 +69,17 @@ class Query(BaseModel): """Alias __str__ to __repr__.""" return repr(self) - def digest(self): + def digest(self) -> str: """Create SHA256 hash digest of model representation.""" return hashlib.sha256(repr(self).encode()).hexdigest() - def random(self): + def random(self) -> str: """Create a random string to prevent client or proxy caching.""" return hashlib.sha256( secrets.token_bytes(8) + repr(self).encode() + secrets.token_bytes(8) ).hexdigest() - def validate_query_target(self): + def validate_query_target(self) -> None: """Validate a query target after all fields/relationships havebeen initialized.""" # Run config/rule-based validations. self.directive.validate_target(self.query_target) @@ -98,7 +88,7 @@ class Query(BaseModel): manager.execute(query=self) log.debug("Validation passed for query {!r}", self) - def dict(self) -> t.Dict[str, str]: + def dict(self) -> t.Dict[str, t.Union[t.List[str], str]]: """Include only public fields.""" return super().dict(include={"query_location", "query_target", "query_type"}) @@ -107,15 +97,6 @@ class Query(BaseModel): """Get this query's device object by query_location.""" return self._state.devices[self.query_location] - @validator("query_type") - def validate_query_type(cls, value): - """Ensure a requested query type exists.""" - devices = use_state("devices") - if any((device.has_directives(value) for device in devices)): - return value - - raise QueryTypeNotFound(name=value) - @validator("query_location") def validate_query_location(cls, value): """Ensure query_location is defined.""" @@ -126,3 +107,12 @@ class Query(BaseModel): raise QueryLocationNotFound(location=value) return value + + @validator("query_type") + def validate_query_type(cls, value: t.Any): + """Ensure a requested query type exists.""" + devices = use_state("devices") + if any((device.has_directives(value) for device in devices)): + return value + + raise QueryTypeNotFound(query_type=value) diff --git a/hyperglass/models/directive.py b/hyperglass/models/directive.py index 1ab3b8e..7c811ca 100644 --- a/hyperglass/models/directive.py +++ b/hyperglass/models/directive.py @@ -125,9 +125,13 @@ class RuleWithIP(Rule): def validate_target(self, target: "QueryTarget", *, multiple: bool) -> bool: """Validate an IP address target against this rule's conditions.""" + if isinstance(target, t.List): - self._passed = False - raise InputValidationError("Target must be a single value") + if len(target) > 1: + self._passed = False + raise InputValidationError("Target must be a single value") + target = target[0] + try: # Attempt to use IP object factory to create an IP address object valid_target = ip_network(target) @@ -257,6 +261,7 @@ class Directive(HyperglassUniqueModel, unique_by=("id", "table_output")): table_output: t.Optional[StrictStr] groups: t.List[StrictStr] = [] multiple: StrictBool = False + multiple_separator: StrictStr = " " def validate_target(self, target: str) -> bool: """Validate a target against all configured rules.""" diff --git a/hyperglass/models/main.py b/hyperglass/models/main.py index a87615b..cf0ebea 100644 --- a/hyperglass/models/main.py +++ b/hyperglass/models/main.py @@ -302,7 +302,7 @@ class MultiModel(GenericModel, t.Generic[MultiModelT]): `Model` is yielded. """ for search in searches: - pattern = re.compile(fr".*{search}.*", re.IGNORECASE) + pattern = re.compile(rf".*{search}.*", re.IGNORECASE) for item in self: if pattern.match(getattr(item, self.unique_by)): yield item diff --git a/hyperglass/models/ui.py b/hyperglass/models/ui.py index c30f4af..c074347 100644 --- a/hyperglass/models/ui.py +++ b/hyperglass/models/ui.py @@ -34,7 +34,7 @@ class UILocation(HyperglassModel): id: StrictStr name: StrictStr - group: StrictStr + group: Optional[StrictStr] avatar: Optional[StrictStr] description: Optional[StrictStr] directives: List[UIDirective] = [] @@ -43,7 +43,7 @@ class UILocation(HyperglassModel): class UIDevices(HyperglassModel): """UI: Devices.""" - group: StrictStr + group: Optional[StrictStr] locations: List[UILocation] = [] diff --git a/hyperglass/ui/components/looking-glass-form.tsx b/hyperglass/ui/components/looking-glass-form.tsx index 87693fa..bfca496 100644 --- a/hyperglass/ui/components/looking-glass-form.tsx +++ b/hyperglass/ui/components/looking-glass-form.tsx @@ -52,7 +52,7 @@ export const LookingGlassForm = (): JSX.Element => { enforce(data.queryLocation).isArrayOf(enforce.isString()).isNotEmpty(); }); test('queryTarget', noQueryTarget, () => { - enforce(data.queryTarget).longerThan(1); + enforce(data.queryTarget).isArrayOf(enforce.isString()).isNotEmpty(); }); test('queryType', noQueryType, () => { enforce(data.queryType).inside(queryTypes); @@ -62,7 +62,7 @@ export const LookingGlassForm = (): JSX.Element => { const formInstance = useForm({ resolver: vestResolver(formSchema), defaultValues: { - queryTarget: '', + queryTarget: [], queryLocation: [], queryType: '', }, @@ -72,8 +72,8 @@ export const LookingGlassForm = (): JSX.Element => { // const isFqdnQuery = useIsFqdn(form.queryTarget, form.queryType); const isFqdnQuery = useCallback( - (target: string, fieldType: Directive['fieldType'] | null): boolean => - fieldType === 'text' && isFQDN(target), + (target: string | string[], fieldType: Directive['fieldType'] | null): boolean => + typeof target === 'string' && fieldType === 'text' && isFQDN(target), [], ); @@ -138,15 +138,19 @@ export const LookingGlassForm = (): JSX.Element => { } else if (e.field === 'queryType' && isString(e.value)) { setValue('queryType', e.value); setFormValue('queryType', e.value); - if (form.queryTarget !== '') { + if (form.queryTarget.length !== 0) { // Reset queryTarget as well, so that, for example, selecting BGP Community, and selecting // a community, then changing the queryType to BGP Route doesn't preserve the selected // community as the queryTarget. - setFormValue('queryTarget', ''); + setFormValue('queryTarget', []); setTarget({ display: '' }); } - } else if (e.field === 'queryTarget' && isString(e.value)) { - setFormValue('queryTarget', e.value); + } else if (e.field === 'queryTarget') { + if (isString(e.value)) { + setFormValue('queryTarget', [e.value]); + } else if (Array.isArray(e.value)) { + setFormValue('queryTarget', e.value); + } } } @@ -213,7 +217,7 @@ export const LookingGlassForm = (): JSX.Element => { flexDir="column" mr={{ base: 0, lg: 2 }} > - + diff --git a/hyperglass/ui/components/query-target.tsx b/hyperglass/ui/components/query-target.tsx index 0b9a520..8d17aa9 100644 --- a/hyperglass/ui/components/query-target.tsx +++ b/hyperglass/ui/components/query-target.tsx @@ -62,7 +62,7 @@ export const QueryTarget = (props: QueryTargetProps): JSX.Element => { function handleInputChange(e: React.ChangeEvent): void { setTarget({ display: e.target.value }); - onChange({ field: name, value: e.target.value }); + onChange({ field: name, value: [e.target.value] }); } const handleSelectChange: SelectOnChange = e => { diff --git a/hyperglass/ui/components/resolved-target.tsx b/hyperglass/ui/components/resolved-target.tsx index 6ce8a6a..9130d6e 100644 --- a/hyperglass/ui/components/resolved-target.tsx +++ b/hyperglass/ui/components/resolved-target.tsx @@ -58,7 +58,7 @@ export const ResolvedTarget = (props: ResolvedTargetProps): JSX.Element => { const answer6 = useMemo(() => findAnswer(data6), [data6]); function selectTarget(value: string): void { - setFormValue('queryTarget', value); + setFormValue('queryTarget', [value]); setStatus('results'); } diff --git a/hyperglass/ui/components/results/tags.tsx b/hyperglass/ui/components/results/tags.tsx index 867cd7f..d03c9a3 100644 --- a/hyperglass/ui/components/results/tags.tsx +++ b/hyperglass/ui/components/results/tags.tsx @@ -84,7 +84,7 @@ export const Tags = (): JSX.Element => { >