diff --git a/hyperglass/api/__init__.py b/hyperglass/api/__init__.py index 1b296ab..ad337a0 100644 --- a/hyperglass/api/__init__.py +++ b/hyperglass/api/__init__.py @@ -250,7 +250,6 @@ app.add_api_route( if STATE.params.docs.enable: app.add_api_route(path=STATE.params.docs.uri, endpoint=docs, include_in_schema=False) app.openapi = _custom_openapi - log.debug("API Docs config: {}", app.openapi()) app.mount("/images", StaticFiles(directory=IMAGES_DIR), name="images") app.mount("/custom", StaticFiles(directory=CUSTOM_DIR), name="custom") diff --git a/hyperglass/api/routes.py b/hyperglass/api/routes.py index b8d5a1d..725a624 100644 --- a/hyperglass/api/routes.py +++ b/hyperglass/api/routes.py @@ -1,7 +1,6 @@ """API Routes.""" # Standard Library -import json import time import typing as t from datetime import datetime @@ -20,6 +19,8 @@ from hyperglass.constants import __version__ from hyperglass.models.ui import UIParameters from hyperglass.exceptions import HyperglassError from hyperglass.models.api import Query +from hyperglass.models.data import OutputDataModel +from hyperglass.util.typing import is_type from hyperglass.execution.main import execute from hyperglass.models.config.params import Params from hyperglass.models.config.devices import Devices @@ -103,20 +104,11 @@ async def query( log.info("Starting query execution for query {}", query_data.summary) cache_response = cache.get_map(cache_key, "output") - json_output = False - - if query_data.device.structured_output and query_data.query_type in ( - "bgp_route", - "bgp_community", - "bgp_aspath", - ): - json_output = True - cached = False runtime = 65535 if cache_response: - log.debug("Query {} exists in cache", cache_key) + log.debug("Query {!r} exists in cache", query_data) # If a cached response exists, reset the expiration time. cache.expire(cache_key, expire_in=state.params.cache.timeout) @@ -126,8 +118,7 @@ async def query( timestamp = cache.get_map(cache_key, "timestamp") elif not cache_response: - log.debug("No existing cache entry for query {}", cache_key) - log.debug("Created new cache key {} entry for query {}", cache_key, query_data.summary) + log.debug("Created new cache entry {} entry for query {!r}", cache_key, query_data) timestamp = query_data.timestamp @@ -135,40 +126,43 @@ async def query( if state.params.fake_output: # Return fake, static data for development purposes, if enabled. - cache_output = await fake_output(json_output) + output = await fake_output(True) else: # Pass request to execution module - cache_output = await execute(query_data) + output = await execute(query_data) endtime = time.time() elapsedtime = round(endtime - starttime, 4) - log.debug("Query {} took {} seconds to run.", cache_key, elapsedtime) + log.debug("{!r} took {} seconds to run", query_data, elapsedtime) - if cache_output is None: + if output is None: raise HyperglassError(message=state.params.messages.general, alert="danger") - # Create a cache entry + json_output = is_type(output, OutputDataModel) + if json_output: - raw_output = json.dumps(cache_output) + raw_output = output.export_dict() else: - raw_output = str(cache_output) + raw_output = str(output) + cache.set_map_item(cache_key, "output", raw_output) cache.set_map_item(cache_key, "timestamp", timestamp) cache.expire(cache_key, expire_in=state.params.cache.timeout) - log.debug("Added cache entry for query: {}", cache_key) + log.debug("Added cache entry for query {!r}", query_data) runtime = int(round(elapsedtime, 0)) # If it does, return the cached entry cache_response = cache.get_map(cache_key, "output") + + json_output = is_type(cache_response, t.Dict) response_format = "text/plain" if json_output: response_format = "application/json" - log.debug("Cache match for {}:\n{}", cache_key, cache_response) - log.success("Completed query execution for query {}", query_data.summary) + log.success("Completed query execution for query {!r}", query_data) return { "output": cache_response, diff --git a/hyperglass/defaults/directives/juniper.py b/hyperglass/defaults/directives/juniper.py index 462469b..6d5ef72 100644 --- a/hyperglass/defaults/directives/juniper.py +++ b/hyperglass/defaults/directives/juniper.py @@ -21,15 +21,16 @@ JuniperBGPRoute = BuiltinDirective( Rule( condition="0.0.0.0/0", action="permit", - command="show route protocol table inet.0 {target} detail", + command="show route protocol bgp table inet.0 {target} detail", ), Rule( condition="::/0", action="permit", - command="show route protocol table inet6.0 {target} detail", + command="show route protocol bgp table inet6.0 {target} detail", ), ], field=Text(description="IP Address, Prefix, or Hostname"), + table_output="__hyperglass_juniper_bgp_route_table__", platforms=["juniper"], ) @@ -41,12 +42,13 @@ JuniperBGPASPath = BuiltinDirective( condition="*", action="permit", commands=[ - 'show route protocol table inet.0 aspath-regex "{target}"', - 'show route protocol table inet6.0 aspath-regex "{target}"', + 'show route protocol bgp table inet.0 aspath-regex "{target}"', + 'show route protocol bgp table inet6.0 aspath-regex "{target}"', ], ) ], field=Text(description="AS Path Regular Expression"), + table_output="__hyperglass_juniper_bgp_aspath_table__", platforms=["juniper"], ) @@ -58,12 +60,13 @@ JuniperBGPCommunity = BuiltinDirective( condition="*", action="permit", commands=[ - 'show route protocol table inet.0 community "{target}" detail', - 'show route protocol table inet6.0 community "{target}" detail', + 'show route protocol bgp table inet.0 community "{target}" detail', + 'show route protocol bgp table inet6.0 community "{target}" detail', ], ) ], - field=Text(description="AS Path Regular Expression"), + field=Text(description="BGP Community String"), + table_output="__hyperglass_juniper_bgp_community_table__", platforms=["juniper"], ) @@ -124,7 +127,6 @@ JuniperBGPRouteTable = BuiltinDirective( ), ], field=Text(description="IP Address, Prefix, or Hostname"), - table_output=True, platforms=["juniper"], ) @@ -142,7 +144,6 @@ JuniperBGPASPathTable = BuiltinDirective( ) ], field=Text(description="AS Path Regular Expression"), - table_output=True, platforms=["juniper"], ) @@ -159,7 +160,6 @@ JuniperBGPCommunityTable = BuiltinDirective( ], ) ], - field=Text(description="AS Path Regular Expression"), - table_output=True, + field=Text(description="BGP Community String"), platforms=["juniper"], ) diff --git a/hyperglass/execution/drivers/_common.py b/hyperglass/execution/drivers/_common.py index 7ff2a54..bd85986 100644 --- a/hyperglass/execution/drivers/_common.py +++ b/hyperglass/execution/drivers/_common.py @@ -5,7 +5,6 @@ import typing as t from abc import ABC, abstractmethod # Project -from hyperglass.log import log from hyperglass.types import Series from hyperglass.plugins import OutputPluginManager @@ -41,12 +40,9 @@ class Connection(ABC): async def response(self, output: Series[str]) -> t.Union["OutputDataModel", str]: """Send output through common parsers.""" - log.debug("Pre-parsed responses:\n{}", output) - response = self.plugin_manager.execute(output=output, query=self.query_data) if response is None: response = () - log.debug("Post-parsed responses:\n{}", response) return response diff --git a/hyperglass/execution/drivers/ssh_netmiko.py b/hyperglass/execution/drivers/ssh_netmiko.py index 57ca5f7..6e6a161 100644 --- a/hyperglass/execution/drivers/ssh_netmiko.py +++ b/hyperglass/execution/drivers/ssh_netmiko.py @@ -97,7 +97,6 @@ class NetmikoConnection(SSHConnection): for query in self.query: raw = nm_connect_direct.send_command(query, **send_args) responses += (raw,) - log.debug(f'Raw response for command "{query}":\n{raw}') nm_connect_direct.disconnect() diff --git a/hyperglass/execution/main.py b/hyperglass/execution/main.py index 6b5f8cc..fc192eb 100644 --- a/hyperglass/execution/main.py +++ b/hyperglass/execution/main.py @@ -89,7 +89,6 @@ async def execute(query: "Query") -> Union["OutputDataModel", str]: if not output: raise ResponseEmpty(query=query) - log.debug("Output for query {!r}:\n{!r}", query, output) signal.alarm(0) return output diff --git a/hyperglass/models/api/query.py b/hyperglass/models/api/query.py index ab8e59b..0c69276 100644 --- a/hyperglass/models/api/query.py +++ b/hyperglass/models/api/query.py @@ -17,7 +17,6 @@ from hyperglass.state import use_state from hyperglass.exceptions.public import ( InputInvalid, QueryTypeNotFound, - QueryGroupNotFound, QueryLocationNotFound, ) from hyperglass.exceptions.private import InputValidationError @@ -36,7 +35,7 @@ class Query(BaseModel): # Directive `id` field query_type: StrictStr # Directive `groups` member - query_group: Optional[StrictStr] + query_group: Optional[StrictStr] = None query_target: constr(strip_whitespace=True, min_length=1) class Config: @@ -74,12 +73,10 @@ class Query(BaseModel): self.timestamp = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S") state = use_state() self._state = state - for command in self.device.commands: - if command.id == self.query_type: - self.directive = command - break - else: - raise QueryTypeNotFound(query_type=self.query_type) + query_directives = self.device.directives.matching(self.query_type) + if len(query_directives) < 1: + raise QueryTypeNotFound(query_type=self.query_type) + self.directive = query_directives[0] try: self.validate_query_target() except InputValidationError as err: @@ -151,10 +148,7 @@ class Query(BaseModel): def validate_query_type(cls, value): """Ensure a requested query type exists.""" devices = use_state("devices") - directive_ids = [ - directive.id for device in devices.objects for directive in device.commands - ] - if value in directive_ids: + if any((device.has_directives(value) for device in devices.objects)): return value raise QueryTypeNotFound(name=value) @@ -171,18 +165,3 @@ class Query(BaseModel): raise QueryLocationNotFound(location=value) return value - - @validator("query_group") - def validate_query_group(cls, value): - """Ensure query_group is defined.""" - devices = use_state("devices") - groups = { - group - for device in devices.objects - for directive in device.commands - for group in directive.groups - } - if value in groups: - return value - - raise QueryGroupNotFound(group=value) diff --git a/hyperglass/models/config/devices.py b/hyperglass/models/config/devices.py index b01c69d..0dce116 100644 --- a/hyperglass/models/config/devices.py +++ b/hyperglass/models/config/devices.py @@ -7,7 +7,7 @@ from pathlib import Path from ipaddress import IPv4Address, IPv6Address # Third Party -from pydantic import StrictInt, StrictStr, StrictBool, validator, root_validator +from pydantic import StrictInt, StrictStr, StrictBool, validator # Project from hyperglass.log import log @@ -53,8 +53,8 @@ class Device(HyperglassModelWithId, extra="allow"): port: StrictInt = 22 ssl: Optional[Ssl] platform: StrictStr - directives: Directives structured_output: Optional[StrictBool] + directives: Directives = Directives() driver: Optional[SupportedDriver] attrs: Dict[str, str] = {} @@ -190,32 +190,35 @@ class Device(HyperglassModelWithId, extra="allow"): value.cert = cert_file return value - @root_validator(pre=True) - def validate_device(cls, values: Dict[str, Any]) -> Dict[str, Any]: + @validator("platform", pre=True, always=True) + def validate_platform(cls: "Device", value: Any, values: Dict[str, Any]) -> str: """Validate & rewrite device platform, set default `directives`.""" - platform = values.get("platform") - if platform is None: + if value is None: # Ensure device platform is defined. raise ConfigError( "Device '{device}' is missing a 'platform' (Network Operating System) property", device={values["name"]}, ) - if platform in SCRAPE_HELPERS.keys(): - # Rewrite NOS to helper value if needed. - platform = SCRAPE_HELPERS[platform] + if value in SCRAPE_HELPERS.keys(): + # Rewrite platform to helper value if needed. + value = SCRAPE_HELPERS[value] # Verify device platform is supported by hyperglass. - supported, _ = validate_platform(platform) + supported, _ = validate_platform(value) if not supported: - raise UnsupportedDevice(platform) - - values["platform"] = platform + raise UnsupportedDevice(value) + return value + @validator("directives", pre=True, always=True) + def validate_directives(cls: "Device", value, values) -> "Directives": + """Associate directive IDs to loaded directive objects.""" directives = use_state("directives") - directive_ids = values.get("directives", []) + directive_ids = value or [] + structured_output = values.get("structured_output", False) + platform = values.get("platform") # Directive options directive_options = DirectiveOptions( @@ -236,7 +239,7 @@ class Device(HyperglassModelWithId, extra="allow"): # Directives matching provided IDs. device_directives = directives.filter(*directive_ids) # Matching built-in directives for this device's platform. - builtins = directives.device_builtins(platform=platform) + builtins = directives.device_builtins(platform=platform, table_output=structured_output) if directive_options.builtins is True: # Add all builtins. @@ -245,8 +248,7 @@ class Device(HyperglassModelWithId, extra="allow"): # If the user provides a list of builtin directives to include, add only those. device_directives += builtins.matching(*directive_options.builtins) - values["directives"] = device_directives - return values + return device_directives @validator("driver") def validate_driver(cls, value: Optional[str], values: Dict) -> Dict: diff --git a/hyperglass/models/data/__init__.py b/hyperglass/models/data/__init__.py index 5afa4d3..02d8f27 100644 --- a/hyperglass/models/data/__init__.py +++ b/hyperglass/models/data/__init__.py @@ -6,7 +6,7 @@ from typing import Union # Local from .bgp_route import BGPRouteTable -OutputDataModel = Union["BGPRouteTable"] +OutputDataModel = Union[BGPRouteTable] __all__ = ( "BGPRouteTable", diff --git a/hyperglass/models/directive.py b/hyperglass/models/directive.py index f145cee..03fb6b8 100644 --- a/hyperglass/models/directive.py +++ b/hyperglass/models/directive.py @@ -23,7 +23,7 @@ from hyperglass.settings import Settings from hyperglass.exceptions.private import InputValidationError # Local -from .main import MultiModel, HyperglassModel, HyperglassModelWithId +from .main import MultiModel, HyperglassModel, HyperglassUniqueModel from .fields import Action if t.TYPE_CHECKING: @@ -227,7 +227,7 @@ class RuleWithoutValidation(Rule): RuleType = t.Union[RuleWithIPv4, RuleWithIPv6, RuleWithPattern, RuleWithoutValidation] -class Directive(HyperglassModelWithId): +class Directive(HyperglassUniqueModel, unique_by=("id", "table_output")): """A directive contains commands that can be run on a device, as long as defined rules are met.""" __hyperglass_builtin__: t.ClassVar[bool] = False @@ -239,10 +239,8 @@ class Directive(HyperglassModelWithId): info: t.Optional[FilePath] plugins: t.List[StrictStr] = [] disable_builtins: StrictBool = False - table_output: StrictBool = False - groups: t.List[ - StrictStr - ] = [] # TODO: Flesh this out. Replace VRFs, but use same logic in React to filter available commands for multi-device queries. + table_output: t.Optional[StrictStr] + groups: t.List[StrictStr] = [] def validate_target(self, target: str) -> bool: """Validate a target against all configured rules.""" @@ -305,7 +303,7 @@ class Directive(HyperglassModelWithId): return value -class BuiltinDirective(Directive): +class BuiltinDirective(Directive, unique_by=("id", "table_output", "platforms")): """Natively-supported directive.""" __hyperglass_builtin__: t.ClassVar[bool] = True @@ -318,13 +316,21 @@ DirectiveT = t.Union[BuiltinDirective, Directive] class Directives(MultiModel[Directive], model=Directive, unique_by="id"): """Collection of directives.""" - def device_builtins(self, *, platform: str): + def device_builtins(self, *, platform: str, table_output: bool): """Get builtin directives for a device.""" + return Directives( *( - directive + self.table_if_available(directive) if table_output else directive # noqa: IF100 GFY for directive in self if directive.__hyperglass_builtin__ is True and platform in getattr(directive, "platforms", ()) ) ) + + def table_if_available(self, directive: "Directive") -> "Directive": + """Get the table-output variant of a directive if it exists.""" + for _directive in self: + if _directive.id == directive.table_output: + return _directive + return directive diff --git a/hyperglass/models/main.py b/hyperglass/models/main.py index 3fda6cf..a87615b 100644 --- a/hyperglass/models/main.py +++ b/hyperglass/models/main.py @@ -4,6 +4,7 @@ # Standard Library import re +import json import typing as t from pathlib import Path @@ -93,6 +94,34 @@ class HyperglassModel(BaseModel): return yaml.safe_dump(json.loads(self.export_json(**export_kwargs)), *args, **kwargs) +class HyperglassUniqueModel(HyperglassModel): + """hyperglass model that is unique by its `id` field.""" + + _unique_fields: t.ClassVar[Series[str]] = () + + def __init_subclass__(cls, *, unique_by: Series[str], **kw: t.Any) -> None: + """Assign unique fields to class.""" + cls._unique_fields = tuple(unique_by) + return super().__init_subclass__(**kw) + + def __eq__(self: "HyperglassUniqueModel", other: "HyperglassUniqueModel") -> bool: + """Other model is equal to this model.""" + if not isinstance(other, self.__class__): + return False + if hash(self) == hash(other): + return True + return False + + def __ne__(self: "HyperglassUniqueModel", other: "HyperglassUniqueModel") -> bool: + """Other model is not equal to this model.""" + return not self.__eq__(other) + + def __hash__(self: "HyperglassUniqueModel") -> int: + """Create a hashed representation of this model's name.""" + fields = dict(zip(self._unique_fields, (getattr(self, f) for f in self._unique_fields))) + return hash(json.dumps(fields)) + + class HyperglassModelWithId(HyperglassModel): """hyperglass model that is unique by its `id` field.""" diff --git a/hyperglass/models/tests/test_multi_model.py b/hyperglass/models/tests/test_multi_model.py index 17a3941..91fa848 100644 --- a/hyperglass/models/tests/test_multi_model.py +++ b/hyperglass/models/tests/test_multi_model.py @@ -4,7 +4,7 @@ from pydantic import BaseModel # Local -from ..main import HyperglassMultiModel +from ..main import MultiModel class Item(BaseModel): @@ -32,7 +32,7 @@ ITEMS_3 = [ def test_multi_model(): - model = HyperglassMultiModel(*ITEMS_1, model=Item, accessor="id") + model = MultiModel(*ITEMS_1, model=Item, accessor="id") assert model.count == 3 assert len([o for o in model]) == model.count # noqa: C416 (Iteration testing) assert model["item1"].name == "Item One" diff --git a/hyperglass/plugins/_manager.py b/hyperglass/plugins/_manager.py index e969793..057fbe4 100644 --- a/hyperglass/plugins/_manager.py +++ b/hyperglass/plugins/_manager.py @@ -50,14 +50,13 @@ class PluginManager(t.Generic[PluginT]): def __next__(self: "PluginManager") -> PluginT: """Plugin manager iteration.""" - if self._index <= len(self.plugins): - result = self.plugins[self._index - 1] + if self._index <= len(self.plugins()): + result = self.plugins()[self._index - 1] self._index += 1 return result self._index = 0 raise StopIteration - @property def plugins(self: "PluginManager", builtins: bool = True) -> t.List[PluginT]: """Get all plugins, with built-in plugins last.""" plugins = self._state.plugins(self._type) @@ -81,7 +80,7 @@ class PluginManager(t.Generic[PluginT]): def methods(self: "PluginManager", name: str) -> t.Generator[t.Callable, None, None]: """Get methods of all registered plugins matching `name`.""" - for plugin in self.plugins: + for plugin in self.plugins(): if hasattr(plugin, name): method = getattr(plugin, name) if callable(method): @@ -137,7 +136,7 @@ class InputPluginManager(PluginManager[InputPlugin], type="input"): If any plugin returns `False`, execution is halted. """ result = None - for plugin in (plugin for plugin in self.plugins if directive.id in plugin.directives): + for plugin in (plugin for plugin in self.plugins() if directive.id in plugin.directives): if result is False: return result result = plugin.validate(query) @@ -154,10 +153,14 @@ class OutputPluginManager(PluginManager[OutputPlugin], type="output"): """ result = output for plugin in ( - plugin for plugin in self.plugins if query.directive.id in plugin.directives + plugin + for plugin in self.plugins() + if query.directive.id in plugin.directives and query.device.platform in plugin.platforms ): + log.debug("Output Plugin {!r} starting with\n{!r}", plugin.name, result) + result = plugin.process(output=result, query=query) + log.debug("Output Plugin {!r} completed with\n{!r}", plugin.name, result) if result is False: return result # Pass the result of each plugin to the next plugin. - result = plugin.process(output=result, query=query) return result diff --git a/hyperglass/state/store.py b/hyperglass/state/store.py index 486929d..9414adc 100644 --- a/hyperglass/state/store.py +++ b/hyperglass/state/store.py @@ -1,8 +1,6 @@ """Primary state container.""" # Standard Library -import codecs -import pickle import typing as t # Local @@ -11,7 +9,6 @@ from .manager import StateManager if t.TYPE_CHECKING: # Project from hyperglass.models.ui import UIParameters - from hyperglass.models.system import HyperglassSystem from hyperglass.plugins._base import HyperglassPlugin from hyperglass.models.directive import Directive, Directives from hyperglass.models.config.params import Params @@ -27,35 +24,15 @@ PluginT = t.TypeVar("PluginT", bound="HyperglassPlugin") class HyperglassState(StateManager): """Primary hyperglass state container.""" - def __init__(self, *, settings: "HyperglassSystem") -> None: - """Initialize state store and reset plugins.""" - super().__init__(settings=settings) - - # Ensure plugins are empty. - self.reset_plugins("output") - self.reset_plugins("input") - def add_plugin(self, _type: str, plugin: "HyperglassPlugin") -> None: """Add a plugin to its list by type.""" current = self.plugins(_type) - plugins = { - # Create a base64 representation of a picked plugin. - codecs.encode(pickle.dumps(p), "base64").decode() - # Merge current plugins with the new plugin. - for p in [*current, plugin] - } - self.redis.set(("plugins", _type), list(plugins)) + self.redis.set(("plugins", _type), list({*current, plugin})) def remove_plugin(self, _type: str, plugin: "HyperglassPlugin") -> None: """Remove a plugin from its list by type.""" current = self.plugins(_type) - plugins = { - # Create a base64 representation of a picked plugin. - codecs.encode(pickle.dumps(p), "base64").decode() - # Merge current plugins with the new plugin. - for p in current - if p != plugin - } + plugins = {p for p in current if p != plugin} self.redis.set(("plugins", _type), list(plugins)) def reset_plugins(self, _type: str) -> None: @@ -99,5 +76,4 @@ class HyperglassState(StateManager): def plugins(self, _type: str) -> t.List[PluginT]: """Get plugins by type.""" - current = self.redis.get(("plugins", _type), raise_if_none=False, value_if_none=[]) - return list({pickle.loads(codecs.decode(plugin.encode(), "base64")) for plugin in current}) + return self.redis.get(("plugins", _type), raise_if_none=False, value_if_none=[])