mirror/checktheroads-hyperglass
1
0
Fork 0
mirror of https://github.com/checktheroads/hyperglass synced 2024-05-11 05:55:08 +00:00
Code Releases Activity

Plugin/directive fixes

Browse Source
This commit is contained in:
thatmattlove
2021-09-21 07:54:16 -07:00
parent e62af507ee
commit 7d5d64c0e2
14 changed files with 113 additions and 131 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
hyperglass/api/__init__.py
View File

@@ -250,7 +250,6 @@ app.add_api_route(
if STATE.params.docs.enable:
app.add_api_route(path=STATE.params.docs.uri, endpoint=docs, include_in_schema=False)
app.openapi = _custom_openapi
log.debug("API Docs config: {}", app.openapi())
app.mount("/images", StaticFiles(directory=IMAGES_DIR), name="images")
app.mount("/custom", StaticFiles(directory=CUSTOM_DIR), name="custom")

40
hyperglass/api/routes.py
View File

@@ -1,7 +1,6 @@
"""API Routes."""
# Standard Library
import json
import time
import typing as t
from datetime import datetime
@@ -20,6 +19,8 @@ from hyperglass.constants import __version__
from hyperglass.models.ui import UIParameters
from hyperglass.exceptions import HyperglassError
from hyperglass.models.api import Query
from hyperglass.models.data import OutputDataModel
from hyperglass.util.typing import is_type
from hyperglass.execution.main import execute
from hyperglass.models.config.params import Params
from hyperglass.models.config.devices import Devices
@@ -103,20 +104,11 @@ async def query(
log.info("Starting query execution for query {}", query_data.summary)
cache_response = cache.get_map(cache_key, "output")
json_output = False
if query_data.device.structured_output and query_data.query_type in (
"bgp_route",
"bgp_community",
"bgp_aspath",
):
json_output = True
cached = False
runtime = 65535
if cache_response:
log.debug("Query {} exists in cache", cache_key)
log.debug("Query {!r} exists in cache", query_data)
# If a cached response exists, reset the expiration time.
cache.expire(cache_key, expire_in=state.params.cache.timeout)
@@ -126,8 +118,7 @@ async def query(
timestamp = cache.get_map(cache_key, "timestamp")
elif not cache_response:
log.debug("No existing cache entry for query {}", cache_key)
log.debug("Created new cache key {} entry for query {}", cache_key, query_data.summary)
log.debug("Created new cache entry {} entry for query {!r}", cache_key, query_data)
timestamp = query_data.timestamp
@@ -135,40 +126,43 @@ async def query(
if state.params.fake_output:
# Return fake, static data for development purposes, if enabled.
cache_output = await fake_output(json_output)
output = await fake_output(True)
else:
# Pass request to execution module
cache_output = await execute(query_data)
output = await execute(query_data)
endtime = time.time()
elapsedtime = round(endtime - starttime, 4)
log.debug("Query {} took {} seconds to run.", cache_key, elapsedtime)
log.debug("{!r} took {} seconds to run", query_data, elapsedtime)
if cache_output is None:
if output is None:
raise HyperglassError(message=state.params.messages.general, alert="danger")
# Create a cache entry
json_output = is_type(output, OutputDataModel)
if json_output:
raw_output = json.dumps(cache_output)
raw_output = output.export_dict()
else:
raw_output = str(cache_output)
raw_output = str(output)
cache.set_map_item(cache_key, "output", raw_output)
cache.set_map_item(cache_key, "timestamp", timestamp)
cache.expire(cache_key, expire_in=state.params.cache.timeout)
log.debug("Added cache entry for query: {}", cache_key)
log.debug("Added cache entry for query {!r}", query_data)
runtime = int(round(elapsedtime, 0))
# If it does, return the cached entry
cache_response = cache.get_map(cache_key, "output")
json_output = is_type(cache_response, t.Dict)
response_format = "text/plain"
if json_output:
response_format = "application/json"
log.debug("Cache match for {}:\n{}", cache_key, cache_response)
log.success("Completed query execution for query {}", query_data.summary)
log.success("Completed query execution for query {!r}", query_data)
return {
"output": cache_response,

22
hyperglass/defaults/directives/juniper.py
View File

@@ -21,15 +21,16 @@ JuniperBGPRoute = BuiltinDirective(
Rule(
condition="0.0.0.0/0",
action="permit",
command="show route protocol table inet.0 {target} detail",
command="show route protocol bgp table inet.0 {target} detail",
),
Rule(
condition="::/0",
action="permit",
command="show route protocol table inet6.0 {target} detail",
command="show route protocol bgp table inet6.0 {target} detail",
),
],
field=Text(description="IP Address, Prefix, or Hostname"),
table_output="__hyperglass_juniper_bgp_route_table__",
platforms=["juniper"],
)
@@ -41,12 +42,13 @@ JuniperBGPASPath = BuiltinDirective(
condition="*",
action="permit",
commands=[
'show route protocol table inet.0 aspath-regex "{target}"',
'show route protocol table inet6.0 aspath-regex "{target}"',
'show route protocol bgp table inet.0 aspath-regex "{target}"',
'show route protocol bgp table inet6.0 aspath-regex "{target}"',
],
)
],
field=Text(description="AS Path Regular Expression"),
table_output="__hyperglass_juniper_bgp_aspath_table__",
platforms=["juniper"],
)
@@ -58,12 +60,13 @@ JuniperBGPCommunity = BuiltinDirective(
condition="*",
action="permit",
commands=[
'show route protocol table inet.0 community "{target}" detail',
'show route protocol table inet6.0 community "{target}" detail',
'show route protocol bgp table inet.0 community "{target}" detail',
'show route protocol bgp table inet6.0 community "{target}" detail',
],
)
],
field=Text(description="AS Path Regular Expression"),
field=Text(description="BGP Community String"),
table_output="__hyperglass_juniper_bgp_community_table__",
platforms=["juniper"],
)
@@ -124,7 +127,6 @@ JuniperBGPRouteTable = BuiltinDirective(
),
],
field=Text(description="IP Address, Prefix, or Hostname"),
table_output=True,
platforms=["juniper"],
)
@@ -142,7 +144,6 @@ JuniperBGPASPathTable = BuiltinDirective(
)
],
field=Text(description="AS Path Regular Expression"),
table_output=True,
platforms=["juniper"],
)
@@ -159,7 +160,6 @@ JuniperBGPCommunityTable = BuiltinDirective(
],
)
],
field=Text(description="AS Path Regular Expression"),
table_output=True,
field=Text(description="BGP Community String"),
platforms=["juniper"],
)

4
hyperglass/execution/drivers/_common.py
View File

@@ -5,7 +5,6 @@ import typing as t
from abc import ABC, abstractmethod
# Project
from hyperglass.log import log
from hyperglass.types import Series
from hyperglass.plugins import OutputPluginManager
@@ -41,12 +40,9 @@ class Connection(ABC):
async def response(self, output: Series[str]) -> t.Union["OutputDataModel", str]:
"""Send output through common parsers."""
log.debug("Pre-parsed responses:\n{}", output)
response = self.plugin_manager.execute(output=output, query=self.query_data)
if response is None:
response = ()
log.debug("Post-parsed responses:\n{}", response)
return response

1
hyperglass/execution/drivers/ssh_netmiko.py
View File

@@ -97,7 +97,6 @@ class NetmikoConnection(SSHConnection):
for query in self.query:
raw = nm_connect_direct.send_command(query, **send_args)
responses += (raw,)
log.debug(f'Raw response for command "{query}":\n{raw}')
nm_connect_direct.disconnect()

1
hyperglass/execution/main.py
View File

@@ -89,7 +89,6 @@ async def execute(query: "Query") -> Union["OutputDataModel", str]:
if not output:
raise ResponseEmpty(query=query)
log.debug("Output for query {!r}:\n{!r}", query, output)
signal.alarm(0)
return output

31
hyperglass/models/api/query.py
View File

@@ -17,7 +17,6 @@ from hyperglass.state import use_state
from hyperglass.exceptions.public import (
InputInvalid,
QueryTypeNotFound,
QueryGroupNotFound,
QueryLocationNotFound,
)
from hyperglass.exceptions.private import InputValidationError
@@ -36,7 +35,7 @@ class Query(BaseModel):
# Directive `id` field
query_type: StrictStr
# Directive `groups` member
query_group: Optional[StrictStr]
query_group: Optional[StrictStr] = None
query_target: constr(strip_whitespace=True, min_length=1)
class Config:
@@ -74,12 +73,10 @@ class Query(BaseModel):
self.timestamp = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
state = use_state()
self._state = state
for command in self.device.commands:
if command.id == self.query_type:
self.directive = command
break
else:
query_directives = self.device.directives.matching(self.query_type)
if len(query_directives) < 1:
raise QueryTypeNotFound(query_type=self.query_type)
self.directive = query_directives[0]
try:
self.validate_query_target()
except InputValidationError as err:
@@ -151,10 +148,7 @@ class Query(BaseModel):
def validate_query_type(cls, value):
"""Ensure a requested query type exists."""
devices = use_state("devices")
directive_ids = [
directive.id for device in devices.objects for directive in device.commands
]
if value in directive_ids:
if any((device.has_directives(value) for device in devices.objects)):
return value
raise QueryTypeNotFound(name=value)
@@ -171,18 +165,3 @@ class Query(BaseModel):
raise QueryLocationNotFound(location=value)
return value
@validator("query_group")
def validate_query_group(cls, value):
"""Ensure query_group is defined."""
devices = use_state("devices")
groups = {
group
for device in devices.objects
for directive in device.commands
for group in directive.groups
}
if value in groups:
return value
raise QueryGroupNotFound(group=value)

36
hyperglass/models/config/devices.py
View File

@@ -7,7 +7,7 @@ from pathlib import Path
from ipaddress import IPv4Address, IPv6Address
# Third Party
from pydantic import StrictInt, StrictStr, StrictBool, validator, root_validator
from pydantic import StrictInt, StrictStr, StrictBool, validator
# Project
from hyperglass.log import log
@@ -53,8 +53,8 @@ class Device(HyperglassModelWithId, extra="allow"):
port: StrictInt = 22
ssl: Optional[Ssl]
platform: StrictStr
directives: Directives
structured_output: Optional[StrictBool]
directives: Directives = Directives()
driver: Optional[SupportedDriver]
attrs: Dict[str, str] = {}
@@ -190,32 +190,35 @@ class Device(HyperglassModelWithId, extra="allow"):
value.cert = cert_file
return value
@root_validator(pre=True)
def validate_device(cls, values: Dict[str, Any]) -> Dict[str, Any]:
@validator("platform", pre=True, always=True)
def validate_platform(cls: "Device", value: Any, values: Dict[str, Any]) -> str:
"""Validate & rewrite device platform, set default `directives`."""
platform = values.get("platform")
if platform is None:
if value is None:
# Ensure device platform is defined.
raise ConfigError(
"Device '{device}' is missing a 'platform' (Network Operating System) property",
device={values["name"]},
)
if platform in SCRAPE_HELPERS.keys():
# Rewrite NOS to helper value if needed.
platform = SCRAPE_HELPERS[platform]
if value in SCRAPE_HELPERS.keys():
# Rewrite platform to helper value if needed.
value = SCRAPE_HELPERS[value]
# Verify device platform is supported by hyperglass.
supported, _ = validate_platform(platform)
supported, _ = validate_platform(value)
if not supported:
raise UnsupportedDevice(platform)
values["platform"] = platform
raise UnsupportedDevice(value)
return value
@validator("directives", pre=True, always=True)
def validate_directives(cls: "Device", value, values) -> "Directives":
"""Associate directive IDs to loaded directive objects."""
directives = use_state("directives")
directive_ids = values.get("directives", [])
directive_ids = value or []
structured_output = values.get("structured_output", False)
platform = values.get("platform")
# Directive options
directive_options = DirectiveOptions(
@@ -236,7 +239,7 @@ class Device(HyperglassModelWithId, extra="allow"):
# Directives matching provided IDs.
device_directives = directives.filter(*directive_ids)
# Matching built-in directives for this device's platform.
builtins = directives.device_builtins(platform=platform)
builtins = directives.device_builtins(platform=platform, table_output=structured_output)
if directive_options.builtins is True:
# Add all builtins.
@@ -245,8 +248,7 @@ class Device(HyperglassModelWithId, extra="allow"):
# If the user provides a list of builtin directives to include, add only those.
device_directives += builtins.matching(*directive_options.builtins)
values["directives"] = device_directives
return values
return device_directives
@validator("driver")
def validate_driver(cls, value: Optional[str], values: Dict) -> Dict:

2
hyperglass/models/data/__init__.py
View File

@@ -6,7 +6,7 @@ from typing import Union
# Local
from .bgp_route import BGPRouteTable
OutputDataModel = Union["BGPRouteTable"]
OutputDataModel = Union[BGPRouteTable]
__all__ = (
"BGPRouteTable",

24
hyperglass/models/directive.py
View File

@@ -23,7 +23,7 @@ from hyperglass.settings import Settings
from hyperglass.exceptions.private import InputValidationError
# Local
from .main import MultiModel, HyperglassModel, HyperglassModelWithId
from .main import MultiModel, HyperglassModel, HyperglassUniqueModel
from .fields import Action
if t.TYPE_CHECKING:
@@ -227,7 +227,7 @@ class RuleWithoutValidation(Rule):
RuleType = t.Union[RuleWithIPv4, RuleWithIPv6, RuleWithPattern, RuleWithoutValidation]
class Directive(HyperglassModelWithId):
class Directive(HyperglassUniqueModel, unique_by=("id", "table_output")):
"""A directive contains commands that can be run on a device, as long as defined rules are met."""
__hyperglass_builtin__: t.ClassVar[bool] = False
@@ -239,10 +239,8 @@ class Directive(HyperglassModelWithId):
info: t.Optional[FilePath]
plugins: t.List[StrictStr] = []
disable_builtins: StrictBool = False
table_output: StrictBool = False
groups: t.List[
StrictStr
] = [] # TODO: Flesh this out. Replace VRFs, but use same logic in React to filter available commands for multi-device queries.
table_output: t.Optional[StrictStr]
groups: t.List[StrictStr] = []
def validate_target(self, target: str) -> bool:
"""Validate a target against all configured rules."""
@@ -305,7 +303,7 @@ class Directive(HyperglassModelWithId):
return value
class BuiltinDirective(Directive):
class BuiltinDirective(Directive, unique_by=("id", "table_output", "platforms")):
"""Natively-supported directive."""
__hyperglass_builtin__: t.ClassVar[bool] = True
@@ -318,13 +316,21 @@ DirectiveT = t.Union[BuiltinDirective, Directive]
class Directives(MultiModel[Directive], model=Directive, unique_by="id"):
"""Collection of directives."""
def device_builtins(self, *, platform: str):
def device_builtins(self, *, platform: str, table_output: bool):
"""Get builtin directives for a device."""
return Directives(
*(
directive
self.table_if_available(directive) if table_output else directive # noqa: IF100 GFY
for directive in self
if directive.__hyperglass_builtin__ is True
and platform in getattr(directive, "platforms", ())
)
)
def table_if_available(self, directive: "Directive") -> "Directive":
"""Get the table-output variant of a directive if it exists."""
for _directive in self:
if _directive.id == directive.table_output:
return _directive
return directive

29
hyperglass/models/main.py
View File

@@ -4,6 +4,7 @@
# Standard Library
import re
import json
import typing as t
from pathlib import Path
@@ -93,6 +94,34 @@ class HyperglassModel(BaseModel):
return yaml.safe_dump(json.loads(self.export_json(**export_kwargs)), *args, **kwargs)
class HyperglassUniqueModel(HyperglassModel):
"""hyperglass model that is unique by its `id` field."""
_unique_fields: t.ClassVar[Series[str]] = ()
def __init_subclass__(cls, *, unique_by: Series[str], **kw: t.Any) -> None:
"""Assign unique fields to class."""
cls._unique_fields = tuple(unique_by)
return super().__init_subclass__(**kw)
def __eq__(self: "HyperglassUniqueModel", other: "HyperglassUniqueModel") -> bool:
"""Other model is equal to this model."""
if not isinstance(other, self.__class__):
return False
if hash(self) == hash(other):
return True
return False
def __ne__(self: "HyperglassUniqueModel", other: "HyperglassUniqueModel") -> bool:
"""Other model is not equal to this model."""
return not self.__eq__(other)
def __hash__(self: "HyperglassUniqueModel") -> int:
"""Create a hashed representation of this model's name."""
fields = dict(zip(self._unique_fields, (getattr(self, f) for f in self._unique_fields)))
return hash(json.dumps(fields))
class HyperglassModelWithId(HyperglassModel):
"""hyperglass model that is unique by its `id` field."""

4
hyperglass/models/tests/test_multi_model.py
View File

@@ -4,7 +4,7 @@
from pydantic import BaseModel
# Local
from ..main import HyperglassMultiModel
from ..main import MultiModel
class Item(BaseModel):
@@ -32,7 +32,7 @@ ITEMS_3 = [
def test_multi_model():
model = HyperglassMultiModel(*ITEMS_1, model=Item, accessor="id")
model = MultiModel(*ITEMS_1, model=Item, accessor="id")
assert model.count == 3
assert len([o for o in model]) == model.count # noqa: C416 (Iteration testing)
assert model["item1"].name == "Item One"

17
hyperglass/plugins/_manager.py
View File

@@ -50,14 +50,13 @@ class PluginManager(t.Generic[PluginT]):
def __next__(self: "PluginManager") -> PluginT:
"""Plugin manager iteration."""
if self._index <= len(self.plugins):
result = self.plugins[self._index - 1]
if self._index <= len(self.plugins()):
result = self.plugins()[self._index - 1]
self._index += 1
return result
self._index = 0
raise StopIteration
@property
def plugins(self: "PluginManager", builtins: bool = True) -> t.List[PluginT]:
"""Get all plugins, with built-in plugins last."""
plugins = self._state.plugins(self._type)
@@ -81,7 +80,7 @@ class PluginManager(t.Generic[PluginT]):
def methods(self: "PluginManager", name: str) -> t.Generator[t.Callable, None, None]:
"""Get methods of all registered plugins matching `name`."""
for plugin in self.plugins:
for plugin in self.plugins():
if hasattr(plugin, name):
method = getattr(plugin, name)
if callable(method):
@@ -137,7 +136,7 @@ class InputPluginManager(PluginManager[InputPlugin], type="input"):
If any plugin returns `False`, execution is halted.
"""
result = None
for plugin in (plugin for plugin in self.plugins if directive.id in plugin.directives):
for plugin in (plugin for plugin in self.plugins() if directive.id in plugin.directives):
if result is False:
return result
result = plugin.validate(query)
@@ -154,10 +153,14 @@ class OutputPluginManager(PluginManager[OutputPlugin], type="output"):
"""
result = output
for plugin in (
plugin for plugin in self.plugins if query.directive.id in plugin.directives
plugin
for plugin in self.plugins()
if query.directive.id in plugin.directives and query.device.platform in plugin.platforms
):
log.debug("Output Plugin {!r} starting with\n{!r}", plugin.name, result)
result = plugin.process(output=result, query=query)
log.debug("Output Plugin {!r} completed with\n{!r}", plugin.name, result)
if result is False:
return result
# Pass the result of each plugin to the next plugin.
result = plugin.process(output=result, query=query)
return result

30
hyperglass/state/store.py
View File

@@ -1,8 +1,6 @@
"""Primary state container."""
# Standard Library
import codecs
import pickle
import typing as t
# Local
@@ -11,7 +9,6 @@ from .manager import StateManager
if t.TYPE_CHECKING:
# Project
from hyperglass.models.ui import UIParameters
from hyperglass.models.system import HyperglassSystem
from hyperglass.plugins._base import HyperglassPlugin
from hyperglass.models.directive import Directive, Directives
from hyperglass.models.config.params import Params
@@ -27,35 +24,15 @@ PluginT = t.TypeVar("PluginT", bound="HyperglassPlugin")
class HyperglassState(StateManager):
"""Primary hyperglass state container."""
def __init__(self, *, settings: "HyperglassSystem") -> None:
"""Initialize state store and reset plugins."""
super().__init__(settings=settings)
# Ensure plugins are empty.
self.reset_plugins("output")
self.reset_plugins("input")
def add_plugin(self, _type: str, plugin: "HyperglassPlugin") -> None:
"""Add a plugin to its list by type."""
current = self.plugins(_type)
plugins = {
# Create a base64 representation of a picked plugin.
codecs.encode(pickle.dumps(p), "base64").decode()
# Merge current plugins with the new plugin.
for p in [*current, plugin]
}
self.redis.set(("plugins", _type), list(plugins))
self.redis.set(("plugins", _type), list({*current, plugin}))
def remove_plugin(self, _type: str, plugin: "HyperglassPlugin") -> None:
"""Remove a plugin from its list by type."""
current = self.plugins(_type)
plugins = {
# Create a base64 representation of a picked plugin.
codecs.encode(pickle.dumps(p), "base64").decode()
# Merge current plugins with the new plugin.
for p in current
if p != plugin
}
plugins = {p for p in current if p != plugin}
self.redis.set(("plugins", _type), list(plugins))
def reset_plugins(self, _type: str) -> None:
@@ -99,5 +76,4 @@ class HyperglassState(StateManager):
def plugins(self, _type: str) -> t.List[PluginT]:
"""Get plugins by type."""
current = self.redis.get(("plugins", _type), raise_if_none=False, value_if_none=[])
return list({pickle.loads(codecs.decode(plugin.encode(), "base64")) for plugin in current})
return self.redis.get(("plugins", _type), raise_if_none=False, value_if_none=[])