diff --git a/hyperglass/api/__init__.py b/hyperglass/api/__init__.py index a9de039..3bdd7d3 100644 --- a/hyperglass/api/__init__.py +++ b/hyperglass/api/__init__.py @@ -242,15 +242,13 @@ app.add_api_route( # Enable certificate import route only if a device using # hyperglass-agent is defined. -for device in devices.routers: - if device.nos in TRANSPORT_REST: - app.add_api_route( - path="/api/import-agent-certificate/", - endpoint=import_certificate, - methods=["POST"], - include_in_schema=False, - ) - break +if [n for n in devices.all_nos if n in TRANSPORT_REST]: + app.add_api_route( + path="/api/import-agent-certificate/", + endpoint=import_certificate, + methods=["POST"], + include_in_schema=False, + ) if params.docs.enable: app.add_api_route(path=params.docs.uri, endpoint=docs, include_in_schema=False) diff --git a/hyperglass/api/models/query.py b/hyperglass/api/models/query.py index ea1f5a1..bd74c2d 100644 --- a/hyperglass/api/models/query.py +++ b/hyperglass/api/models/query.py @@ -124,16 +124,19 @@ class Query(BaseModel): @property def device(self): """Get this query's device object by query_location.""" - return getattr(devices, self.query_location) + return devices[self.query_location] + + @property + def query(self): + """Get this query's configuration object.""" + return params.queries[self.query_type] def export_dict(self, pretty=False): """Create dictionary representation of instance.""" if pretty: - loc = getattr(devices, self.query_location) - query_type = getattr(params.queries, self.query_type) items = { - "query_location": loc.display_name, - "query_type": query_type.display_name, + "query_location": self.device.display_name, + "query_type": self.query.display_name, "query_vrf": self.query_vrf.display_name, "query_target": str(self.query_target), } @@ -163,12 +166,12 @@ class Query(BaseModel): Returns: {str} -- Valid query_type """ - query_type_obj = getattr(params.queries, value) - if not query_type_obj.enable: + query = params.queries[value] + if not query.enable: raise InputInvalid( params.messages.feature_not_enabled, level="warning", - feature=query_type_obj.display_name, + feature=query.display_name, ) return value @@ -208,7 +211,7 @@ class Query(BaseModel): {str} -- Valid query_vrf """ vrf_object = get_vrf_object(value) - device = getattr(devices, values["query_location"]) + device = devices[values["query_location"]] device_vrf = None for vrf in device.vrfs: if vrf == vrf_object: diff --git a/hyperglass/api/routes.py b/hyperglass/api/routes.py index 3f0d5df..76fd4fd 100644 --- a/hyperglass/api/routes.py +++ b/hyperglass/api/routes.py @@ -13,7 +13,6 @@ from fastapi.openapi.docs import get_redoc_html, get_swagger_ui_html # Project from hyperglass.log import log -from hyperglass.util import clean_name from hyperglass.cache import AsyncCache from hyperglass.encode import jwt_decode from hyperglass.external import Webhook, bgptools @@ -167,14 +166,9 @@ async def import_certificate(encoded_request: EncodedRequest): """Import a certificate from hyperglass-agent.""" # Try to match the requested device name with configured devices - matched_device = None - requested_device_name = clean_name(encoded_request.device) - for device in devices.routers: - if device.name == requested_device_name: - matched_device = device - break - - if matched_device is None: + try: + matched_device = devices[encoded_request.device] + except AttributeError: raise HTTPException( detail=f"Device {str(encoded_request.device)} not found", status_code=404 ) @@ -191,10 +185,12 @@ async def import_certificate(encoded_request: EncodedRequest): try: # Write certificate to file import_public_key( - app_path=APP_PATH, device_name=device.name, keystring=decoded_request + app_path=APP_PATH, + device_name=matched_device.name, + keystring=decoded_request, ) - except RuntimeError as import_error: - raise HyperglassError(str(import_error), level="danger") + except RuntimeError as err: + raise HyperglassError(str(err), level="danger") return { "output": f"Added public key for {encoded_request.device}", @@ -226,7 +222,7 @@ async def routers(): "vrfs": {-1: {"name", "display_name"}}, } ) - for d in devices.routers + for d in devices.objects ] diff --git a/hyperglass/configuration/__init__.py b/hyperglass/configuration/__init__.py index eea268f..0ce7a6e 100644 --- a/hyperglass/configuration/__init__.py +++ b/hyperglass/configuration/__init__.py @@ -28,9 +28,6 @@ from hyperglass.constants import ( __version__, ) from hyperglass.exceptions import ConfigError, ConfigInvalid, ConfigMissing -from hyperglass.configuration.models import params as _params -from hyperglass.configuration.models import routers as _routers -from hyperglass.configuration.models import commands as _commands from hyperglass.configuration.defaults import ( CREDIT, DEFAULT_HELP, @@ -38,6 +35,9 @@ from hyperglass.configuration.defaults import ( DEFAULT_DETAILS, ) from hyperglass.configuration.markdown import get_markdown +from hyperglass.configuration.models.params import Params +from hyperglass.configuration.models.devices import Devices +from hyperglass.configuration.models.commands import Commands set_app_path(required=True) @@ -165,7 +165,7 @@ set_log_level(logger=log, debug=user_config.get("debug", True)) # Map imported user configuration to expected schema. log.debug("Unvalidated configuration from {}: {}", CONFIG_MAIN, user_config) -params = _validate_config(config=user_config, importer=_params.Params) +params = _validate_config(config=user_config, importer=Params) # Re-evaluate debug state after config is validated log_level = current_log_level(log) @@ -178,16 +178,12 @@ elif not params.debug and log_level == "debug": # Map imported user commands to expected schema. _user_commands = _config_optional(CONFIG_COMMANDS) log.debug("Unvalidated commands from {}: {}", CONFIG_COMMANDS, _user_commands) -commands = _validate_config( - config=_user_commands, importer=_commands.Commands.import_params -) +commands = _validate_config(config=_user_commands, importer=Commands.import_params) # Map imported user devices to expected schema. _user_devices = _config_required(CONFIG_DEVICES) log.debug("Unvalidated devices from {}: {}", CONFIG_DEVICES, _user_devices) -devices = _validate_config( - config=_user_devices.get("routers", []), importer=_routers.Routers._import, -) +devices = _validate_config(config=_user_devices.get("routers", []), importer=Devices) # Validate commands are both supported and properly mapped. _validate_nos_commands(devices.all_nos, commands) @@ -226,7 +222,7 @@ try: # If keywords are unmodified (default), add the org name & # site_title. - if _params.Params().site_keywords == params.site_keywords: + if Params().site_keywords == params.site_keywords: params.site_keywords = sorted( {*params.site_keywords, params.org_name, params.site_title} ) @@ -258,7 +254,7 @@ def _build_frontend_networks(): {dict} -- Frontend networks """ frontend_dict = {} - for device in devices.routers: + for device in devices.objects: if device.network.display_name in frontend_dict: frontend_dict[device.network.display_name].update( { @@ -302,7 +298,7 @@ def _build_frontend_devices(): {dict} -- Frontend devices """ frontend_dict = {} - for device in devices.routers: + for device in devices.objects: if device.name in frontend_dict: frontend_dict[device.name].update( { @@ -348,11 +344,11 @@ def _build_networks(): {dict} -- Networks & devices """ networks = [] - _networks = list(set({device.network.display_name for device in devices.routers})) + _networks = list(set({device.network.display_name for device in devices.objects})) for _network in _networks: network_def = {"display_name": _network, "locations": []} - for device in devices.routers: + for device in devices.objects: if device.network.display_name == _network: network_def["locations"].append( { @@ -374,7 +370,7 @@ def _build_networks(): def _build_vrfs(): vrfs = [] - for device in devices.routers: + for device in devices.objects: for vrf in device.vrfs: vrf_dict = { diff --git a/hyperglass/configuration/models/credential.py b/hyperglass/configuration/models/credential.py new file mode 100644 index 0000000..65f130f --- /dev/null +++ b/hyperglass/configuration/models/credential.py @@ -0,0 +1,14 @@ +"""Validate credential configuration variables.""" + +# Third Party +from pydantic import SecretStr, StrictStr + +# Project +from hyperglass.models import HyperglassModel + + +class Credential(HyperglassModel): + """Model for per-credential config in devices.yaml.""" + + username: StrictStr + password: SecretStr diff --git a/hyperglass/configuration/models/credentials.py b/hyperglass/configuration/models/credentials.py deleted file mode 100644 index 1c1ad2d..0000000 --- a/hyperglass/configuration/models/credentials.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Validate credential configuration variables.""" - -# Third Party -from pydantic import SecretStr, StrictStr - -# Project -from hyperglass.util import clean_name -from hyperglass.models import HyperglassModel - - -class Credential(HyperglassModel): - """Model for per-credential config in devices.yaml.""" - - username: StrictStr - password: SecretStr - - -class Credentials(HyperglassModel): - """Base model for credentials class.""" - - @classmethod - def import_params(cls, input_params): - """Import credentials with corrected field names. - - Arguments: - input_params {dict} -- Credential definition - - Returns: - {object} -- Validated credential object - """ - obj = Credentials() - for (credname, params) in input_params.items(): - cred = clean_name(credname) - setattr(Credentials, cred, Credential(**params)) - return obj diff --git a/hyperglass/configuration/models/routers.py b/hyperglass/configuration/models/devices.py similarity index 72% rename from hyperglass/configuration/models/routers.py rename to hyperglass/configuration/models/devices.py index 705fc8b..4ebdbfe 100644 --- a/hyperglass/configuration/models/routers.py +++ b/hyperglass/configuration/models/devices.py @@ -3,23 +3,24 @@ # Standard Library import os import re -from typing import List, Optional +from typing import Any, Dict, List, Union, Optional from pathlib import Path +from ipaddress import IPv4Address, IPv6Address # Third Party from pydantic import StrictInt, StrictStr, StrictBool, validator # Project from hyperglass.log import log -from hyperglass.util import clean_name, validate_nos +from hyperglass.util import validate_nos, resolve_hostname from hyperglass.models import HyperglassModel, HyperglassModelExtra from hyperglass.constants import SCRAPE_HELPERS, SUPPORTED_STRUCTURED_OUTPUT from hyperglass.exceptions import ConfigError, UnsupportedDevice from hyperglass.configuration.models.ssl import Ssl -from hyperglass.configuration.models.vrfs import Vrf, Info -from hyperglass.configuration.models.proxies import Proxy -from hyperglass.configuration.models.networks import Network -from hyperglass.configuration.models.credentials import Credential +from hyperglass.configuration.models.vrf import Vrf, Info +from hyperglass.configuration.models.proxy import Proxy +from hyperglass.configuration.models.network import Network +from hyperglass.configuration.models.credential import Credential _default_vrf = { "name": "default", @@ -38,11 +39,11 @@ _default_vrf = { } -class Router(HyperglassModel): +class Device(HyperglassModel): """Validation model for per-router config in devices.yaml.""" name: StrictStr - address: StrictStr + address: Union[IPv4Address, IPv6Address, StrictStr] network: Network credential: Credential proxy: Optional[Proxy] @@ -56,6 +57,35 @@ class Router(HyperglassModel): vrf_names: List[StrictStr] = [] structured_output: Optional[StrictBool] + def __hash__(self) -> int: + """Make device object hashable so the object can be deduplicated with set().""" + return hash((self.name,)) + + def __eq__(self, other: Any) -> bool: + """Make device object comparable so the object can be deduplicated with set().""" + result = False + + if isinstance(other, HyperglassModel): + result = self.name == other.name + + return result + + @property + def _target(self): + return str(self.address) + + @validator("address") + def validate_address(cls, value, values): + """Ensure a hostname is resolvable.""" + if not isinstance(value, (IPv4Address, IPv6Address)): + if not any(resolve_hostname(value)): + raise ConfigError( + "Device '{d}' has an address of '{a}', which is not resolvable.", + d=values["name"], + a=value, + ) + return value + @validator("structured_output", pre=True, always=True) def validate_structured_output(cls, value, values): """Validate structured output is supported on the device & set a default. @@ -101,18 +131,6 @@ class Router(HyperglassModel): return value - @validator("name") - def validate_name(cls, value): - """Remove or replace unsupported characters from field values. - - Arguments: - value {str} -- Raw name/location - - Returns: - {} -- Valid name/location - """ - return clean_name(value) - @validator("ssl") def validate_ssl(cls, value, values): """Set default cert file location if undefined. @@ -219,17 +237,18 @@ class Router(HyperglassModel): return vrfs -class Routers(HyperglassModelExtra): +class Devices(HyperglassModelExtra): """Validation model for device configurations.""" hostnames: List[StrictStr] = [] vrfs: List[StrictStr] = [] display_vrfs: List[StrictStr] = [] - routers: List[Router] = [] - networks: List[StrictStr] = [] + vrf_objects: List[Vrf] = [] + objects: List[Device] = [] + all_nos: List[StrictStr] = [] + default_vrf: Vrf = Vrf(name="default", display_name="Global") - @classmethod - def _import(cls, input_params): + def __init__(self, input_params: List[Dict]) -> None: """Import loaded YAML, initialize per-network definitions. Remove unsupported characters from device names, dynamically @@ -243,33 +262,27 @@ class Routers(HyperglassModelExtra): {object} -- Validated routers object """ vrfs = set() - networks = set() display_vrfs = set() vrf_objects = set() all_nos = set() - router_objects = [] - routers = Routers() - routers.hostnames = [] - routers.vrfs = [] - routers.display_vrfs = [] + objects = set() + hostnames = set() + + init_kwargs = {} for definition in input_params: # Validate each router config against Router() model/schema - router = Router(**definition) - - # Set a class attribute for each router so each router's - # attributes can be accessed with `devices.router_hostname` - setattr(routers, router.name, router) + device = Device(**definition) # Add router-level attributes (assumed to be unique) to # class lists, e.g. so all hostnames can be accessed as a # list with `devices.hostnames`, same for all router # classes, for when iteration over all routers is required. - routers.hostnames.append(router.name) - router_objects.append(router) - all_nos.add(router.nos) + hostnames.add(device.name) + objects.add(device) + all_nos.add(device.nos) - for vrf in router.vrfs: + for vrf in device.vrfs: # For each configured router VRF, add its name and # display_name to a class set (for automatic de-duping). @@ -278,34 +291,43 @@ class Routers(HyperglassModelExtra): # Also add the names to a router-level list so each # router's VRFs and display VRFs can be easily accessed. - router.display_vrfs.append(vrf.display_name) - router.vrf_names.append(vrf.name) + device.display_vrfs.append(vrf.display_name) + device.vrf_names.append(vrf.name) # Add a 'default_vrf' attribute to the devices class # which contains the configured default VRF display name. - if vrf.name == "default" and not hasattr(cls, "default_vrf"): - routers.default_vrf = { - "name": vrf.name, - "display_name": vrf.display_name, - } + if vrf.name == "default" and not hasattr(self, "default_vrf"): + init_kwargs["default_vrf"] = Vrf( + name=vrf.name, display_name=vrf.display_name + ) # Add the native VRF objects to a set (for automatic # de-duping), but exlcude device-specific fields. - _copy_params = { - "deep": True, - "exclude": {"ipv4": {"source_address"}, "ipv6": {"source_address"}}, - } - vrf_objects.add(vrf.copy(**_copy_params)) + vrf_objects.add( + vrf.copy( + deep=True, + exclude={ + "ipv4": {"source_address"}, + "ipv6": {"source_address"}, + }, + ) + ) # Convert the de-duplicated sets to a standard list, add lists - # as class attributes. - routers.vrfs = list(vrfs) - routers.display_vrfs = list(display_vrfs) - routers.vrf_objects = list(vrf_objects) - routers.networks = list(networks) - routers.all_nos = list(all_nos) + # as class attributes. Sort router list by router name attribute + init_kwargs["hostnames"] = list(hostnames) + init_kwargs["all_nos"] = list(all_nos) + init_kwargs["vrfs"] = list(vrfs) + init_kwargs["display_vrfs"] = list(vrfs) + init_kwargs["vrf_objects"] = list(vrf_objects) + init_kwargs["objects"] = sorted(objects, key=lambda x: x.display_name) - # Sort router list by router name attribute - routers.routers = sorted(router_objects, key=lambda x: x.display_name) + super().__init__(**init_kwargs) - return routers + def __getitem__(self, accessor: str) -> Device: + """Get a device by its name.""" + for device in self.objects: + if device.name == accessor: + return device + + raise AttributeError(f"No device named '{accessor}'") diff --git a/hyperglass/configuration/models/network.py b/hyperglass/configuration/models/network.py new file mode 100644 index 0000000..cd539fc --- /dev/null +++ b/hyperglass/configuration/models/network.py @@ -0,0 +1,22 @@ +"""Validate network configuration variables.""" + +# Third Party +from pydantic import Field, StrictStr + +# Project +from hyperglass.models import HyperglassModel + + +class Network(HyperglassModel): + """Validation Model for per-network/asn config in devices.yaml.""" + + name: StrictStr = Field( + ..., + title="Network Name", + description="Internal name of the device's primary network.", + ) + display_name: StrictStr = Field( + ..., + title="Network Display Name", + description="Display name of the device's primary network.", + ) diff --git a/hyperglass/configuration/models/networks.py b/hyperglass/configuration/models/networks.py deleted file mode 100644 index 592d492..0000000 --- a/hyperglass/configuration/models/networks.py +++ /dev/null @@ -1,50 +0,0 @@ -"""Validate network configuration variables.""" - -# Third Party -from pydantic import Field, StrictStr - -# Project -from hyperglass.util import clean_name -from hyperglass.models import HyperglassModel - - -class Network(HyperglassModel): - """Validation Model for per-network/asn config in devices.yaml.""" - - name: StrictStr = Field( - ..., - title="Network Name", - description="Internal name of the device's primary network.", - ) - display_name: StrictStr = Field( - ..., - title="Network Display Name", - description="Display name of the device's primary network.", - ) - - -class Networks(HyperglassModel): - """Base model for networks class.""" - - @classmethod - def import_params(cls, input_params): - """Import loaded YAML, initialize per-network definitions. - - Remove unsupported characters from network names, dynamically - set attributes for the networks class. Add cls.networks - attribute so network objects can be accessed inside a dict. - - Arguments: - input_params {dict} -- Unvalidated network definitions - - Returns: - {object} -- Validated networks object - """ - obj = Networks() - networks = {} - for (netname, params) in input_params.items(): - netname = clean_name(netname) - setattr(Networks, netname, Network(**params)) - networks.update({netname: Network(**params).dict()}) - Networks.networks = networks - return obj diff --git a/hyperglass/configuration/models/proxies.py b/hyperglass/configuration/models/proxies.py deleted file mode 100644 index 1569785..0000000 --- a/hyperglass/configuration/models/proxies.py +++ /dev/null @@ -1,57 +0,0 @@ -"""Validate SSH proxy configuration variables.""" - -# Third Party -from pydantic import StrictInt, StrictStr, validator - -# Project -from hyperglass.util import clean_name -from hyperglass.models import HyperglassModel -from hyperglass.exceptions import UnsupportedDevice -from hyperglass.configuration.models.credentials import Credential - - -class Proxy(HyperglassModel): - """Validation model for per-proxy config in devices.yaml.""" - - name: StrictStr - address: StrictStr - port: StrictInt = 22 - credential: Credential - nos: StrictStr = "linux_ssh" - - @validator("nos") - def supported_nos(cls, value): - """Verify NOS is supported by hyperglass. - - Raises: - UnsupportedDevice: Raised if NOS is not supported. - - Returns: - {str} -- Valid NOS name - """ - if not value == "linux_ssh": - raise UnsupportedDevice(f'"{value}" device type is not supported.') - return value - - -class Proxies(HyperglassModel): - """Validation model for SSH proxy configuration.""" - - @classmethod - def import_params(cls, input_params): - """Import loaded YAML, initialize per-proxy definitions. - - Remove unsupported characters from proxy names, dynamically - set attributes for the proxies class. - - Arguments: - input_params {dict} -- Unvalidated proxy definitions - - Returns: - {object} -- Validated proxies object - """ - obj = Proxies() - for (devname, params) in input_params.items(): - dev = clean_name(devname) - setattr(Proxies, dev, Proxy(**params)) - return obj diff --git a/hyperglass/configuration/models/proxy.py b/hyperglass/configuration/models/proxy.py new file mode 100644 index 0000000..3dba5ae --- /dev/null +++ b/hyperglass/configuration/models/proxy.py @@ -0,0 +1,56 @@ +"""Validate SSH proxy configuration variables.""" + +# Standard Library +from typing import Union +from ipaddress import IPv4Address, IPv6Address + +# Third Party +from pydantic import StrictInt, StrictStr, validator + +# Project +from hyperglass.util import resolve_hostname +from hyperglass.models import HyperglassModel +from hyperglass.exceptions import ConfigError, UnsupportedDevice +from hyperglass.configuration.models.credential import Credential + + +class Proxy(HyperglassModel): + """Validation model for per-proxy config in devices.yaml.""" + + name: StrictStr + address: Union[IPv4Address, IPv6Address, StrictStr] + port: StrictInt = 22 + credential: Credential + nos: StrictStr = "linux_ssh" + + @property + def _target(self): + return str(self.address) + + @validator("address") + def validate_address(cls, value, values): + """Ensure a hostname is resolvable.""" + if not isinstance(value, (IPv4Address, IPv6Address)): + if not any(resolve_hostname(value)): + raise ConfigError( + "Device '{d}' has an address of '{a}', which is not resolvable.", + d=values["name"], + a=value, + ) + return value + + @validator("nos") + def supported_nos(cls, value, values): + """Verify NOS is supported by hyperglass. + + Raises: + UnsupportedDevice: Raised if NOS is not supported. + + Returns: + {str} -- Valid NOS name + """ + if not value == "linux_ssh": + raise UnsupportedDevice( + f"Proxy '{values['name']}' uses NOS '{value}', which is currently unsupported." + ) + return value diff --git a/hyperglass/configuration/models/queries.py b/hyperglass/configuration/models/queries.py index 1ec66fc..8dee591 100644 --- a/hyperglass/configuration/models/queries.py +++ b/hyperglass/configuration/models/queries.py @@ -197,6 +197,13 @@ class Queries(HyperglassModel): ping: Ping = Ping() traceroute: Traceroute = Traceroute() + def __getitem__(self, query_type: str): + """Get a query's object by name.""" + if hasattr(self, query_type): + return getattr(self, query_type) + + raise AttributeError(f"Query '{query_type}' is invalid") + class Config: """Pydantic model configuration.""" diff --git a/hyperglass/configuration/models/vrfs.py b/hyperglass/configuration/models/vrf.py similarity index 100% rename from hyperglass/configuration/models/vrfs.py rename to hyperglass/configuration/models/vrf.py diff --git a/hyperglass/execution/drivers/_common.py b/hyperglass/execution/drivers/_common.py index 201467b..17be92f 100644 --- a/hyperglass/execution/drivers/_common.py +++ b/hyperglass/execution/drivers/_common.py @@ -9,13 +9,13 @@ from hyperglass.parsing.nos import nos_parsers from hyperglass.parsing.common import parsers from hyperglass.api.models.query import Query from hyperglass.execution.construct import Construct -from hyperglass.configuration.models.routers import Router +from hyperglass.configuration.models.devices import Device class Connection: """Base transport driver class.""" - def __init__(self, device: Router, query_data: Query) -> None: + def __init__(self, device: Device, query_data: Query) -> None: """Initialize connection to device.""" self.device = device self.query_data = query_data diff --git a/hyperglass/execution/drivers/agent.py b/hyperglass/execution/drivers/agent.py index b5cd38d..5685861 100644 --- a/hyperglass/execution/drivers/agent.py +++ b/hyperglass/execution/drivers/agent.py @@ -23,14 +23,7 @@ from hyperglass.execution.drivers._common import Connection class AgentConnection(Connection): - """Connect to target device via specified transport. - - scrape_direct() directly connects to devices via SSH - - scrape_proxied() connects to devices via an SSH proxy - - rest() connects to devices via HTTP for RESTful API communication - """ + """Connect to target device via hyperglass-agent.""" async def collect(self) -> Iterable: # noqa: C901 """Connect to a device running hyperglass-agent via HTTP.""" @@ -60,7 +53,7 @@ class AgentConnection(Connection): else: http_protocol = "http" endpoint = "{protocol}://{address}:{port}/query/".format( - protocol=http_protocol, address=self.device.address, port=self.device.port + protocol=http_protocol, address=self.device._target, port=self.device.port ) log.debug(f"URL endpoint: {endpoint}") diff --git a/hyperglass/execution/drivers/ssh.py b/hyperglass/execution/drivers/ssh.py index 12c0fb8..9e226ae 100644 --- a/hyperglass/execution/drivers/ssh.py +++ b/hyperglass/execution/drivers/ssh.py @@ -23,11 +23,11 @@ class SSHConnection(Connection): """Set up an SSH tunnel according to a device's configuration.""" try: return open_tunnel( - proxy.address, + proxy._target, proxy.port, ssh_username=proxy.credential.username, ssh_password=proxy.credential.password.get_secret_value(), - remote_bind_address=(self.device.address, self.device.port), + remote_bind_address=(self.device._target, self.device.port), local_bind_address=("localhost", 0), skip_tunnel_checkup=False, gateway_timeout=params.request_timeout - 2, diff --git a/hyperglass/execution/drivers/ssh_netmiko.py b/hyperglass/execution/drivers/ssh_netmiko.py index b77d406..c1d2e40 100644 --- a/hyperglass/execution/drivers/ssh_netmiko.py +++ b/hyperglass/execution/drivers/ssh_netmiko.py @@ -43,7 +43,7 @@ class NetmikoConnection(SSHConnection): log.debug("Connecting directly to {}", self.device.name) netmiko_args = { - "host": host or self.device.address, + "host": host or self.device._target, "port": port or self.device.port, "device_type": self.device.nos, "username": self.device.credential.username, diff --git a/hyperglass/execution/drivers/ssh_scrapli.py b/hyperglass/execution/drivers/ssh_scrapli.py index 3867319..69ab771 100644 --- a/hyperglass/execution/drivers/ssh_scrapli.py +++ b/hyperglass/execution/drivers/ssh_scrapli.py @@ -72,7 +72,7 @@ class ScrapliConnection(SSHConnection): log.debug("Connecting directly to {}", self.device.name) driver_kwargs = { - "host": host or self.device.address, + "host": host or self.device._target, "port": port or self.device.port, "auth_username": self.device.credential.username, "auth_password": self.device.credential.password.get_secret_value(), diff --git a/hyperglass/execution/main.py b/hyperglass/execution/main.py index d1071bf..02becc8 100644 --- a/hyperglass/execution/main.py +++ b/hyperglass/execution/main.py @@ -14,7 +14,7 @@ from typing import Any, Dict, Union, Callable from hyperglass.log import log from hyperglass.util import validate_nos from hyperglass.exceptions import DeviceTimeout, ResponseEmpty -from hyperglass.configuration import params, devices +from hyperglass.configuration import params from hyperglass.api.models.query import Query from hyperglass.execution.drivers import ( AgentConnection, @@ -42,29 +42,28 @@ async def execute(query: Query) -> Union[str, Dict]: """Initiate query validation and execution.""" output = params.messages.general - device = getattr(devices, query.query_location) log.debug(f"Received query for {query}") - log.debug(f"Matched device config: {device}") + log.debug(f"Matched device config: {query.device}") - supported, driver_name = validate_nos(device.nos) + supported, driver_name = validate_nos(query.device.nos) mapped_driver = DRIVER_MAP.get(driver_name, NetmikoConnection) - driver = mapped_driver(device, query) + driver = mapped_driver(query.device, query) timeout_args = { "unformatted_msg": params.messages.connection_error, - "device_name": device.display_name, + "device_name": query.device.display_name, "error": params.messages.request_timeout, } - if device.proxy: - timeout_args["proxy"] = device.proxy.name + if query.device.proxy: + timeout_args["proxy"] = query.device.proxy.name signal.signal(signal.SIGALRM, handle_timeout(**timeout_args)) signal.alarm(params.request_timeout - 1) - if device.proxy: + if query.device.proxy: proxy = driver.setup_proxy() with proxy() as tunnel: response = await driver.collect( @@ -76,7 +75,9 @@ async def execute(query: Query) -> Union[str, Dict]: output = await driver.parsed_response(response) if output == "" or output == "\n": - raise ResponseEmpty(params.messages.no_output, device_name=device.display_name) + raise ResponseEmpty( + params.messages.no_output, device_name=query.device.display_name + ) log.debug(f"Output for query: {query.json()}:\n{repr(output)}") signal.alarm(0) diff --git a/hyperglass/models.py b/hyperglass/models.py index 8f650d1..b9608da 100644 --- a/hyperglass/models.py +++ b/hyperglass/models.py @@ -17,7 +17,6 @@ from pydantic import ( # Project from hyperglass.log import log -from hyperglass.util import clean_name IntFloat = TypeVar("IntFloat", StrictInt, StrictFloat) @@ -25,6 +24,18 @@ _WEBHOOK_TITLE = "hyperglass received a valid query with the following data" _ICON_URL = "https://res.cloudinary.com/hyperglass/image/upload/v1593192484/icon.png" +def clean_name(_name: str) -> str: + """Remove unsupported characters from field names. + + Converts any "desirable" seperators to underscore, then removes all + characters that are unsupported in Python class variable names. + Also removes leading numbers underscores. + """ + _replaced = re.sub(r"[\-|\.|\@|\~|\:\/|\s]", "_", _name) + _scrubbed = "".join(re.findall(r"([a-zA-Z]\w+|\_+)", _replaced)) + return _scrubbed.lower() + + class HyperglassModel(BaseModel): """Base model for all hyperglass configuration models.""" diff --git a/hyperglass/util/__init__.py b/hyperglass/util/__init__.py index 70c339d..79458d8 100644 --- a/hyperglass/util/__init__.py +++ b/hyperglass/util/__init__.py @@ -8,9 +8,9 @@ import math import shutil import asyncio from queue import Queue -from typing import Dict, Union, Iterable, Optional +from typing import Dict, Union, Iterable, Optional, Generator from pathlib import Path -from ipaddress import IPv4Address, IPv6Address +from ipaddress import IPv4Address, IPv6Address, ip_address from threading import Thread # Third Party @@ -18,6 +18,7 @@ from loguru._logger import Logger as LoguruLogger # Project from hyperglass.log import log +from hyperglass.models import HyperglassModel def cpu_count(multiplier: int = 0): @@ -33,18 +34,6 @@ def cpu_count(multiplier: int = 0): return multiprocessing.cpu_count() * multiplier -def clean_name(_name: str) -> str: - """Remove unsupported characters from field names. - - Converts any "desirable" seperators to underscore, then removes all - characters that are unsupported in Python class variable names. - Also removes leading numbers underscores. - """ - _replaced = re.sub(r"[\-|\.|\@|\~|\:\/|\s]", "_", _name) - _scrubbed = "".join(re.findall(r"([a-zA-Z]\w+|\_+)", _replaced)) - return _scrubbed.lower() - - def check_path( path: Union[Path, str], mode: str = "r", create: bool = False ) -> Optional[Path]: @@ -925,3 +914,34 @@ def validation_error_message(*errors: Dict) -> str: errs += (f'Field: {loc}\n Error: {err["msg"]}\n',) return "\n".join(errs) + + +def resolve_hostname(hostname: str) -> Generator: + """Resolve a hostname via DNS/hostfile.""" + from socket import getaddrinfo, gaierror + + log.debug("Ensuring '{}' is resolvable...", hostname) + + ip4 = None + ip6 = None + try: + res = getaddrinfo(hostname, None) + if len(res) == 2: + addr = ip_address(res[0][4][0]) + if addr.version == 6: + ip6 = addr + else: + ip4 = addr + elif len(res) == 4: + addr1 = ip_address(res[0][4][0]) + addr2 = ip_address(res[2][4][0]) + for a in (addr1, addr2): + if a.version == 4: + ip4 = a + elif a.version == 6: + ip6 = a + except gaierror: + pass + + yield ip4 + yield ip6 diff --git a/validate_examples.py b/validate_examples.py index 0c7c8ee..4013d5c 100644 --- a/validate_examples.py +++ b/validate_examples.py @@ -50,12 +50,12 @@ def _comment_optional_files(): def _validate_devices(): - from hyperglass.configuration.models.routers import Routers + from hyperglass.configuration.models.devices import Devices with DEVICES.open() as raw: devices_dict = yaml.safe_load(raw.read()) or {} try: - Routers._import(devices_dict.get("routers", [])) + Devices(devices_dict.get("routers", [])) except Exception as e: raise ValueError(str(e)) return True