diff --git a/hyperglass/api/models/query.py b/hyperglass/api/models/query.py index 8ab2077..8ada1bf 100644 --- a/hyperglass/api/models/query.py +++ b/hyperglass/api/models/query.py @@ -121,6 +121,10 @@ class Query(BaseModel): ) return f'Query({", ".join(items)})' + @property + def device(self): + return getattr(devices, self.query_location) + def export_dict(self): """Create dictionary representation of instance.""" return { diff --git a/hyperglass/api/models/response.py b/hyperglass/api/models/response.py index c2a96de..d285f09 100644 --- a/hyperglass/api/models/response.py +++ b/hyperglass/api/models/response.py @@ -1,12 +1,14 @@ """Response model.""" + # Standard Library -from typing import List, Optional +from typing import List, Union, Optional # Third Party from pydantic import BaseModel, StrictInt, StrictStr, StrictBool, constr # Project from hyperglass.configuration import params +from hyperglass.parsing.models.serialized import ParsedRoutes class QueryError(BaseModel): @@ -55,14 +57,14 @@ class QueryError(BaseModel): class QueryResponse(BaseModel): """Query response model.""" - output: StrictStr + output: Union[ParsedRoutes, StrictStr] level: constr(regex=r"success") = "success" random: StrictStr cached: StrictBool runtime: StrictInt keywords: List[StrictStr] = [] timestamp: StrictStr - _format: constr(regex=r"(application\/json|text\/plain)") = "text/plain" + format: constr(regex=r"(application\/json|text\/plain)") = "text/plain" class Config: """Pydantic model configuration.""" @@ -91,7 +93,7 @@ class QueryResponse(BaseModel): "example": "2020-04-18 14:45:37", }, "format": { - "alias": "format", + # "alias": "format", "title": "Format", "description": "Response [MIME Type](http://www.iana.org/assignments/media-types/media-types.xhtml). Supported values: `text/plain` and `application/json`.", "example": "text/plain", diff --git a/hyperglass/api/routes.py b/hyperglass/api/routes.py index bcafa09..39a7171 100644 --- a/hyperglass/api/routes.py +++ b/hyperglass/api/routes.py @@ -2,6 +2,7 @@ # Standard Library import os +import json import time # Third Party @@ -53,8 +54,8 @@ async def query(query_data: Query, request: Request): # Define cache entry expiry time cache_timeout = params.cache.timeout - log.debug(f"Cache Timeout: {cache_timeout}") + log.debug(f"Cache Timeout: {cache_timeout}") log.info(f"Starting query execution for query {query_data.summary}") cache_response = await cache.get_dict(cache_key, "output") @@ -88,7 +89,11 @@ async def query(query_data: Query, request: Request): raise HyperglassError(message=params.messages.general, alert="danger") # Create a cache entry - await cache.set_dict(cache_key, "output", str(cache_output)) + if query_data.device.structured_output: + raw_output = json.dumps(cache_output) + else: + raw_output = str(cache_output) + await cache.set_dict(cache_key, "output", raw_output) await cache.set_dict(cache_key, "timestamp", timestamp) await cache.expire(cache_key, seconds=cache_timeout) @@ -99,6 +104,12 @@ async def query(query_data: Query, request: Request): # If it does, return the cached entry cache_response = await cache.get_dict(cache_key, "output") + if query_data.device.structured_output: + response_format = "application/json" + cache_response = json.loads(cache_response) + else: + response_format = "text/plain" + log.debug(f"Cache match for {cache_key}:\n {cache_response}") log.success(f"Completed query execution for {query_data.summary}") @@ -108,6 +119,7 @@ async def query(query_data: Query, request: Request): "cached": cached, "runtime": runtime, "timestamp": timestamp, + "format": response_format, "random": query_data.random(), "level": "success", "keywords": [], diff --git a/hyperglass/configuration/models/commands/arista.py b/hyperglass/configuration/models/commands/arista.py index 7b447c7..59ce6c6 100644 --- a/hyperglass/configuration/models/commands/arista.py +++ b/hyperglass/configuration/models/commands/arista.py @@ -1,42 +1,56 @@ """Arista Command Model.""" +# Third Party +from pydantic import StrictStr + # Project from hyperglass.configuration.models.commands.common import CommandSet, CommandGroup -_ipv4_default = { - "bgp_route": "show ip bgp {target}", - "bgp_aspath": "show ip bgp regexp {target}", - "bgp_community": "show ip bgp community {target}", - "ping": "ping ip {target} source {source}", - "traceroute": "traceroute ip {target} source {source}", -} -_ipv6_default = { - "bgp_route": "show ipv6 bgp {target}", - "bgp_aspath": "show ipv6 bgp regexp {target}", - "bgp_community": "show ipv6 bgp community {target}", - "ping": "ping ipv6 {target} source {source}", - "traceroute": "traceroute ipv6 {target} source {source}", -} -_ipv4_vpn = { - "bgp_route": "show ip bgp {target} vrf {vrf}", - "bgp_aspath": "show ip bgp regexp {target} vrf {vrf}", - "bgp_community": "show ip bgp community {target} vrf {vrf}", - "ping": "ping vrf {vrf} ip {target} source {source}", - "traceroute": "traceroute vrf {vrf} ip {target} source {source}", -} -_ipv6_vpn = { - "bgp_route": "show ipv6 bgp {target} vrf {vrf}", - "bgp_aspath": "show ipv6 bgp regexp {target} vrf {vrf}", - "bgp_community": "show ipv6 bgp community {target} vrf {vrf}", - "ping": "ping vrf {vrf} ipv6 {target} source {source}", - "traceroute": "traceroute vrf {vrf} ipv6 {target} source {source}", -} + +class _IPv4(CommandSet): + """Validation model for non-default dual afi commands.""" + + bgp_route: StrictStr = "show ip bgp {target}" + bgp_aspath: StrictStr = "show ip bgp regexp {target}" + bgp_community: StrictStr = "show ip bgp community {target}" + ping: StrictStr = "ping ip {target} source {source}" + traceroute: StrictStr = "traceroute ip {target} source {source}" + + +class _IPv6(CommandSet): + """Validation model for non-default ipv4 commands.""" + + bgp_route: StrictStr = "show ipv6 bgp {target}" + bgp_aspath: StrictStr = "show ipv6 bgp regexp {target}" + bgp_community: StrictStr = "show ipv6 bgp community {target}" + ping: StrictStr = "ping ipv6 {target} source {source}" + traceroute: StrictStr = "traceroute ipv6 {target} source {source}" + + +class _VPNIPv4(CommandSet): + """Validation model for non-default ipv6 commands.""" + + bgp_route: StrictStr = "show ip bgp {target} vrf {vrf}" + bgp_aspath: StrictStr = "show ip bgp regexp {target} vrf {vrf}" + bgp_community: StrictStr = "show ip bgp community {target} vrf {vrf}" + ping: StrictStr = "ping vrf {vrf} ip {target} source {source}" + traceroute: StrictStr = "traceroute vrf {vrf} ip {target} source {source}" + + +class _VPNIPv6(CommandSet): + """Validation model for non-default ipv6 commands.""" + + bgp_route: StrictStr = "show ipv6 bgp {target} vrf {vrf}" + bgp_aspath: StrictStr = "show ipv6 bgp regexp {target} vrf {vrf}" + bgp_community: StrictStr = "show ipv6 bgp community {target} vrf {vrf}" + ping: StrictStr = "ping vrf {vrf} ipv6 {target} source {source}" + traceroute: StrictStr = "traceroute vrf {vrf} ipv6 {target} source {source}" class AristaCommands(CommandGroup): """Validation model for default arista commands.""" - ipv4_default: CommandSet = CommandSet(**_ipv4_default) - ipv6_default: CommandSet = CommandSet(**_ipv6_default) - ipv4_vpn: CommandSet = CommandSet(**_ipv4_vpn) - ipv6_vpn: CommandSet = CommandSet(**_ipv6_vpn) + ipv4_default: _IPv4 = _IPv4() + ipv6_default: _IPv6 = _IPv6() + ipv4_vpn: _VPNIPv4 = _VPNIPv4() + ipv6_vpn: _VPNIPv6 = _VPNIPv6() diff --git a/hyperglass/configuration/models/commands/cisco_ios.py b/hyperglass/configuration/models/commands/cisco_ios.py index ba84515..76f61aa 100644 --- a/hyperglass/configuration/models/commands/cisco_ios.py +++ b/hyperglass/configuration/models/commands/cisco_ios.py @@ -1,42 +1,62 @@ """Cisco IOS Command Model.""" +# Third Party +from pydantic import StrictStr + # Project from hyperglass.configuration.models.commands.common import CommandSet, CommandGroup -_ipv4_default = { - "bgp_route": "show bgp ipv4 unicast {target} | exclude pathid:|Epoch", - "bgp_aspath": 'show bgp ipv4 unicast quote-regexp "{target}"', - "bgp_community": "show bgp ipv4 unicast community {target}", - "ping": "ping {target} repeat 5 source {source}", - "traceroute": "traceroute {target} timeout 1 probe 2 source {source}", -} -_ipv6_default = { - "bgp_route": "show bgp ipv6 unicast {target} | exclude pathid:|Epoch", - "bgp_aspath": 'show bgp ipv6 unicast quote-regexp "{target}"', - "bgp_community": "show bgp ipv6 unicast community {target}", - "ping": "ping ipv6 {target} repeat 5 source {source}", - "traceroute": "traceroute ipv6 {target} timeout 1 probe 2 source {source}", -} -_ipv4_vpn = { - "bgp_route": "show bgp vpnv4 unicast vrf {vrf} {target}", - "bgp_aspath": 'show bgp vpnv4 unicast vrf {vrf} quote-regexp "{target}"', - "bgp_community": "show bgp vpnv4 unicast vrf {vrf} community {target}", - "ping": "ping vrf {vrf} {target} repeat 5 source {source}", - "traceroute": "traceroute vrf {vrf} {target} timeout 1 probe 2 source {source}", -} -_ipv6_vpn = { - "bgp_route": "show bgp vpnv6 unicast vrf {vrf} {target}", - "bgp_aspath": 'show bgp vpnv6 unicast vrf {vrf} quote-regexp "{target}"', - "bgp_community": "show bgp vpnv6 unicast vrf {vrf} community {target}", - "ping": "ping vrf {vrf} {target} repeat 5 source {source}", - "traceroute": "traceroute vrf {vrf} {target} timeout 1 probe 2 source {source}", -} + +class _IPv4(CommandSet): + """Default commands for ipv4 commands.""" + + bgp_community: StrictStr = "show bgp ipv4 unicast community {target}" + bgp_aspath: StrictStr = 'show bgp ipv4 unicast quote-regexp "{target}"' + bgp_route: StrictStr = "show bgp ipv4 unicast {target} | exclude pathid:|Epoch" + ping: StrictStr = "ping {target} repeat 5 source {source}" + traceroute: StrictStr = "traceroute {target} timeout 1 probe 2 source {source}" + + +class _IPv6(CommandSet): + """Default commands for ipv6 commands.""" + + bgp_community: StrictStr = "show bgp ipv6 unicast community {target}" + bgp_aspath: StrictStr = 'show bgp ipv6 unicast quote-regexp "{target}"' + bgp_route: StrictStr = "show bgp ipv6 unicast {target} | exclude pathid:|Epoch" + ping: StrictStr = ("ping ipv6 {target} repeat 5 source {source}") + traceroute: StrictStr = ( + "traceroute ipv6 {target} timeout 1 probe 2 source {source}" + ) + + +class _VPNIPv4(CommandSet): + """Default commands for dual afi commands.""" + + bgp_community: StrictStr = "show bgp vpnv4 unicast vrf {vrf} community {target}" + bgp_aspath: StrictStr = 'show bgp vpnv4 unicast vrf {vrf} quote-regexp "{target}"' + bgp_route: StrictStr = "show bgp vpnv4 unicast vrf {vrf} {target}" + ping: StrictStr = "ping vrf {vrf} {target} repeat 5 source {source}" + traceroute: StrictStr = ( + "traceroute vrf {vrf} {target} timeout 1 probe 2 source {source}" + ) + + +class _VPNIPv6(CommandSet): + """Default commands for dual afi commands.""" + + bgp_community: StrictStr = "show bgp vpnv6 unicast vrf {vrf} community {target}" + bgp_aspath: StrictStr = 'show bgp vpnv6 unicast vrf {vrf} quote-regexp "{target}"' + bgp_route: StrictStr = "show bgp vpnv6 unicast vrf {vrf} {target}" + ping: StrictStr = "ping vrf {vrf} {target} repeat 5 source {source}" + traceroute: StrictStr = ( + "traceroute vrf {vrf} {target} timeout 1 probe 2 source {source}" + ) class CiscoIOSCommands(CommandGroup): """Validation model for default cisco_ios commands.""" - ipv4_default: CommandSet = CommandSet(**_ipv4_default) - ipv6_default: CommandSet = CommandSet(**_ipv6_default) - ipv4_vpn: CommandSet = CommandSet(**_ipv4_vpn) - ipv6_vpn: CommandSet = CommandSet(**_ipv6_vpn) + ipv4_default: _IPv4 = _IPv4() + ipv6_default: _IPv6 = _IPv6() + ipv4_vpn: _VPNIPv4 = _VPNIPv4() + ipv6_vpn: _VPNIPv6 = _VPNIPv6() diff --git a/hyperglass/configuration/models/commands/cisco_nxos.py b/hyperglass/configuration/models/commands/cisco_nxos.py index 4248038..fb8be90 100644 --- a/hyperglass/configuration/models/commands/cisco_nxos.py +++ b/hyperglass/configuration/models/commands/cisco_nxos.py @@ -1,42 +1,56 @@ """Cisco NX-OS Command Model.""" +# Third Party +from pydantic import StrictStr + # Project from hyperglass.configuration.models.commands.common import CommandSet, CommandGroup -_ipv4_default = { - "bgp_route": "show bgp ipv4 unicast {target}", - "bgp_aspath": 'show bgp ipv4 unicast regexp "{target}"', - "bgp_community": "show bgp ipv4 unicast community {target}", - "ping": "ping {target} source {source}", - "traceroute": "traceroute {target} source {source}", -} -_ipv6_default = { - "bgp_route": "show bgp ipv6 unicast {target}", - "bgp_aspath": 'show bgp ipv6 unicast regexp "{target}"', - "bgp_community": "show bgp ipv6 unicast community {target}", - "ping": "ping6 {target} source {source}", - "traceroute": "traceroute6 {target} source {source}", -} -_ipv4_vpn = { - "bgp_route": "show bgp ipv4 unicast {target} vrf {vrf}", - "bgp_aspath": 'show bgp ipv4 unicast regexp "{target}" vrf {vrf}', - "bgp_community": "show bgp ipv4 unicast community {target} vrf {vrf}", - "ping": "ping {target} source {source} vrf {vrf}", - "traceroute": "traceroute {target} source {source} vrf {vrf}", -} -_ipv6_vpn = { - "bgp_route": "show bgp ipv6 unicast {target} vrf {vrf}", - "bgp_aspath": 'show bgp ipv6 unicast regexp "{target}" vrf {vrf}', - "bgp_community": "show bgp ipv6 unicast community {target} vrf {vrf}", - "ping": "ping6 {target} source {source} vrf {vrf}", - "traceroute": "traceroute6 {target} source {source} vrf {vrf}", -} + +class _IPv4(CommandSet): + """Validation model for non-default dual afi commands.""" + + bgp_route: StrictStr = "show bgp ipv4 unicast {target}" + bgp_aspath: StrictStr = 'show bgp ipv4 unicast regexp "{target}"' + bgp_community: StrictStr = "show bgp ipv4 unicast community {target}" + ping: StrictStr = "ping {target} source {source}" + traceroute: StrictStr = "traceroute {target} source {source}" + + +class _IPv6(CommandSet): + """Validation model for non-default ipv4 commands.""" + + bgp_route: StrictStr = "show bgp ipv6 unicast {target}" + bgp_aspath: StrictStr = 'show bgp ipv6 unicast regexp "{target}"' + bgp_community: StrictStr = "show bgp ipv6 unicast community {target}" + ping: StrictStr = "ping6 {target} source {source}" + traceroute: StrictStr = "traceroute6 {target} source {source}" + + +class _VPNIPv4(CommandSet): + """Validation model for non-default ipv6 commands.""" + + bgp_route: StrictStr = "show bgp ipv4 unicast {target} vrf {vrf}" + bgp_aspath: StrictStr = 'show bgp ipv4 unicast regexp "{target}" vrf {vrf}' + bgp_community: StrictStr = "show bgp ipv4 unicast community {target} vrf {vrf}" + ping: StrictStr = "ping {target} source {source} vrf {vrf}" + traceroute: StrictStr = "traceroute {target} source {source} vrf {vrf}" + + +class _VPNIPv6(CommandSet): + """Validation model for non-default ipv6 commands.""" + + bgp_route: StrictStr = "show bgp ipv6 unicast {target} vrf {vrf}" + bgp_aspath: StrictStr = 'show bgp ipv6 unicast regexp "{target}" vrf {vrf}' + bgp_community: StrictStr = "show bgp ipv6 unicast community {target} vrf {vrf}" + ping: StrictStr = "ping6 {target} source {source} vrf {vrf}" + traceroute: StrictStr = "traceroute6 {target} source {source} vrf {vrf}" class CiscoNXOSCommands(CommandGroup): """Validation model for default cisco_nxos commands.""" - ipv4_default: CommandSet = CommandSet(**_ipv4_default) - ipv6_default: CommandSet = CommandSet(**_ipv6_default) - ipv4_vpn: CommandSet = CommandSet(**_ipv4_vpn) - ipv6_vpn: CommandSet = CommandSet(**_ipv6_vpn) + ipv4_default: _IPv4 = _IPv4() + ipv6_default: _IPv6 = _IPv6() + ipv4_vpn: _VPNIPv4 = _VPNIPv4() + ipv6_vpn: _VPNIPv6 = _VPNIPv6() diff --git a/hyperglass/configuration/models/commands/cisco_xr.py b/hyperglass/configuration/models/commands/cisco_xr.py index 478e324..1aaaa72 100644 --- a/hyperglass/configuration/models/commands/cisco_xr.py +++ b/hyperglass/configuration/models/commands/cisco_xr.py @@ -1,42 +1,56 @@ """Cisco IOS XR Command Model.""" +# Third Party +from pydantic import StrictStr + # Project from hyperglass.configuration.models.commands.common import CommandSet, CommandGroup -_ipv4_default = { - "bgp_route": "show bgp ipv4 unicast {target}", - "bgp_aspath": "show bgp ipv4 unicast regexp {target}", - "bgp_community": "show bgp ipv4 unicast community {target}", - "ping": "ping ipv4 {target} count 5 source {source}", - "traceroute": "traceroute ipv4 {target} timeout 1 probe 2 source {source}", -} -_ipv6_default = { - "bgp_route": "show bgp ipv6 unicast {target}", - "bgp_aspath": "show bgp ipv6 unicast regexp {target}", - "bgp_community": "show bgp ipv6 unicast community {target}", - "ping": "ping ipv6 {target} count 5 source {source}", - "traceroute": "traceroute ipv6 {target} timeout 1 probe 2 source {source}", -} -_ipv4_vpn = { - "bgp_route": "show bgp vpnv4 unicast vrf {vrf} {target}", - "bgp_aspath": "show bgp vpnv4 unicast vrf {vrf} regexp {target}", - "bgp_community": "show bgp vpnv4 unicast vrf {vrf} community {target}", - "ping": "ping vrf {vrf} {target} count 5 source {source}", - "traceroute": "traceroute vrf {vrf} {target} timeout 1 probe 2 source {source}", -} -_ipv6_vpn = { - "bgp_route": "show bgp vpnv6 unicast vrf {vrf} {target}", - "bgp_aspath": "show bgp vpnv6 unicast vrf {vrf} regexp {target}", - "bgp_community": "show bgp vpnv6 unicast vrf {vrf} community {target}", - "ping": "ping vrf {vrf} {target} count 5 source {source}", - "traceroute": "traceroute vrf {vrf} {target} timeout 1 probe 2 source {source}", -} + +class _IPv4(CommandSet): + """Validation model for non-default dual afi commands.""" + + bgp_route: StrictStr = "show bgp ipv4 unicast {target}" + bgp_aspath: StrictStr = "show bgp ipv4 unicast regexp {target}" + bgp_community: StrictStr = "show bgp ipv4 unicast community {target}" + ping: StrictStr = "ping ipv4 {target} count 5 source {source}" + traceroute: StrictStr = "traceroute ipv4 {target} timeout 1 probe 2 source {source}" + + +class _IPv6(CommandSet): + """Validation model for non-default ipv4 commands.""" + + bgp_route: StrictStr = "show bgp ipv6 unicast {target}" + bgp_aspath: StrictStr = "show bgp ipv6 unicast regexp {target}" + bgp_community: StrictStr = "show bgp ipv6 unicast community {target}" + ping: StrictStr = "ping ipv6 {target} count 5 source {source}" + traceroute: StrictStr = "traceroute ipv6 {target} timeout 1 probe 2 source {source}" + + +class _VPNIPv4(CommandSet): + """Validation model for non-default ipv6 commands.""" + + bgp_route: StrictStr = "show bgp vpnv4 unicast vrf {vrf} {target}" + bgp_aspath: StrictStr = "show bgp vpnv4 unicast vrf {vrf} regexp {target}" + bgp_community: StrictStr = "show bgp vpnv4 unicast vrf {vrf} community {target}" + ping: StrictStr = "ping vrf {vrf} {target} count 5 source {source}" + traceroute: StrictStr = "traceroute vrf {vrf} {target} timeout 1 probe 2 source {source}" + + +class _VPNIPv6(CommandSet): + """Validation model for non-default ipv6 commands.""" + + bgp_route: StrictStr = "show bgp vpnv6 unicast vrf {vrf} {target}" + bgp_aspath: StrictStr = "show bgp vpnv6 unicast vrf {vrf} regexp {target}" + bgp_community: StrictStr = "show bgp vpnv6 unicast vrf {vrf} community {target}" + ping: StrictStr = "ping vrf {vrf} {target} count 5 source {source}" + traceroute: StrictStr = "traceroute vrf {vrf} {target} timeout 1 probe 2 source {source}" class CiscoXRCommands(CommandGroup): """Validation model for default cisco_xr commands.""" - ipv4_default: CommandSet = CommandSet(**_ipv4_default) - ipv6_default: CommandSet = CommandSet(**_ipv6_default) - ipv4_vpn: CommandSet = CommandSet(**_ipv4_vpn) - ipv6_vpn: CommandSet = CommandSet(**_ipv6_vpn) + ipv4_default: _IPv4 = _IPv4() + ipv6_default: _IPv6 = _IPv6() + ipv4_vpn: _VPNIPv4 = _VPNIPv4() + ipv6_vpn: _VPNIPv6 = _VPNIPv6() diff --git a/hyperglass/configuration/models/commands/common.py b/hyperglass/configuration/models/commands/common.py index e0e74a1..ccb7d4f 100644 --- a/hyperglass/configuration/models/commands/common.py +++ b/hyperglass/configuration/models/commands/common.py @@ -1,12 +1,10 @@ """Models common to entire commands module.""" -from typing import Optional - # Third Party from pydantic import StrictStr # Project -from hyperglass.models import HyperglassModel +from hyperglass.models import HyperglassModel, HyperglassModelExtra class CommandSet(HyperglassModel): @@ -19,14 +17,10 @@ class CommandSet(HyperglassModel): traceroute: StrictStr -class CommandGroup(HyperglassModel): +class CommandGroup(HyperglassModelExtra): """Validation model for all commands.""" ipv4_default: CommandSet ipv6_default: CommandSet ipv4_vpn: CommandSet ipv6_vpn: CommandSet - structured: Optional["CommandGroup"] - - -CommandGroup.update_forward_refs() diff --git a/hyperglass/configuration/models/commands/huawei.py b/hyperglass/configuration/models/commands/huawei.py index 8d09652..4f7dbf8 100644 --- a/hyperglass/configuration/models/commands/huawei.py +++ b/hyperglass/configuration/models/commands/huawei.py @@ -1,42 +1,56 @@ """Huawei Command Model.""" +# Third Party +from pydantic import StrictStr + # Project from hyperglass.configuration.models.commands.common import CommandSet, CommandGroup -_ipv4_default = { - "bgp_route": "display bgp routing-table {target}", - "bgp_aspath": "display bgp routing-table regular-expression {target}", - "bgp_community": "display bgp routing-table regular-expression {target}", - "ping": "ping -c 5 -a {source} {target}", - "traceroute": "tracert -q 2 -f 1 -a {source} {target}", -} -_ipv6_default = { - "bgp_route": "display bgp ipv6 routing-table {target}", - "bgp_aspath": "display bgp ipv6 routing-table regular-expression {target}", - "bgp_community": "display bgp ipv6 routing-table community {target}", - "ping": "ping ipv6 -c 5 -a {source} {target}", - "traceroute": "tracert ipv6 -q 2 -f 1 -a {source} {target}", -} -_ipv4_vpn = { - "bgp_route": "display bgp vpnv4 vpn-instance {vrf} routing-table {target}", - "bgp_aspath": "display bgp vpnv4 vpn-instance {vrf} routing-table regular-expression {target}", - "bgp_community": "display bgp vpnv4 vpn-instance {vrf} routing-table regular-expression {target}", - "ping": "ping -vpn-instance {vrf} -c 5 -a {source} {target}", - "traceroute": "tracert -q 2 -f 1 -vpn-instance {vrf} -a {source} {target}", -} -_ipv6_vpn = { - "bgp_route": "display bgp vpnv6 vpn-instance {vrf} routing-table {target}", - "bgp_aspath": "display bgp vpnv6 vpn-instance {vrf} routing-table regular-expression {target}", - "bgp_community": "display bgp vpnv6 vpn-instance {vrf} routing-table regular-expression {target}", - "ping": "ping vpnv6 vpn-instance {vrf} -c 5 -a {source} {target}", - "traceroute": "tracert -q 2 -f 1 vpn-instance {vrf} -a {source} {target}", -} + +class _IPv4(CommandSet): + """Default commands for ipv4 commands.""" + + bgp_community: StrictStr = "display bgp routing-table regular-expression {target}" + bgp_aspath: StrictStr = "display bgp routing-table regular-expression {target}" + bgp_route: StrictStr = "display bgp routing-table {target}" + ping: StrictStr = "ping -c 5 -a {source} {target}" + traceroute: StrictStr = "tracert -q 2 -f 1 -a {source} {target}" + + +class _IPv6(CommandSet): + """Default commands for ipv6 commands.""" + + bgp_community: StrictStr = "display bgp ipv6 routing-table community {target}" + bgp_aspath: StrictStr = "display bgp ipv6 routing-table regular-expression {target}" + bgp_route: StrictStr = "display bgp ipv6 routing-table {target}" + ping: StrictStr = "ping ipv6 -c 5 -a {source} {target}" + traceroute: StrictStr = "tracert ipv6 -q 2 -f 1 -a {source} {target}" + + +class _VPNIPv4(CommandSet): + """Default commands for dual afi commands.""" + + bgp_community: StrictStr = "display bgp vpnv4 vpn-instance {vrf} routing-table regular-expression {target}" + bgp_aspath: StrictStr = "display bgp vpnv4 vpn-instance {vrf} routing-table regular-expression {target}" + bgp_route: StrictStr = "display bgp vpnv4 vpn-instance {vrf} routing-table {target}" + ping: StrictStr = "ping -vpn-instance {vrf} -c 5 -a {source} {target}" + traceroute: StrictStr = "tracert -q 2 -f 1 -vpn-instance {vrf} -a {source} {target}" + + +class _VPNIPv6(CommandSet): + """Default commands for dual afi commands.""" + + bgp_community: StrictStr = "display bgp vpnv6 vpn-instance {vrf} routing-table regular-expression {target}" + bgp_aspath: StrictStr = "display bgp vpnv6 vpn-instance {vrf} routing-table regular-expression {target}" + bgp_route: StrictStr = "display bgp vpnv6 vpn-instance {vrf} routing-table {target}" + ping: StrictStr = "ping vpnv6 vpn-instance {vrf} -c 5 -a {source} {target}" + traceroute: StrictStr = "tracert -q 2 -f 1 vpn-instance {vrf} -a {source} {target}" class HuaweiCommands(CommandGroup): """Validation model for default huawei commands.""" - ipv4_default: CommandSet = CommandSet(**_ipv4_default) - ipv6_default: CommandSet = CommandSet(**_ipv6_default) - ipv4_vpn: CommandSet = CommandSet(**_ipv4_vpn) - ipv6_vpn: CommandSet = CommandSet(**_ipv6_vpn) + ipv4_default: _IPv4 = _IPv4() + ipv6_default: _IPv6 = _IPv6() + ipv4_vpn: _VPNIPv4 = _VPNIPv4() + ipv6_vpn: _VPNIPv6 = _VPNIPv6() diff --git a/hyperglass/configuration/models/commands/juniper.py b/hyperglass/configuration/models/commands/juniper.py index 15bcfef..0be6737 100644 --- a/hyperglass/configuration/models/commands/juniper.py +++ b/hyperglass/configuration/models/commands/juniper.py @@ -1,63 +1,78 @@ """Juniper Command Model.""" +# Third Party +from pydantic import StrictStr + # Project from hyperglass.configuration.models.commands.common import CommandSet, CommandGroup -_ipv4_default = { - "bgp_route": 'show route protocol bgp table inet.0 {target} detail | except Label | except Label | except "Next hop type" | except Task | except Address | except "Session Id" | except State | except "Next-hop reference" | except destinations | except "Announcement bits"', - "bgp_aspath": 'show route protocol bgp table inet.0 aspath-regex "{target}"', - "bgp_community": "show route protocol bgp table inet.0 community {target}", - "ping": "ping inet {target} count 5 source {source}", - "traceroute": "traceroute inet {target} wait 1 source {source}", -} -_ipv6_default = { - "bgp_route": 'show route protocol bgp table inet6.0 {target} detail | except Label | except Label | except "Next hop type" | except Task | except Address | except "Session Id" | except State | except "Next-hop reference" | except destinations | except "Announcement bits"', - "bgp_aspath": 'show route protocol bgp table inet6.0 aspath-regex "{target}"', - "bgp_community": "show route protocol bgp table inet6.0 community {target}", - "ping": "ping inet6 {target} count 5 source {source}", - "traceroute": "traceroute inet6 {target} wait 2 source {source}", -} -_ipv4_vpn = { - "bgp_route": 'show route protocol bgp table {vrf}.inet.0 {target} detail | except Label | except Label | except "Next hop type" | except Task | except Address | except "Session Id" | except State | except "Next-hop reference" | except destinations | except "Announcement bits"', - "bgp_aspath": 'show route protocol bgp table {vrf}.inet.0 aspath-regex "{target}"', - "bgp_community": "show route protocol bgp table {vrf}.inet.0 community {target}", - "ping": "ping inet routing-instance {vrf} {target} count 5 source {source}", - "traceroute": "traceroute inet routing-instance {vrf} {target} wait 1 source {source}", -} -_ipv6_vpn = { - "bgp_route": 'show route protocol bgp table {vrf}.inet6.0 {target} detail | except Label | except Label | except "Next hop type" | except Task | except Address | except "Session Id" | except State | except "Next-hop reference" | except destinations | except "Announcement bits"', - "bgp_aspath": 'show route protocol bgp table {vrf}.inet6.0 aspath-regex "{target}"', - "bgp_community": "show route protocol bgp table {vrf}.inet6.0 community {target}", - "ping": "ping inet6 routing-instance {vrf} {target} count 5 source {source}", - "traceroute": "traceroute inet6 routing-instance {vrf} {target} wait 2 source {source}", -} + +class _IPv4(CommandSet): + """Validation model for non-default dual afi commands.""" + + bgp_route: StrictStr = 'show route protocol bgp table inet.0 {target} detail | except Label | except Label | except "Next hop type" | except Task | except Address | except "Session Id" | except State | except "Next-hop reference" | except destinations | except "Announcement bits"' + bgp_aspath: StrictStr = 'show route protocol bgp table inet.0 aspath-regex "{target}"' + bgp_community: StrictStr = "show route protocol bgp table inet.0 community {target}" + ping: StrictStr = "ping inet {target} count 5 source {source}" + traceroute: StrictStr = "traceroute inet {target} wait 1 source {source}" + + +class _IPv6(CommandSet): + """Validation model for non-default ipv4 commands.""" + + bgp_route: StrictStr = 'show route protocol bgp table inet6.0 {target} detail | except Label | except Label | except "Next hop type" | except Task | except Address | except "Session Id" | except State | except "Next-hop reference" | except destinations | except "Announcement bits"' + bgp_aspath: StrictStr = 'show route protocol bgp table inet6.0 aspath-regex "{target}"' + bgp_community: StrictStr = "show route protocol bgp table inet6.0 community {target}" + ping: StrictStr = "ping inet6 {target} count 5 source {source}" + traceroute: StrictStr = "traceroute inet6 {target} wait 2 source {source}" + + +class _VPNIPv4(CommandSet): + """Validation model for non-default ipv6 commands.""" + + bgp_route: StrictStr = 'show route protocol bgp table {vrf}.inet.0 {target} detail | except Label | except Label | except "Next hop type" | except Task | except Address | except "Session Id" | except State | except "Next-hop reference" | except destinations | except "Announcement bits"' + bgp_aspath: StrictStr = 'show route protocol bgp table {vrf}.inet.0 aspath-regex "{target}"' + bgp_community: StrictStr = "show route protocol bgp table {vrf}.inet.0 community {target}" + ping: StrictStr = "ping inet routing-instance {vrf} {target} count 5 source {source}" + traceroute: StrictStr = "traceroute inet routing-instance {vrf} {target} wait 1 source {source}" + + +class _VPNIPv6(CommandSet): + """Validation model for non-default ipv6 commands.""" + + bgp_route: StrictStr = 'show route protocol bgp table {vrf}.inet6.0 {target} detail | except Label | except Label | except "Next hop type" | except Task | except Address | except "Session Id" | except State | except "Next-hop reference" | except destinations | except "Announcement bits"' + bgp_aspath: StrictStr = 'show route protocol bgp table {vrf}.inet6.0 aspath-regex "{target}"' + bgp_community: StrictStr = "show route protocol bgp table {vrf}.inet6.0 community {target}" + ping: StrictStr = "ping inet6 routing-instance {vrf} {target} count 5 source {source}" + traceroute: StrictStr = "traceroute inet6 routing-instance {vrf} {target} wait 2 source {source}" + _structured = CommandGroup( ipv4_default=CommandSet( - bgp_route="show route protocol bgp table inet.0 {target} detail | display json", - bgp_aspath='show route protocol bgp table inet.0 aspath-regex "{target}" | display json', - bgp_community="show route protocol bgp table inet.0 community {target} | display json", + bgp_route="show route protocol bgp table inet.0 {target} detail | display xml", + bgp_aspath='show route protocol bgp table inet.0 aspath-regex "{target}" detail | display xml', + bgp_community="show route protocol bgp table inet.0 community {target} detail | display xml", ping="ping inet {target} count 5 source {source}", traceroute="traceroute inet {target} wait 1 source {source}", ), ipv6_default=CommandSet( - bgp_route="show route protocol bgp table inet6.0 {target} detail | display json", - bgp_aspath='show route protocol bgp table inet6.0 aspath-regex "{target}" | display json', - bgp_community="show route protocol bgp table inet6.0 community {target} | display json", + bgp_route="show route protocol bgp table inet6.0 {target} detail | display xml", + bgp_aspath='show route protocol bgp table inet6.0 aspath-regex "{target}" detail | display xml', + bgp_community="show route protocol bgp table inet6.0 community {target} detail | display xml", ping="ping inet6 {target} count 5 source {source}", traceroute="traceroute inet6 {target} wait 2 source {source}", ), ipv4_vpn=CommandSet( - bgp_route="show route protocol bgp table {vrf}.inet.0 {target} detail | display json", - bgp_aspath='show route protocol bgp table {vrf}.inet.0 aspath-regex "{target}" | display json', - bgp_community="show route protocol bgp table {vrf}.inet.0 community {target} | display json", + bgp_route="show route protocol bgp table {vrf}.inet.0 {target} detail | display xml", + bgp_aspath='show route protocol bgp table {vrf}.inet.0 aspath-regex "{target}" detail | display xml', + bgp_community="show route protocol bgp table {vrf}.inet.0 community {target} detail | display xml", ping="ping inet routing-instance {vrf} {target} count 5 source {source}", traceroute="traceroute inet routing-instance {vrf} {target} wait 1 source {source}", ), ipv6_vpn=CommandSet( - bgp_route="show route protocol bgp table {vrf}.inet6.0 {target} detail | display json", - bgp_aspath='show route protocol bgp table {vrf}.inet6.0 aspath-regex "{target}" | display json', - bgp_community="show route protocol bgp table {vrf}.inet6.0 community {target} | display json", + bgp_route="show route protocol bgp table {vrf}.inet6.0 {target} detail | display xml", + bgp_aspath='show route protocol bgp table {vrf}.inet6.0 aspath-regex "{target}" detail | display xml', + bgp_community="show route protocol bgp table {vrf}.inet6.0 community {target} detail | display xml", ping="ping inet6 routing-instance {vrf} {target} count 5 source {source}", traceroute="traceroute inet6 routing-instance {vrf} {target} wait 1 source {source}", ), @@ -67,8 +82,12 @@ _structured = CommandGroup( class JuniperCommands(CommandGroup): """Validation model for default juniper commands.""" - ipv4_default: CommandSet = CommandSet(**_ipv4_default) - ipv6_default: CommandSet = CommandSet(**_ipv6_default) - ipv4_vpn: CommandSet = CommandSet(**_ipv4_vpn) - ipv6_vpn: CommandSet = CommandSet(**_ipv6_vpn) - structured: CommandGroup = _structured + ipv4_default: _IPv4 = _IPv4() + ipv6_default: _IPv6 = _IPv6() + ipv4_vpn: _VPNIPv4 = _VPNIPv4() + ipv6_vpn: _VPNIPv6 = _VPNIPv6() + + def __init__(self, **kwargs): + """Initialize command group, ensure structured fields are not overridden.""" + super().__init__(**kwargs) + self.structured = _structured diff --git a/hyperglass/configuration/models/messages.py b/hyperglass/configuration/models/messages.py index 716693b..02b109c 100644 --- a/hyperglass/configuration/models/messages.py +++ b/hyperglass/configuration/models/messages.py @@ -75,7 +75,7 @@ class Messages(HyperglassModel): description="Displayed when a query VRF is not configured on any devices. The hyperglass UI only shows configured VRFs, so this error is most likely to appear when using the hyperglass API. `{vrf_name}` may be used to display the VRF in question.", ) no_output: StrictStr = Field( - "No output.", + "The query completed, but no matching results were found.", title="No Output", description="Displayed when hyperglass can connect to a device and execute a query, but the response is empty.", ) diff --git a/hyperglass/constants.py b/hyperglass/constants.py index 2dd5e19..56f07bd 100644 --- a/hyperglass/constants.py +++ b/hyperglass/constants.py @@ -27,6 +27,7 @@ DNS_OVER_HTTPS = { } PARSED_RESPONSE_FIELDS = ( + ("Prefix", "prefix", "left"), ("Active", "active", None), ("RPKI State", "rpki_state", "center"), ("AS Path", "as_path", "left"), diff --git a/hyperglass/execution/construct.py b/hyperglass/execution/construct.py index f4da2ec..2368a13 100644 --- a/hyperglass/execution/construct.py +++ b/hyperglass/execution/construct.py @@ -8,7 +8,7 @@ hyperglass API modules. # Standard Library import re import json as _json -import operator +from operator import attrgetter # Project from hyperglass.log import log @@ -120,9 +120,17 @@ class Construct: Returns: {str} -- Command string """ - command = operator.attrgetter( - f"{self.device.nos}.{afi.protocol}.{self.query_data.query_type}" - )(commands) + if self.device.structured_output: + cmd_paths = ( + self.device.nos, + "structured", + afi.protocol, + self.query_data.query_type, + ) + else: + cmd_paths = (self.device.nos, afi.protocol, self.query_data.query_type) + + command = attrgetter(".".join(cmd_paths))(commands) return command.format( target=self.target, source=str(afi.source_address), diff --git a/hyperglass/execution/execute.py b/hyperglass/execution/execute.py index 2c1be77..f6510b7 100644 --- a/hyperglass/execution/execute.py +++ b/hyperglass/execution/execute.py @@ -34,6 +34,7 @@ from hyperglass.exceptions import ( DeviceTimeout, ResponseEmpty, ) +from hyperglass.parsing.nos import nos_parsers from hyperglass.configuration import params, devices from hyperglass.parsing.common import parsers from hyperglass.execution.construct import Construct @@ -79,9 +80,23 @@ class Connect: Returns: {str} -- Parsed output """ - for coro in parsers: - output = await coro(commands=self.query, output=output) - return output + log.debug(f"Pre-parsed responses:\n{output}") + parsed = () + if not self.device.structured_output: + for coro in parsers: + for response in output: + _output = await coro(commands=self.query, output=response) + parsed += (_output,) + response = "\n\n".join(parsed) + elif ( + self.device.structured_output + and self.device.nos in nos_parsers.keys() + and self.query_type in nos_parsers[self.device.nos].keys() + ): + func = nos_parsers[self.device.nos][self.query_type] + response = func(output) + log.debug(f"Post-parsed responses:\n{response}") + return response async def scrape_proxied(self): """Connect to a device via an SSH proxy. @@ -150,12 +165,9 @@ class Connect: raw = nm_connect_direct.send_command(query) responses += (raw,) log.debug(f'Raw response for command "{query}":\n{raw}') - response = "\n\n".join(responses) nm_connect_direct.disconnect() - log.debug(f"Response type:\n{type(response)}") - except (NetMikoTimeoutException, NetmikoTimeoutError) as scrape_error: log.error( "Timeout connecting to device {loc}: {e}", @@ -187,7 +199,7 @@ class Connect: proxy=self.device.proxy.name, error=params.messages.general, ) - if response is None: + if not responses: raise ScrapeError( params.messages.connection_error, device_name=self.device.display_name, @@ -195,7 +207,7 @@ class Connect: error=params.messages.no_response, ) signal.alarm(0) - return await self.parsed_response(response) + return await self.parsed_response(responses) async def scrape_direct(self): """Connect directly to a device. @@ -235,8 +247,6 @@ class Connect: responses += (raw,) log.debug(f'Raw response for command "{query}":\n{raw}') - response = "\n\n".join(responses) - nm_connect_direct.disconnect() except (NetMikoTimeoutException, NetmikoTimeoutError) as scrape_error: @@ -260,7 +270,7 @@ class Connect: proxy=None, error=params.messages.authentication_error, ) - if response is None: + if not responses: raise ScrapeError( params.messages.connection_error, device_name=self.device.display_name, @@ -268,7 +278,7 @@ class Connect: error=params.messages.no_response, ) signal.alarm(0) - return await self.parsed_response(response) + return await self.parsed_response(responses) async def rest(self): # noqa: C901 """Connect to a device running hyperglass-agent via HTTP.""" @@ -305,7 +315,7 @@ class Connect: try: async with httpx.AsyncClient(**client_params) as http_client: - responses = [] + responses = () for query in self.query: encoded_query = await jwt_encode( @@ -329,13 +339,11 @@ class Connect: secret=self.device.credential.password.get_secret_value(), ) log.debug(f"Decoded Response: {decoded}") + responses += (decoded,) - responses.append(decoded) else: log.error(raw_response.text) - response = "\n\n".join(responses) - log.debug(f"Output for query {self.query}:\n{response}") except httpx.exceptions.HTTPError as rest_error: msg = parse_exception(rest_error) log.error(f"Error connecting to device {self.device.name}: {msg}") @@ -368,7 +376,7 @@ class Connect: error=params.messages.general, ) - if not response: + if not responses: log.error(f"No response from device {self.device.name}") raise RestError( params.messages.connection_error, @@ -376,7 +384,7 @@ class Connect: error=params.messages.no_response, ) - return await self.parsed_response(response) + return await self.parsed_response(responses) class Execute: diff --git a/hyperglass/parsing/juniper.py b/hyperglass/parsing/juniper.py index 68dae3d..ddb3139 100644 --- a/hyperglass/parsing/juniper.py +++ b/hyperglass/parsing/juniper.py @@ -5,21 +5,40 @@ import xmltodict # Project from hyperglass.log import log -from hyperglass.exceptions import ParsingError +from hyperglass.exceptions import ParsingError, ResponseEmpty +from hyperglass.configuration import params from hyperglass.parsing.models.juniper import JuniperRoute def parse_juniper(output): """Parse a Juniper BGP XML response.""" - try: - parsed = xmltodict.parse(output)["rpc-reply"]["route-information"][ - "route-table" - ] - validated = JuniperRoute(**parsed) - return validated.serialize().export_dict() - except xmltodict.expat.ExpatError as err: - log.critical(str(err)) - raise ParsingError("Error parsing response data") - except KeyError as err: - log.critical(f"'{str(err)}' was not found in the response") - raise ParsingError("Error parsing response data") + data = {} + for i, response in enumerate(output): + try: + parsed = xmltodict.parse(response, force_list=("rt", "rt-entry")) + + if "rpc-reply" in parsed.keys(): + parsed = parsed["rpc-reply"]["route-information"]["route-table"] + elif "route-information" in parsed.keys(): + parsed = parsed["route-information"]["route-table"] + + if "rt" not in parsed: + raise ResponseEmpty(params.messages.no_output) + + validated = JuniperRoute(**parsed) + serialized = validated.serialize().export_dict() + + if i == 0: + data.update(serialized) + else: + data["routes"].extend(serialized["routes"]) + + except xmltodict.expat.ExpatError as err: + log.critical(str(err)) + raise ParsingError("Error parsing response data") + + except KeyError as err: + log.critical(f"'{str(err)}' was not found in the response") + raise ParsingError("Error parsing response data") + + return data diff --git a/hyperglass/parsing/linux.py b/hyperglass/parsing/linux.py new file mode 100644 index 0000000..2aba430 --- /dev/null +++ b/hyperglass/parsing/linux.py @@ -0,0 +1,81 @@ +# flake8: noqa +# WORK IN PROGRESS + +"""Linux-style parsers for ping & traceroute.""" + +# Standard Library +import re + +# Project +from hyperglass.exceptions import ParsingError + + +def _process_numbers(numbers): + """Convert string to float or int.""" + for num in numbers: + num = float(num) + if num.is_integer(): + num = int(num) + yield num + + +def parse_linux_ping(output): + """Parse standard Linux-style ping output to structured data. + + Example: + 64 bytes from 1.1.1.1: icmp_seq=0 ttl=59 time=1.151 ms + 64 bytes from 1.1.1.1: icmp_seq=1 ttl=59 time=1.180 ms + 64 bytes from 1.1.1.1: icmp_seq=2 ttl=59 time=1.170 ms + 64 bytes from 1.1.1.1: icmp_seq=3 ttl=59 time=1.338 ms + 64 bytes from 1.1.1.1: icmp_seq=4 ttl=59 time=4.913 ms + + --- 1.1.1.1 ping statistics --- + 5 packets transmitted, 5 packets received, 0% packet loss + round-trip min/avg/max/stddev = 1.151/1.950/4.913/1.483 ms + """ + try: + # Extract target host + host = re.findall(r"^PING (.+) \(.+\): \d+ data bytes", output)[0] + + # Separate echo replies from summary info + replies, _stats = re.split(r"--- .+ ---", output) + replies = [l for l in replies.splitlines()[1:] if l] + + reply_stats = [] + for line in replies: + # Extract the numerical values from each echo reply line + bytes_seq_ttl_rtt = re.findall( + r"(\d+) bytes.+ icmp_seq=(\d+) ttl=(\d+) time=(\d+\.\d+).*", line + )[0] + + _bytes, seq, ttl, rtt = _process_numbers(bytes_seq_ttl_rtt) + + reply_stats.append( + {"bytes": _bytes, "sequence": seq, "ttl": ttl, "rtt": rtt} + ) + + stats = [l for l in _stats.splitlines() if l] + + # Extract the top summary line numbers & process + tx_rx_loss = re.findall(r"(\d+) .+, (\d+) .+, (\d+)\%.+", stats[0])[0] + tx, rx, loss = _process_numbers(tx_rx_loss) + + # Extract the bottom summary line numbers & process + rt = stats[1].split(" = ")[1] + min_max_avg = rt.split("/")[:-1] + _min, _max, _avg = _process_numbers(min_max_avg) + + return { + "host": host, + "transmitted": tx, + "received": rx, + "loss_percent": loss, + "min_rtt": _min, + "max_rtt": _max, + "avg_rtt": _avg, + "replies": reply_stats, + } + + except (KeyError, ValueError) as err: + # KeyError for empty findalls, ValueError for regex errors + raise ParsingError("Error parsing ping response: {e}", e=str(err)) diff --git a/hyperglass/parsing/models/frr.py b/hyperglass/parsing/models/frr.py new file mode 100644 index 0000000..2ae7521 --- /dev/null +++ b/hyperglass/parsing/models/frr.py @@ -0,0 +1,114 @@ +"""Data Models for Parsing FRRouting JSON Response.""" + +# Standard Library +from typing import List +from datetime import datetime + +# Third Party +from pydantic import StrictInt, StrictStr, StrictBool, constr, root_validator + +# Project +from hyperglass.log import log +from hyperglass.models import HyperglassModel +from hyperglass.parsing.models.serialized import ParsedRoutes + + +def _alias_generator(field): + components = field.split("_") + return components[0] + "".join(x.title() for x in components[1:]) + + +class _FRRBase(HyperglassModel): + class Config: + alias_generator = _alias_generator + extra = "ignore" + + +class FRRNextHop(_FRRBase): + """FRR Next Hop Model.""" + + ip: StrictStr + afi: StrictStr + metric: StrictInt + accessible: StrictBool + used: StrictBool + + +class FRRPeer(_FRRBase): + """FRR Peer Model.""" + + peer_id: StrictStr + router_id: StrictStr + type: constr(regex=r"(internal|external)") + + +class FRRPath(_FRRBase): + """FRR Path Model.""" + + aspath: List[StrictInt] + aggregator_as: StrictInt + aggregator_id: StrictStr + med: StrictInt = 0 + localpref: StrictInt + weight: StrictInt + valid: StrictBool + last_update: StrictInt + bestpath: StrictBool + community: List[StrictStr] + nexthops: List[FRRNextHop] + peer: FRRPeer + + @root_validator(pre=True) + def validate_path(cls, values): + """Extract meaningful data from FRR response.""" + new = values.copy() + new["aspath"] = values["aspath"]["segments"][0]["list"] + new["community"] = values["community"]["list"] + new["lastUpdate"] = values["lastUpdate"]["epoch"] + bestpath = values.get("bestpath", {}) + new["bestpath"] = bestpath.get("overall", False) + return new + + +class FRRRoute(_FRRBase): + """FRR Route Model.""" + + prefix: StrictStr + paths: List[FRRPath] = [] + + def serialize(self): + """Convert the FRR-specific fields to standard parsed data model.""" + + # TODO: somehow, get the actual VRF + vrf = "default" + + routes = [] + for route in self.paths: + now = datetime.utcnow().timestamp() + then = datetime.utcfromtimestamp(route.last_update).timestamp() + age = int(now - then) + routes.append( + { + "prefix": self.prefix, + "active": route.bestpath, + "age": age, + "weight": route.weight, + "med": route.med, + "local_preference": route.localpref, + "as_path": route.aspath, + "communities": route.community, + "next_hop": route.nexthops[0].ip, + "source_as": route.aggregator_as, + "source_rid": route.aggregator_id, + "peer_rid": route.peer.peer_id, + # TODO: somehow, get the actual RPKI state + "rpki_state": 3, + } + ) + + serialized = ParsedRoutes( + vrf=vrf, count=len(routes), routes=routes, winning_weight="high", + ) + + log.info("Serialized FRR response: {}", serialized) + return serialized diff --git a/hyperglass/parsing/models/frr_bgp_community.json b/hyperglass/parsing/models/frr_bgp_community.json new file mode 100644 index 0000000..a28d7ad --- /dev/null +++ b/hyperglass/parsing/models/frr_bgp_community.json @@ -0,0 +1,84 @@ +{ + "vrfId": 0, + "vrfName": "default", + "tableVersion": 32856694, + "routerId": "199.34.92.3", + "defaultLocPrf": 100, + "localAS": 14525, + "routes": { + "38.29.214.0/24": [ + { + "valid": true, + "bestpath": true, + "pathFrom": "internal", + "prefix": "38.29.214.0", + "prefixLen": 24, + "network": "38.29.214.0/24", + "med": 0, + "metric": 0, + "localpref": 450, + "locPrf": 450, + "weight": 200, + "peerId": "199.34.92.1", + "aspath": "", + "path": "", + "origin": "IGP", + "nexthops": [{ "ip": "199.34.92.1", "afi": "ipv4", "used": true }] + } + ], + "199.34.92.0/23": [ + { + "valid": true, + "pathFrom": "internal", + "prefix": "199.34.92.0", + "prefixLen": 23, + "network": "199.34.92.0/23", + "med": 0, + "metric": 0, + "localpref": 450, + "locPrf": 450, + "weight": 200, + "peerId": "199.34.92.10", + "aspath": "", + "path": "", + "origin": "IGP", + "nexthops": [{ "ip": "199.34.92.10", "afi": "ipv4", "used": true }] + }, + { + "valid": true, + "pathFrom": "internal", + "prefix": "199.34.92.0", + "prefixLen": 23, + "network": "199.34.92.0/23", + "med": 0, + "metric": 0, + "localpref": 450, + "locPrf": 450, + "weight": 200, + "peerId": "199.34.92.9", + "aspath": "", + "path": "", + "origin": "IGP", + "nexthops": [{ "ip": "199.34.92.9", "afi": "ipv4", "used": true }] + }, + { + "valid": true, + "bestpath": true, + "pathFrom": "internal", + "prefix": "199.34.92.0", + "prefixLen": 23, + "network": "199.34.92.0/23", + "med": 0, + "metric": 0, + "localpref": 450, + "locPrf": 450, + "weight": 200, + "peerId": "199.34.92.1", + "aspath": "", + "path": "", + "origin": "IGP", + "nexthops": [{ "ip": "199.34.92.1", "afi": "ipv4", "used": true }] + } + ] + } +} diff --git a/hyperglass/parsing/models/frr_bgp_route.json b/hyperglass/parsing/models/frr_bgp_route.json new file mode 100644 index 0000000..90b936d --- /dev/null +++ b/hyperglass/parsing/models/frr_bgp_route.json @@ -0,0 +1,224 @@ +{ + "prefix": "1.1.1.0/24", + "paths": [ + { + "aspath": { + "string": "174 13335", + "segments": [ + { + "type": "as-sequence", + "list": [174, 13335] + } + ], + "length": 2 + }, + "aggregatorAs": 13335, + "aggregatorId": "108.162.239.1", + "origin": "IGP", + "med": 25090, + "metric": 25090, + "localpref": 100, + "weight": 100, + "valid": true, + "community": { + "string": "174:21001 174:22003 14525:0 14525:40 14525:1021 14525:2840 14525:3003 14525:4004 14525:9001", + "list": [ + "174:21001", + "174:22003", + "14525:0", + "14525:40", + "14525:1021", + "14525:2840", + "14525:3003", + "14525:4004", + "14525:9001" + ] + }, + "lastUpdate": { + "epoch": 1588417118, + "string": "Sat May 2 10:58:38 2020\n" + }, + "nexthops": [ + { + "ip": "199.34.92.7", + "afi": "ipv4", + "metric": 1100, + "accessible": true, + "used": true + } + ], + "peer": { + "peerId": "199.34.92.7", + "routerId": "199.34.92.7", + "hostname": "er01.dtn01", + "type": "internal" + } + }, + { + "aspath": { + "string": "1299 13335", + "segments": [ + { + "type": "as-sequence", + "list": [1299, 13335] + } + ], + "length": 2 + }, + "aggregatorAs": 13335, + "aggregatorId": "162.158.140.1", + "origin": "IGP", + "med": 0, + "metric": 0, + "localpref": 150, + "weight": 200, + "valid": true, + "bestpath": { + "bestpathFromAs": 1299 + }, + "community": { + "string": "1299:35000 14525:0 14525:40 14525:1021 14525:2840 14525:3001 14525:4001 14525:9003", + "list": [ + "1299:35000", + "14525:0", + "14525:40", + "14525:1021", + "14525:2840", + "14525:3001", + "14525:4001", + "14525:9003" + ] + }, + "lastUpdate": { + "epoch": 1588584980, + "string": "Mon May 4 09:36:20 2020\n" + }, + "nexthops": [ + { + "ip": "199.34.92.9", + "afi": "ipv4", + "metric": 200, + "accessible": true, + "used": true + } + ], + "peer": { + "peerId": "199.34.92.9", + "routerId": "199.34.92.9", + "type": "internal" + } + }, + { + "aspath": { + "string": "6939 13335", + "segments": [ + { + "type": "as-sequence", + "list": [6939, 13335] + } + ], + "length": 2 + }, + "aggregatorAs": 13335, + "aggregatorId": "172.68.129.1", + "origin": "IGP", + "med": 0, + "metric": 0, + "localpref": 100, + "weight": 100, + "valid": true, + "bestpath": { + "bestpathFromAs": 6939 + }, + "community": { + "string": "6939:7107 6939:8840 6939:9001 14525:0 14525:40 14525:1021 14525:2840 14525:3002 14525:4003 14525:9002", + "list": [ + "6939:7107", + "6939:8840", + "6939:9001", + "14525:0", + "14525:40", + "14525:1021", + "14525:2840", + "14525:3002", + "14525:4003", + "14525:9002" + ] + }, + "lastUpdate": { + "epoch": 1586990260, + "string": "Wed Apr 15 22:37:40 2020\n" + }, + "nexthops": [ + { + "ip": "199.34.92.6", + "afi": "ipv4", + "metric": 151, + "accessible": true, + "used": true + } + ], + "peer": { + "peerId": "199.34.92.6", + "routerId": "199.34.92.6", + "type": "internal" + } + }, + { + "aspath": { + "string": "174 13335", + "segments": [ + { + "type": "as-sequence", + "list": [174, 13335] + } + ], + "length": 2 + }, + "aggregatorAs": 13335, + "aggregatorId": "162.158.140.1", + "origin": "IGP", + "med": 2020, + "metric": 2020, + "localpref": 150, + "weight": 200, + "valid": true, + "bestpath": { + "bestpathFromAs": 174, + "overall": true + }, + "community": { + "string": "174:21001 174:22013 14525:0 14525:20 14525:1021 14525:2840 14525:3001 14525:4001 14525:9001", + "list": [ + "174:21001", + "174:22013", + "14525:0", + "14525:20", + "14525:1021", + "14525:2840", + "14525:3001", + "14525:4001", + "14525:9001" + ] + }, + "lastUpdate": { + "epoch": 1588584997, + "string": "Mon May 4 09:36:37 2020\n" + }, + "nexthops": [ + { + "ip": "199.34.92.1", + "afi": "ipv4", + "metric": 101, + "accessible": true, + "used": true + } + ], + "peer": { + "peerId": "199.34.92.1", + "routerId": "199.34.92.1", + "type": "internal" + } + } + ] +} diff --git a/hyperglass/parsing/models/juniper.py b/hyperglass/parsing/models/juniper.py index 7813076..6329821 100644 --- a/hyperglass/parsing/models/juniper.py +++ b/hyperglass/parsing/models/juniper.py @@ -121,7 +121,7 @@ class JuniperRoute(_JuniperBase): total_route_count: int active_route_count: int hidden_route_count: int - rt: JuniperRouteTable + rt: List[JuniperRouteTable] def serialize(self): """Convert the Juniper-specific fields to standard parsed data model.""" @@ -131,37 +131,35 @@ class JuniperRoute(_JuniperBase): else: vrf = vrf_parts[0] - prefix = "/".join( - str(i) for i in (self.rt.rt_destination, self.rt.rt_prefix_length) - ) - - structure = { - "vrf": vrf, - "prefix": prefix, - "count": self.rt.rt_entry_count, - "winning_weight": "low", - } - routes = [] - for route in self.rt.rt_entry: - routes.append( - { - "active": route.active_tag, - "age": route.age, - "weight": route.preference, - "med": route.metric, - "local_preference": route.local_preference, - "as_path": route.as_path, - "communities": route.communities, - "next_hop": route.next_hop, - "source_as": route.source_as, - "source_rid": route.source_rid, - "peer_rid": route.peer_rid, - "rpki_state": route.validation_state, - } + count = 0 + for table in self.rt: + count += table.rt_entry_count + prefix = "/".join( + str(i) for i in (table.rt_destination, table.rt_prefix_length) ) + for route in table.rt_entry: + routes.append( + { + "prefix": prefix, + "active": route.active_tag, + "age": route.age, + "weight": route.preference, + "med": route.metric, + "local_preference": route.local_preference, + "as_path": route.as_path, + "communities": route.communities, + "next_hop": route.next_hop, + "source_as": route.source_as, + "source_rid": route.source_rid, + "peer_rid": route.peer_rid, + "rpki_state": route.validation_state, + } + ) - serialized = ParsedRoutes(routes=routes, **structure) + serialized = ParsedRoutes( + vrf=vrf, count=count, routes=routes, winning_weight="low", + ) log.info("Serialized Juniper response: {}", serialized) return serialized diff --git a/hyperglass/parsing/models/serialized.py b/hyperglass/parsing/models/serialized.py index da8b2ba..f1343c7 100644 --- a/hyperglass/parsing/models/serialized.py +++ b/hyperglass/parsing/models/serialized.py @@ -13,6 +13,7 @@ from hyperglass.models import HyperglassModel class ParsedRouteEntry(HyperglassModel): """Per-Route Response Model.""" + prefix: StrictStr active: StrictBool age: StrictInt weight: StrictInt @@ -31,7 +32,6 @@ class ParsedRoutes(HyperglassModel): """Parsed Response Model.""" vrf: StrictStr - prefix: StrictStr count: StrictInt = 0 routes: List[ParsedRouteEntry] winning_weight: constr(regex=r"(low|high)") diff --git a/hyperglass/parsing/nos.py b/hyperglass/parsing/nos.py new file mode 100644 index 0000000..b23db2f --- /dev/null +++ b/hyperglass/parsing/nos.py @@ -0,0 +1,10 @@ +# Project +from hyperglass.parsing.juniper import parse_juniper + +nos_parsers = { + "juniper": { + "bgp_route": parse_juniper, + "bgp_aspath": parse_juniper, + "bgp_community": parse_juniper, + } +} diff --git a/hyperglass/ui/components/BGPTable.js b/hyperglass/ui/components/BGPTable.js new file mode 100644 index 0000000..e20580b --- /dev/null +++ b/hyperglass/ui/components/BGPTable.js @@ -0,0 +1,251 @@ +import * as React from "react"; +import { + Flex, + Icon, + Popover, + PopoverArrow, + PopoverContent, + PopoverTrigger, + Text, + Tooltip, + useColorMode +} from "@chakra-ui/core"; +import { MdLastPage } from "react-icons/md"; +import dayjs from "dayjs"; +import relativeTimePlugin from "dayjs/plugin/relativeTime"; +import utcPlugin from "dayjs/plugin/utc"; +import useConfig from "~/components/HyperglassProvider"; +import Table from "~/components/Table/index"; + +dayjs.extend(relativeTimePlugin); +dayjs.extend(utcPlugin); + +const isActiveColor = { + true: { dark: "green.300", light: "green.500" }, + false: { dark: "gray.300", light: "gray.500" } +}; + +const arrowColor = { + true: { dark: "blackAlpha.500", light: "blackAlpha.500" }, + false: { dark: "whiteAlpha.300", light: "blackAlpha.500" } +}; + +const rpkiIcon = ["not-allowed", "check-circle", "warning", "question"]; + +const rpkiColor = { + true: { + dark: ["red.500", "green.600", "yellow.500", "gray.800"], + light: ["red.500", "green.500", "yellow.500", "gray.600"] + }, + false: { + dark: ["red.300", "green.300", "yellow.300", "gray.300"], + light: ["red.400", "green.500", "yellow.400", "gray.500"] + } +}; + +const makeColumns = fields => { + return fields.map(pair => { + const [header, accessor, align] = pair; + let columnConfig = { + Header: header, + accessor: accessor, + align: align, + hidden: false + }; + if (align === null) { + columnConfig.hidden = true; + } + return columnConfig; + }); +}; + +// const longestASNLength = asPath => { +// if (asPath.length === 0) { +// return 0 +// } +// const longest = asPath.reduce((l, c) => { +// const strLongest = String(l); +// const strCurrent = String(c); +// return strCurrent.length > strLongest.length ? strCurrent : strLongest; +// }); +// return longest.length; +// }; + +const MonoField = ({ v, ...props }) => ( + + {v} + +); + +const Active = ({ isActive }) => { + const { colorMode } = useColorMode(); + return ( + + ); +}; + +const Age = ({ inSeconds }) => { + const now = dayjs.utc(); + const then = now.subtract(inSeconds, "seconds"); + return ( + + {now.to(then, true)} + + ); +}; + +const Weight = ({ weight, winningWeight }) => { + const fixMeText = + winningWeight === "low" + ? "Lower Weight is Preferred" + : "Higher Weight is Preferred"; + return ( + + + {weight} + + + ); +}; + +const ASPath = ({ path, active }) => { + const { colorMode } = useColorMode(); + if (path.length === 0) { + return ; + } + let paths = []; + path.map((asn, i) => { + const asnStr = String(asn); + i !== 0 && + paths.push( + + ); + paths.push( + + {asnStr} + + ); + }); + return paths; +}; + +const Communities = ({ communities }) => { + const { colorMode } = useColorMode(); + let component; + communities.length === 0 + ? (component = ( + + + + )) + : (component = ( + + + + + + + {communities.map(c => ( + + ))} + + + )); + return component; +}; + +const RPKIState = ({ state, active }) => { + const { web } = useConfig(); + const { colorMode } = useColorMode(); + const stateText = [ + web.text.rpki_invalid, + web.text.rpki_valid, + web.text.rpki_unknown, + web.text.rpki_unverified + ]; + return ( + + + + ); +}; + +const Cell = ({ data, rawData, longestASN }) => { + const component = { + prefix: , + active: , + age: , + weight: ( + + ), + med: , + local_preference: , + as_path: ( + + ), + communities: , + next_hop: , + source_as: , + source_rid: , + peer_rid: , + rpki_state: + }; + return component[data.column.id] ?? <> ; +}; + +const BGPTable = ({ children: data, ...props }) => { + const config = useConfig(); + const columns = makeColumns(config.parsed_data_fields); + // const allASN = data.routes.map(r => r.as_path).flat(); + // const asLength = longestASNLength(allASN); + + return ( + + } + bordersHorizontal + rowHighlightBg="green" + /> + + ); +}; + +BGPTable.displayName = "BGPTable"; + +export default BGPTable; diff --git a/hyperglass/ui/components/Result.js b/hyperglass/ui/components/Result.js index 8b5da1a..4a59fc0 100644 --- a/hyperglass/ui/components/Result.js +++ b/hyperglass/ui/components/Result.js @@ -1,4 +1,5 @@ -import * as React from "react"; +/** @jsx jsx */ +import { jsx } from "@emotion/core"; import { useEffect, useState } from "react"; import { AccordionItem, @@ -11,7 +12,8 @@ import { Flex, Tooltip, Text, - useColorMode + useColorMode, + useTheme } from "@chakra-ui/core"; import styled from "@emotion/styled"; import LightningBolt from "~/components/icons/LightningBolt"; @@ -25,6 +27,8 @@ import CopyButton from "~/components/CopyButton"; import RequeryButton from "~/components/RequeryButton"; import ResultHeader from "~/components/ResultHeader"; import CacheTimeout from "~/components/CacheTimeout"; +import BGPTable from "~/components/BGPTable"; +import TextOutput from "~/components/TextOutput"; format.extend(String.prototype, {}); @@ -54,16 +58,25 @@ const AccordionHeaderWrapper = styled(Flex)` } `; +const structuredDataComponent = { + bgp_route: BGPTable, + bgp_aspath: BGPTable, + bgp_community: BGPTable, + ping: TextOutput, + traceroute: TextOutput +}; + const statusMap = { success: "success", warning: "warning", error: "warning", danger: "error" }; -const bg = { dark: "gray.800", light: "blackAlpha.100" }; + const color = { dark: "white", light: "black" }; -const selectionBg = { dark: "white", light: "black" }; -const selectionColor = { dark: "black", light: "white" }; +const scrollbar = { dark: "whiteAlpha.300", light: "blackAlpha.300" }; +const scrollbarHover = { dark: "whiteAlpha.400", light: "blackAlpha.400" }; +const scrollbarBg = { dark: "whiteAlpha.50", light: "blackAlpha.50" }; const Result = React.forwardRef( ( @@ -81,6 +94,7 @@ const Result = React.forwardRef( ref ) => { const config = useConfig(); + const theme = useTheme(); const { isSm } = useMedia(); const { colorMode } = useColorMode(); let [{ data, loading, error }, refetch] = useAxios({ @@ -103,12 +117,6 @@ const Result = React.forwardRef( setOpen(!isOpen); setOverride(true); }; - let cleanOutput = - data && - data.output - .split("\\n") - .join("\n") - .replace(/\n\n/g, "\n"); const errorKw = (error && error.response?.data?.keywords) || []; @@ -132,6 +140,11 @@ const Result = React.forwardRef( statusMap[error.response?.data?.level]) ?? "error"; + let Output = TextOutput; + if (data?.format === "application/json") { + Output = structuredDataComponent[queryType]; + } + useEffect(() => { !loading && resultsComplete === null && setComplete(index); }, [loading, resultsComplete]); @@ -148,7 +161,7 @@ const Result = React.forwardRef( css={css({ "&:last-of-type": { borderBottom: "none" }, "&:first-of-type": { borderTop: "none" } - })} + })(theme)} > - + @@ -184,7 +197,21 @@ const Result = React.forwardRef( - {data && !error && ( - - {cleanOutput} - - )} + {!error && data && {data?.output}} {error && ( @@ -264,45 +268,6 @@ const Result = React.forwardRef( )} )} - {/* {config.cache.show_text && data && !error && isSm ? ( - <> - - - - - - - - ) : config.cache.show_text && data && !error ? ( - <> - - - - - - - - ) : null} */} diff --git a/hyperglass/ui/components/Table/index.js b/hyperglass/ui/components/Table/index.js index b5be5c8..b50c894 100644 --- a/hyperglass/ui/components/Table/index.js +++ b/hyperglass/ui/components/Table/index.js @@ -60,7 +60,6 @@ const Table = ({ nextPage, previousPage, setPageSize, - setHiddenColumns, state: { pageIndex, pageSize } } = useTable( { diff --git a/hyperglass/ui/components/TextOutput.js b/hyperglass/ui/components/TextOutput.js new file mode 100644 index 0000000..093032d --- /dev/null +++ b/hyperglass/ui/components/TextOutput.js @@ -0,0 +1,44 @@ +/** @jsx jsx */ +import { jsx } from "@emotion/core"; +import { Box, css, useColorMode } from "@chakra-ui/core"; + +const bg = { dark: "gray.800", light: "blackAlpha.100" }; +const color = { dark: "white", light: "black" }; +const selectionBg = { dark: "white", light: "black" }; +const selectionColor = { dark: "black", light: "white" }; + +const TextOutput = ({ children, ...props }) => { + const { colorMode } = useColorMode(); + + return ( + + {children + .split("\\n") + .join("\n") + .replace(/\n\n/g, "\n")} + + ); +}; + +TextOutput.displayName = "TextOutput"; +export default TextOutput;