add native mikrotik support

hyperglass/constants.py

@@ -65,6 +65,7 @@ TRANSPORT_REST = ("frr", "bird")
 SCRAPE_HELPERS = {
     "junos": "juniper",
     "ios": "cisco_ios",
+    "mikrotik": "mikrotik_routeros",
 }
 
 DRIVER_MAP = {

hyperglass/execution/drivers/ssh_netmiko.py

@@ -10,8 +10,6 @@ from typing import Iterable
 # Third Party
 from netmiko import (
     ConnectHandler,
-    NetmikoAuthError,
-    NetmikoTimeoutError,
     NetMikoTimeoutException,
     NetMikoAuthenticationException,
 )
@@ -20,7 +18,22 @@ from netmiko import (
 from hyperglass.log import log
 from hyperglass.exceptions import AuthError, ScrapeError, DeviceTimeout
 from hyperglass.configuration import params
-from hyperglass.execution.drivers.ssh import SSHConnection
+
+from .ssh import SSHConnection
+
+netmiko_nos_globals = {
+    # Netmiko doesn't currently handle Mikrotik echo verification well,
+    # see ktbyers/netmiko#1600
+    "mikrotik_routeros": {"global_cmd_verify": False},
+    "mikrotik_switchos": {"global_cmd_verify": False},
+}
+
+netmiko_nos_send_args = {
+    # Netmiko doesn't currently handle the Mikrotik prompt properly, see
+    # ktbyers/netmiko#1956
+    "mikrotik_routeros": {"expect_string": r"\S+\s\>\s$"},
+    "mikrotik_switchos": {"expect_string": r"\S+\s\>\s$"},
+}
 
 
 class NetmikoConnection(SSHConnection):
@@ -42,6 +55,10 @@ class NetmikoConnection(SSHConnection):
         else:
             log.debug("Connecting directly to {}", self.device.name)
 
+        global_args = netmiko_nos_globals.get(self.device.nos, {})
+
+        send_args = netmiko_nos_send_args.get(self.device.nos, {})
+
         netmiko_args = {
             "host": host or self.device._target,
             "port": port or self.device.port,
@@ -51,6 +68,7 @@ class NetmikoConnection(SSHConnection):
             "global_delay_factor": params.netmiko_delay_factor,
             "timeout": math.floor(params.request_timeout * 1.25),
             "session_timeout": math.ceil(params.request_timeout - 1),
+            **global_args,
         }
 
         try:
@@ -59,13 +77,13 @@ class NetmikoConnection(SSHConnection):
             responses = ()
 
             for query in self.query:
-                raw = nm_connect_direct.send_command(query)
+                raw = nm_connect_direct.send_command(query, **send_args)
                 responses += (raw,)
                 log.debug(f'Raw response for command "{query}":\n{raw}')
 
             nm_connect_direct.disconnect()
 
-        except (NetMikoTimeoutException, NetmikoTimeoutError) as scrape_error:
+        except NetMikoTimeoutException as scrape_error:
             log.error(str(scrape_error))
             raise DeviceTimeout(
                 params.messages.connection_error,
@@ -73,7 +91,7 @@ class NetmikoConnection(SSHConnection):
                 proxy=None,
                 error=params.messages.request_timeout,
             )
-        except (NetMikoAuthenticationException, NetmikoAuthError) as auth_error:
+        except NetMikoAuthenticationException as auth_error:
             log.error(
                 "Error authenticating to device {loc}: {e}",
                 loc=self.device.name,

hyperglass/models/commands/_mikrotik_base.py

@@ -0,0 +1,55 @@
+"""Base Mikrotik Commands Model."""
+
+# Third Party
+from pydantic import StrictStr
+
+from .common import CommandSet, CommandGroup
+
+
+class _IPv4(CommandSet):
+    """Default commands for ipv4 commands."""
+
+    bgp_community: StrictStr = "ip route print where bgp-communities={target}"
+    bgp_aspath: StrictStr = "ip route print where bgp-as-path={target}"
+    bgp_route: StrictStr = "ip route print where dst-address={target}"
+    ping: StrictStr = "ping src-address={source} count=5 {target}"
+    traceroute: StrictStr = "tool traceroute src-address={source} timeout=1 duration=5 count=1 {target}"
+
+
+class _IPv6(CommandSet):
+    """Default commands for ipv6 commands."""
+
+    bgp_community: StrictStr = "ipv6 route print where bgp-communities={target}"
+    bgp_aspath: StrictStr = "ipv6 route print where bgp-as-path={target}"
+    bgp_route: StrictStr = "ipv6 route print where dst-address={target}"
+    ping: StrictStr = "ping src-address={source} count=5 {target}"
+    traceroute: StrictStr = "tool traceroute src-address={source} timeout=1 duration=5 count=1 {target}"
+
+
+class _VPNIPv4(CommandSet):
+    """Default commands for dual afi commands."""
+
+    bgp_community: StrictStr = "ip route print where bgp-communities={target} routing-mark={vrf}"
+    bgp_aspath: StrictStr = "ip route print where bgp-as-path={target} routing-mark={vrf}"
+    bgp_route: StrictStr = "ip route print where dst-address={target} routing-mark={vrf}"
+    ping: StrictStr = "ping src-address={source} count=5 routing-table={vrf} {target}"
+    traceroute: StrictStr = "tool traceroute src-address={source} timeout=1 duration=5 count=1 routing-table={vrf} {target}"
+
+
+class _VPNIPv6(CommandSet):
+    """Default commands for dual afi commands."""
+
+    bgp_community: StrictStr = "ipv6 route print where bgp-communities={target} routing-mark={vrf}"
+    bgp_aspath: StrictStr = "ipv6 route print where bgp-as-path={target} routing-mark={vrf}"
+    bgp_route: StrictStr = "ipv6 route print where dst-address={target} routing-mark={vrf}"
+    ping: StrictStr = "ping src-address={source} count=5 routing-table={vrf} {target}"
+    traceroute: StrictStr = "tool traceroute src-address={source} timeout=1 duration=5 count=1 routing-table={vrf} {target}"
+
+
+class MikrotikCommands(CommandGroup):
+    """Validation model for default mikrotik commands."""
+
+    ipv4_default: _IPv4 = _IPv4()
+    ipv6_default: _IPv6 = _IPv6()
+    ipv4_vpn: _VPNIPv4 = _VPNIPv4()
+    ipv6_vpn: _VPNIPv6 = _VPNIPv6()

hyperglass/models/commands/mikrotik_routeros.py

@@ -0,0 +1,8 @@
+"""Mikrotik RouterOS Commands Model."""
+from ._mikrotik_base import MikrotikCommands
+
+
+class MikrotikRouterOS(MikrotikCommands):
+    """Alias for mikrotik_routeros."""
+
+    pass

hyperglass/models/commands/mikrotik_switchos.py

@@ -0,0 +1,8 @@
+"""Mikrotik SwitchOS Commands Model."""
+from ._mikrotik_base import MikrotikCommands
+
+
+class MikrotikSwitchOS(MikrotikCommands):
+    """Alias for mikrotik_switchos."""
+
+    pass

hyperglass/parsing/mikrotik.py

@@ -0,0 +1,52 @@
+"""Mikrotik Output Parsing Functions."""
+
+# Standard Library
+import re
+
+END_COLUMNS = ("DISTANCE", "STATUS")
+
+
+def parse_mikrotik(output: str):
+    """Parse Mikrotik output to remove garbage."""
+    if output.split()[-1] in END_COLUMNS:
+        # Mikrotik shows the columns with no rows if there is no data.
+        # Rather than send back an empty table, send back an empty
+        # response which is handled with a warning message.
+        output = ""
+    else:
+        remove_lines = ()
+        all_lines = output.splitlines()
+        # Starting index for rows (after the column row).
+        start = 1
+        # Extract the column row.
+        column_line = " ".join(all_lines[0].split())
+
+        for i, line in enumerate(all_lines[1:]):
+            # Remove all the newline characters (which differ line to
+            # line) for comparison purposes.
+            normalized = " ".join(line.split())
+
+            # Remove ansii characters that aren't caught by Netmiko.
+            normalized = re.sub(r"\\x1b\[\S{2}\s", "", normalized)
+
+            if column_line in normalized:
+                # Mikrotik often re-inserts the column row in the output,
+                # effectively 'starting over'. In that case, re-assign
+                # the column row and starting index to that point.
+                column_line = re.sub(r"\[\S{2}\s", "", line)
+                start = i + 2
+
+            if "[Q quit|D dump|C-z pause]" in normalized:
+                # Remove Mikrotik's unhelpful helpers from the output.
+                remove_lines += (i + 1,)
+
+        # Combine the column row and the data rows from the starting
+        # index onward.
+        lines = [column_line, *all_lines[start:]]
+
+        # Remove any lines marked for removal and re-join with a single
+        # newline character.
+        lines = [l for i, l in enumerate(lines) if i not in remove_lines]
+        output = "\n".join(lines)
+
+    return output