improve response parsing options

hyperglass/execution/drivers/_common.py

@@ -5,12 +5,13 @@ from typing import Iterable
 
 # Project
 from hyperglass.log import log
-from hyperglass.parsing.nos import nos_parsers
+from hyperglass.models.api import Query
+from hyperglass.parsing.nos import scrape_parsers, structured_parsers
 from hyperglass.parsing.common import parsers
-from hyperglass.api.models.query import Query
-from hyperglass.execution.construct import Construct
 from hyperglass.configuration.models.devices import Device
 
+from ._construct import Construct
+
 
 class Connection:
     """Base transport driver class."""
@@ -24,26 +25,40 @@ class Connection:
         self._query = Construct(device=self.device, query_data=self.query_data)
         self.query = self._query.queries()
 
-    async def parsed_response(self, output: Iterable) -> str:
+    async def parsed_response(  # noqa: C901 ("too complex")
+        self, output: Iterable
+    ) -> str:
         """Send output through common parsers."""
 
         log.debug("Pre-parsed responses:\n{}", output)
         parsed = ()
         response = None
 
-        nos_to_parse = nos_parsers.keys()
-        query_types_to_parse = nos_parsers.get(self.device.nos, {}).keys()
+        structured_nos = structured_parsers.keys()
+        structured_query_types = structured_parsers.get(self.device.nos, {}).keys()
+
+        scrape_nos = scrape_parsers.keys()
+        scrape_query_types = scrape_parsers.get(self.device.nos, {}).keys()
 
         if not self.device.structured_output:
-            for coro in parsers:
+            _parsed = ()
+            for func in parsers:
                 for response in output:
-                    _output = await coro(commands=self.query, output=response)
+                    _output = func(commands=self.query, output=response)
+                    _parsed += (_output,)
+            if self.device.nos in scrape_nos and self.query_type in scrape_query_types:
+                func = scrape_parsers[self.device.nos][self.query_type]
+                for response in _parsed:
+                    _output = func(response)
                     parsed += (_output,)
+            else:
+                parsed += _parsed
+
             response = "\n\n".join(parsed)
         elif (
             self.device.structured_output
-            and self.device.nos in nos_to_parse
-            and self.query_type not in query_types_to_parse
+            and self.device.nos in structured_nos
+            and self.query_type not in structured_query_types
         ):
             for coro in parsers:
                 for response in output:
@@ -52,10 +67,10 @@ class Connection:
             response = "\n\n".join(parsed)
         elif (
             self.device.structured_output
-            and self.device.nos in nos_to_parse
-            and self.query_type in query_types_to_parse
+            and self.device.nos in structured_nos
+            and self.query_type in structured_query_types
         ):
-            func = nos_parsers[self.device.nos][self.query_type]
+            func = structured_parsers[self.device.nos][self.query_type]
             response = func(output)
 
         if response is None:

hyperglass/parsing/common.py

@@ -1,7 +1,7 @@
 """Command parsers applied to all unstructured output."""
 
 
-async def remove_command(commands, output):
+def remove_command(commands, output):
     """Remove anything before the command if found in output.
 
     Arguments:

hyperglass/parsing/nos.py

@@ -1,12 +1,29 @@
 """Map NOS and Commands to Parsing Functions."""
 
-# Project
-from hyperglass.parsing.juniper import parse_juniper
+from .juniper import parse_juniper
+from .mikrotik import parse_mikrotik
 
-nos_parsers = {
+structured_parsers = {
     "juniper": {
         "bgp_route": parse_juniper,
         "bgp_aspath": parse_juniper,
         "bgp_community": parse_juniper,
-    }
+    },
+}
+
+scrape_parsers = {
+    "mikrotik_routeros": {
+        "bgp_route": parse_mikrotik,
+        "bgp_aspath": parse_mikrotik,
+        "bgp_community": parse_mikrotik,
+        "ping": parse_mikrotik,
+        "traceroute": parse_mikrotik,
+    },
+    "mikrotik_switchos": {
+        "bgp_route": parse_mikrotik,
+        "bgp_aspath": parse_mikrotik,
+        "bgp_community": parse_mikrotik,
+        "ping": parse_mikrotik,
+        "traceroute": parse_mikrotik,
+    },
 }