Improve external http client typing and add tests

hyperglass/external/_base.py

@@ -1,4 +1,4 @@
-"""Session handler for RIPEStat Data API."""
+"""Session handler for external http data sources."""
 
 # Standard Library
 import re
@@ -6,7 +6,6 @@ import json as _json
 import socket
 import typing as t
 from json import JSONDecodeError
-from types import TracebackType
 from socket import gaierror
 
 # Third Party
@@ -16,10 +15,21 @@ import httpx
 from hyperglass.log import log
 from hyperglass.util import make_repr, parse_exception
 from hyperglass.constants import __version__
+from hyperglass.models.fields import JsonValue, HttpMethod, Primitives
 from hyperglass.exceptions.private import ExternalError
 
+if t.TYPE_CHECKING:
+    # Standard Library
+    from types import TracebackType
 
-def _prepare_dict(_dict):
+    # Project
+    from hyperglass.exceptions._common import ErrorLevel
+    from hyperglass.models.config.logging import Http
+
+D = t.TypeVar("D", bound=t.Dict)
+
+
+def _prepare_dict(_dict: D) -> D:
     return _json.loads(_json.dumps(_dict, default=str))
 
 
@@ -28,16 +38,17 @@ class BaseExternal:
 
     def __init__(
         self,
-        base_url,
-        config=None,
-        uri_prefix="",
-        uri_suffix="",
-        verify_ssl=True,
-        timeout=10,
-        parse=True,
-    ):
+        base_url: str,
+        config: t.Optional["Http"] = None,
+        uri_prefix: str = "",
+        uri_suffix: str = "",
+        verify_ssl: bool = True,
+        timeout: int = 10,
+        parse: bool = True,
+    ) -> None:
         """Initialize connection instance."""
         self.__name__ = getattr(self, "name", "BaseExternal")
+        self.name = self.__name__
         self.config = config
         self.base_url = base_url.strip("/")
         self.uri_prefix = uri_prefix.strip("/")
@@ -55,61 +66,80 @@ class BaseExternal:
         self._asession = httpx.AsyncClient(**session_args)
 
     @classmethod
-    def __init_subclass__(cls, name=None, **kwargs):
+    def __init_subclass__(
+        cls: "BaseExternal", name: t.Optional[str] = None, **kwargs: t.Any
+    ) -> None:
         """Set correct subclass name."""
         super().__init_subclass__(**kwargs)
         cls.name = name or cls.__name__
 
-    async def __aenter__(self):
+    async def __aenter__(self: "BaseExternal") -> "BaseExternal":
         """Test connection on entry."""
         available = await self._atest()
 
         if available:
             log.debug("Initialized session with {}", self.base_url)
             return self
-        else:
-            raise self._exception(f"Unable to create session to {self.name}")
+        raise self._exception(f"Unable to create session to {self.name}")
 
-    async def __aexit__(self, exc_type=None, exc_value=None, traceback=None):
+    async def __aexit__(
+        self: "BaseExternal",
+        exc_type: t.Optional[t.Type[BaseException]] = None,
+        exc_value: t.Optional[BaseException] = None,
+        traceback: t.Optional["TracebackType"] = None,
+    ) -> True:
         """Close connection on exit."""
         log.debug("Closing session with {}", self.base_url)
 
+        if exc_type is not None:
+            log.error(str(exc_value))
+
         await self._asession.aclose()
+        if exc_value is not None:
+            raise exc_value
         return True
 
-    def __enter__(self):
+    def __enter__(self: "BaseExternal") -> "BaseExternal":
         """Test connection on entry."""
         available = self._test()
 
         if available:
             log.debug("Initialized session with {}", self.base_url)
             return self
-        else:
-            raise self._exception(f"Unable to create session to {self.name}")
+        raise self._exception(f"Unable to create session to {self.name}")
 
     def __exit__(
-        self,
+        self: "BaseExternal",
         exc_type: t.Optional[t.Type[BaseException]] = None,
         exc_value: t.Optional[BaseException] = None,
-        exc_traceback: t.Optional[TracebackType] = None,
-    ):
+        exc_traceback: t.Optional["TracebackType"] = None,
+    ) -> bool:
         """Close connection on exit."""
         if exc_type is not None:
             log.error(str(exc_value))
         self._session.close()
+        if exc_value is not None:
+            raise exc_value
+        return True
 
-    def __repr__(self):
+    def __repr__(self: "BaseExternal") -> str:
         """Return user friendly representation of instance."""
         return make_repr(self)
 
-    def _exception(self, message, exc=None, level="warning", **kwargs):
+    def _exception(
+        self: "BaseExternal",
+        message: str,
+        exc: t.Optional[BaseException] = None,
+        level: "ErrorLevel" = "warning",
+        **kwargs: t.Any,
+    ) -> ExternalError:
         """Add stringified exception to message if passed."""
         if exc is not None:
-            message = f"{str(message)}: {str(exc)}"
+            message = f"{message!s}: {exc!s}"
 
         return ExternalError(message=message, level=level, **kwargs)
 
-    def _parse_response(self, response):
+    def _parse_response(self: "BaseExternal", response: httpx.Response) -> t.Any:
         if self.parse:
             parsed = {}
             try:
@@ -124,7 +154,7 @@ class BaseExternal:
             parsed = response
         return parsed
 
-    def _test(self):
+    def _test(self: "BaseExternal") -> bool:
         """Open a low-level connection to the base URL to ensure its port is open."""
         log.debug("Testing connection to {}", self.base_url)
 
@@ -146,15 +176,17 @@ class BaseExternal:
 
         except gaierror as err:
             # Raised if the target isn't listening on the port
-            raise self._exception(f"{self.name} appears to be unreachable", err) from None
+            raise self._exception(
+                f"{self.name!r} appears to be unreachable at {self.base_url!r}", err
+            ) from None
 
         return True
 
-    async def _atest(self):
+    async def _atest(self: "BaseExternal") -> bool:
         """Open a low-level connection to the base URL to ensure its port is open."""
         return self._test()
 
-    def _build_request(self, **kwargs):
+    def _build_request(self: "BaseExternal", **kwargs: t.Any) -> t.Dict[str, t.Any]:
         """Process requests parameters into structure usable by http library."""
         # Standard Library
         from operator import itemgetter
@@ -212,16 +244,16 @@ class BaseExternal:
         return request
 
     async def _arequest(  # noqa: C901
-        self,
-        method,
-        endpoint,
-        item=None,
-        headers=None,
-        params=None,
-        data=None,
-        timeout=None,
-        response_required=False,
-    ):
+        self: "BaseExternal",
+        method: HttpMethod,
+        endpoint: str,
+        item: t.Union[str, int, None] = None,
+        headers: t.Dict[str, str] = None,
+        params: t.Dict[str, JsonValue[Primitives]] = None,
+        data: t.Optional[t.Any] = None,
+        timeout: t.Optional[int] = None,
+        response_required: bool = False,
+    ) -> t.Any:
         """Run HTTP POST operation."""
         request = self._build_request(
             method=method,
@@ -249,35 +281,35 @@ class BaseExternal:
 
         return self._parse_response(response)
 
-    async def _aget(self, endpoint, **kwargs):
+    async def _aget(self: "BaseExternal", endpoint: str, **kwargs: t.Any) -> t.Any:
         return await self._arequest(method="GET", endpoint=endpoint, **kwargs)
 
-    async def _apost(self, endpoint, **kwargs):
+    async def _apost(self: "BaseExternal", endpoint: str, **kwargs: t.Any) -> t.Any:
         return await self._arequest(method="POST", endpoint=endpoint, **kwargs)
 
-    async def _aput(self, endpoint, **kwargs):
+    async def _aput(self: "BaseExternal", endpoint: str, **kwargs: t.Any) -> t.Any:
         return await self._arequest(method="PUT", endpoint=endpoint, **kwargs)
 
-    async def _adelete(self, endpoint, **kwargs):
+    async def _adelete(self: "BaseExternal", endpoint: str, **kwargs: t.Any) -> t.Any:
         return await self._arequest(method="DELETE", endpoint=endpoint, **kwargs)
 
-    async def _apatch(self, endpoint, **kwargs):
+    async def _apatch(self: "BaseExternal", endpoint: str, **kwargs: t.Any) -> t.Any:
         return await self._arequest(method="PATCH", endpoint=endpoint, **kwargs)
 
-    async def _ahead(self, endpoint, **kwargs):
+    async def _ahead(self: "BaseExternal", endpoint: str, **kwargs: t.Any) -> t.Any:
         return await self._arequest(method="HEAD", endpoint=endpoint, **kwargs)
 
     def _request(  # noqa: C901
-        self,
-        method,
-        endpoint,
-        item=None,
-        headers=None,
-        params=None,
-        data=None,
-        timeout=None,
-        response_required=False,
-    ):
+        self: "BaseExternal",
+        method: HttpMethod,
+        endpoint: str,
+        item: t.Union[str, int, None] = None,
+        headers: t.Dict[str, str] = None,
+        params: t.Dict[str, JsonValue[Primitives]] = None,
+        data: t.Optional[t.Any] = None,
+        timeout: t.Optional[int] = None,
+        response_required: bool = False,
+    ) -> t.Any:
         """Run HTTP POST operation."""
         request = self._build_request(
             method=method,
@@ -305,20 +337,20 @@ class BaseExternal:
 
         return self._parse_response(response)
 
-    def _get(self, endpoint, **kwargs):
+    def _get(self: "BaseExternal", endpoint: str, **kwargs: t.Any) -> t.Any:
         return self._request(method="GET", endpoint=endpoint, **kwargs)
 
-    def _post(self, endpoint, **kwargs):
+    def _post(self: "BaseExternal", endpoint: str, **kwargs: t.Any) -> t.Any:
         return self._request(method="POST", endpoint=endpoint, **kwargs)
 
-    def _put(self, endpoint, **kwargs):
+    def _put(self: "BaseExternal", endpoint: str, **kwargs: t.Any) -> t.Any:
         return self._request(method="PUT", endpoint=endpoint, **kwargs)
 
-    def _delete(self, endpoint, **kwargs):
+    def _delete(self: "BaseExternal", endpoint: str, **kwargs: t.Any) -> t.Any:
         return self._request(method="DELETE", endpoint=endpoint, **kwargs)
 
-    def _patch(self, endpoint, **kwargs):
+    def _patch(self: "BaseExternal", endpoint: str, **kwargs: t.Any) -> t.Any:
         return self._request(method="PATCH", endpoint=endpoint, **kwargs)
 
-    def _head(self, endpoint, **kwargs):
+    def _head(self: "BaseExternal", endpoint: str, **kwargs: t.Any) -> t.Any:
         return self._request(method="HEAD", endpoint=endpoint, **kwargs)

hyperglass/external/generic.py

@@ -1,20 +1,29 @@
 """Session handler for Generic HTTP API endpoint."""
 
+# Standard Library
+import typing as t
+
 # Project
 from hyperglass.log import log
-from hyperglass.external._base import BaseExternal
 from hyperglass.models.webhook import Webhook
 
+# Local
+from ._base import BaseExternal
+
+if t.TYPE_CHECKING:
+    # Project
+    from hyperglass.models.config.logging import Http
+
 
 class GenericHook(BaseExternal, name="Generic"):
     """Slack session handler."""
 
-    def __init__(self, config):
+    def __init__(self: "GenericHook", config: "Http") -> None:
         """Initialize external base class with http connection details."""
 
         super().__init__(base_url=f"{config.host.scheme}://{config.host.host}", config=config)
 
-    async def send(self, query):
+    async def send(self: "GenericHook", query: t.Dict[str, t.Any]):
         """Send an incoming webhook to http endpoint."""
 
         payload = Webhook(**query)

hyperglass/external/msteams.py

@@ -1,20 +1,25 @@
 """Session handler for Microsoft Teams API."""
 
+import typing as t
+
 # Project
 from hyperglass.log import log
 from hyperglass.external._base import BaseExternal
 from hyperglass.models.webhook import Webhook
 
+if t.TYPE_CHECKING:
+    from hyperglass.models.config.logging import Http
+
 
 class MSTeams(BaseExternal, name="MSTeams"):
     """Microsoft Teams session handler."""
 
-    def __init__(self, config):
+    def __init__(self: "MSTeams", config: "Http") -> None:
         """Initialize external base class with Microsoft Teams connection details."""
 
         super().__init__(base_url="https://outlook.office.com", config=config, parse=False)
 
-    async def send(self, query):
+    async def send(self: "MSTeams", query: t.Dict[str, t.Any]):
         """Send an incoming webhook to Microsoft Teams."""
 
         payload = Webhook(**query)

hyperglass/external/slack.py

@@ -1,20 +1,27 @@
 """Session handler for Slack API."""
 
+# Standard Library
+import typing as t
+
 # Project
 from hyperglass.log import log
 from hyperglass.external._base import BaseExternal
 from hyperglass.models.webhook import Webhook
 
+if t.TYPE_CHECKING:
+    # Project
+    from hyperglass.models.config.logging import Http
+
 
 class SlackHook(BaseExternal, name="Slack"):
     """Slack session handler."""
 
-    def __init__(self, config):
+    def __init__(self: "SlackHook", config: "Http") -> None:
         """Initialize external base class with Slack connection details."""
 
         super().__init__(base_url="https://hooks.slack.com", config=config, parse=False)
 
-    async def send(self, query):
+    async def send(self: "SlackHook", query: t.Dict[str, t.Any]):
         """Send an incoming webhook to Slack."""
 
         payload = Webhook(**query)

hyperglass/external/tests/test_base.py

@@ -0,0 +1,49 @@
+"""Test external http client."""
+# Standard Library
+import asyncio
+
+# Third Party
+import pytest
+
+# Project
+from hyperglass.exceptions.private import ExternalError
+from hyperglass.models.config.logging import Http
+
+# Local
+from .._base import BaseExternal
+
+config = Http(provider="generic", host="https://httpbin.org")
+
+
+def test_base_external_sync():
+    with BaseExternal(base_url="https://httpbin.org", config=config) as client:
+        res1 = client._get("/get")
+        res2 = client._get("/get", params={"key": "value"})
+        res3 = client._post("/post", data={"strkey": "value", "intkey": 1})
+    assert res1["url"] == "https://httpbin.org/get"
+    assert res2["args"].get("key") == "value"
+    assert res3["json"].get("strkey") == "value"
+    assert res3["json"].get("intkey") == 1
+
+    with pytest.raises(ExternalError):
+        with BaseExternal(base_url="https://httpbin.org", config=config, timeout=2) as client:
+            client._get("/delay/4")
+
+
+async def _run_test_base_external_async():
+    async with BaseExternal(base_url="https://httpbin.org", config=config) as client:
+        res1 = await client._aget("/get")
+        res2 = await client._aget("/get", params={"key": "value"})
+        res3 = await client._apost("/post", data={"strkey": "value", "intkey": 1})
+    assert res1["url"] == "https://httpbin.org/get"
+    assert res2["args"].get("key") == "value"
+    assert res3["json"].get("strkey") == "value"
+    assert res3["json"].get("intkey") == 1
+
+    with pytest.raises(ExternalError):
+        async with BaseExternal(base_url="https://httpbin.org", config=config, timeout=2) as client:
+            await client._get("/delay/4")
+
+
+def test_base_external_async():
+    asyncio.run(_run_test_base_external_async())

hyperglass/external/webhooks.py

@@ -1,12 +1,21 @@
 """Convenience functions for webhooks."""
 
+# Standard Library
+import typing as t
+
 # Project
-from hyperglass.external._base import BaseExternal
-from hyperglass.external.slack import SlackHook
-from hyperglass.external.generic import GenericHook
-from hyperglass.external.msteams import MSTeams
 from hyperglass.exceptions.private import UnsupportedError
 
+# Local
+from ._base import BaseExternal
+from .slack import SlackHook
+from .generic import GenericHook
+from .msteams import MSTeams
+
+if t.TYPE_CHECKING:
+    # Project
+    from hyperglass.models.config.logging import Http
+
 PROVIDER_MAP = {
     "generic": GenericHook,
     "msteams": MSTeams,
@@ -17,7 +26,7 @@ PROVIDER_MAP = {
 class Webhook(BaseExternal):
     """Get webhook for provider name."""
 
-    def __new__(cls, config):
+    def __new__(cls: "Webhook", config: "Http") -> "BaseExternal":
         """Return instance for correct provider handler."""
         try:
             provider_class = PROVIDER_MAP[config.provider]

hyperglass/models/fields.py

@@ -8,12 +8,14 @@ import typing as t
 from pydantic import StrictInt, StrictFloat
 
 IntFloat = t.TypeVar("IntFloat", StrictInt, StrictFloat)
+J = t.TypeVar("J")
 
 SupportedDriver = t.Literal["netmiko", "hyperglass_agent"]
 HttpAuthMode = t.Literal["basic", "api_key"]
 HttpProvider = t.Literal["msteams", "slack", "generic"]
 LogFormat = t.Literal["text", "json"]
 Primitives = t.Union[None, float, int, bool, str]
+JsonValue = t.Union[J, t.Sequence[J], t.Dict[str, J]]
 
 
 class AnyUri(str):