restructure location of models for cleaner importing/exporting

.flake8

@@ -7,19 +7,17 @@ exclude=.git, __pycache__, hyperglass/api/examples/*.py, hyperglass/compat/_ssht
 filename=*.py
 per-file-ignores=
     hyperglass/main.py:E402
-    # Disable redefinition warning for exception handlers
-    hyperglass/api.py:F811
     # Disable classmethod warning for validator decorators
-    hyperglass/models.py:N805,E0213,R0903
+    hyperglass/models/*.py:N805,E0213,R0903,E501,C0301
-    # Disable unused import warning for modules
+    hyperglass/models/api/*.py:N805,E0213,R0903,E501,C0301
-    hyperglass/*/__init__.py:F401
+    hyperglass/models/commands/*.py:N805,E0213,R0903,E501,C0301
-    hyperglass/api/models/*.py:N805,E0213,R0903
-    hyperglass/api/models/response.py:E501,C0301
     hyperglass/parsing/models/*.py:N805,E0213,R0903
     hyperglass/configuration/models/*.py:N805,E0213,R0903,E501,C0301
+    # Disable unused import warning for modules
+    hyperglass/*/__init__.py:F401
+    hyperglass/models/*/__init__.py:F401
 ignore=W503,C0330,R504,D202,S403,S301
 select=B, BLK, C, D, E, F, I, II, N, P, PIE, S, R, W
 disable-noqa=False
 hang-closing=False
 max-complexity=10
-# format=${cyan}%(path)s${reset}:${yellow_bold}%(row)d${reset}:${green_bold}%(col)d${reset}: ${red_bold}%(code)s${reset} %(text)s

hyperglass/api/__init__.py

@@ -36,7 +36,7 @@ from hyperglass.api.error_handlers import (
     default_handler,
     validation_handler,
 )
-from hyperglass.api.models.response import (
+from hyperglass.models.api.response import (
     QueryError,
     InfoResponse,
     QueryResponse,

hyperglass/api/models/__init__.py

@@ -1 +0,0 @@
-"""Query & Response Validation Models."""

hyperglass/api/routes.py

@@ -19,10 +19,9 @@ from hyperglass.external import Webhook, bgptools
 from hyperglass.api.tasks import process_headers, import_public_key
 from hyperglass.constants import __version__
 from hyperglass.exceptions import HyperglassError
+from hyperglass.models.api import Query, EncodedRequest
 from hyperglass.configuration import REDIS_CONFIG, params, devices
 from hyperglass.execution.main import execute
-from hyperglass.api.models.query import Query
-from hyperglass.api.models.cert_import import EncodedRequest
 
 APP_PATH = os.environ["hyperglass_directory"]
 

hyperglass/configuration/__init__.py

@@ -28,6 +28,7 @@ from hyperglass.constants import (
     __version__,
 )
 from hyperglass.exceptions import ConfigError, ConfigInvalid, ConfigMissing
+from hyperglass.models.commands import Commands
 from hyperglass.configuration.defaults import (
     CREDIT,
     DEFAULT_HELP,
@@ -37,7 +38,6 @@ from hyperglass.configuration.defaults import (
 from hyperglass.configuration.markdown import get_markdown
 from hyperglass.configuration.models.params import Params
 from hyperglass.configuration.models.devices import Devices
-from hyperglass.configuration.models.commands import Commands
 
 set_app_path(required=True)
 

hyperglass/configuration/models/docs.py

@@ -3,7 +3,8 @@
 from pydantic import Field, HttpUrl, StrictStr, StrictBool, constr
 
 # Project
-from hyperglass.models import AnyUri, HyperglassModel
+from hyperglass.models import HyperglassModel
+from hyperglass.models.fields import AnyUri
 
 DocsMode = constr(regex=r"(swagger|redoc)")
 

hyperglass/configuration/models/params.py

@@ -16,7 +16,8 @@ from pydantic import (
 )
 
 # Project
-from hyperglass.models import IntFloat, HyperglassModel
+from hyperglass.models import HyperglassModel
+from hyperglass.models.fields import IntFloat
 from hyperglass.configuration.models.web import Web
 from hyperglass.configuration.models.docs import Docs
 from hyperglass.configuration.models.cache import Cache

hyperglass/external/__init__.py

@@ -1,5 +1,4 @@
 """Functions & handlers for external data."""
 
-# Project
+from .ripestat import RIPEStat  # noqa: F401
-from hyperglass.external.ripestat import RIPEStat  # noqa: F401
+from .webhooks import Webhook  # noqa: F401
-from hyperglass.external.webhooks import Webhook  # noqa: F401

hyperglass/external/generic.py

@@ -2,8 +2,8 @@
 
 # Project
 from hyperglass.log import log
-from hyperglass.models import Webhook
 from hyperglass.external._base import BaseExternal
+from hyperglass.models.webhook import Webhook
 
 
 class GenericHook(BaseExternal, name="Generic"):

hyperglass/external/msteams.py

@@ -2,8 +2,8 @@
 
 # Project
 from hyperglass.log import log
-from hyperglass.models import Webhook
 from hyperglass.external._base import BaseExternal
+from hyperglass.models.webhook import Webhook
 
 
 class MSTeams(BaseExternal, name="MSTeams"):

hyperglass/external/slack.py

@@ -2,8 +2,8 @@
 
 # Project
 from hyperglass.log import log
-from hyperglass.models import Webhook
 from hyperglass.external._base import BaseExternal
+from hyperglass.models.webhook import Webhook
 
 
 class SlackHook(BaseExternal, name="Slack"):

hyperglass/models/__init__.py

@@ -0,0 +1,3 @@
+"""All Data Models used by hyperglass."""
+
+from .main import HyperglassModel, HyperglassModelExtra

hyperglass/models/api/__init__.py

@@ -0,0 +1,11 @@
+"""Query & Response Validation Models."""
+from .query import Query
+from .response import (
+    QueryError,
+    InfoResponse,
+    QueryResponse,
+    RoutersResponse,
+    CommunityResponse,
+    SupportedQueryResponse,
+)
+from .cert_import import EncodedRequest

hyperglass/models/api/cert_import.py

@@ -6,7 +6,7 @@ from typing import Union
 from pydantic import BaseModel, StrictStr
 
 # Project
-from hyperglass.models import StrictBytes
+from hyperglass.models.fields import StrictBytes
 
 
 class EncodedRequest(BaseModel):

hyperglass/models/api/query.py

@@ -12,8 +12,9 @@ from pydantic import BaseModel, StrictStr, constr, validator
 # Project
 from hyperglass.exceptions import InputInvalid
 from hyperglass.configuration import params, devices
-from hyperglass.api.models.types import SupportedQuery
+
-from hyperglass.api.models.validators import (
+from .types import SupportedQuery
+from .validators import (
     validate_ip,
     validate_aspath,
     validate_community_input,

hyperglass/models/commands/__init__.py

@@ -1,15 +1,16 @@
 """Validate command configuration variables."""
 
-# Project
+from .vyos import VyosCommands
-from hyperglass.models import HyperglassModelExtra
+from ..main import HyperglassModelExtra
-from hyperglass.configuration.models.commands.vyos import VyosCommands
+from .arista import AristaCommands
-from hyperglass.configuration.models.commands.arista import AristaCommands
+from .common import CommandGroup
-from hyperglass.configuration.models.commands.common import CommandGroup
+from .huawei import HuaweiCommands
-from hyperglass.configuration.models.commands.huawei import HuaweiCommands
+from .juniper import JuniperCommands
-from hyperglass.configuration.models.commands.juniper import JuniperCommands
+from .cisco_xr import CiscoXRCommands
-from hyperglass.configuration.models.commands.cisco_xr import CiscoXRCommands
+from .cisco_ios import CiscoIOSCommands
-from hyperglass.configuration.models.commands.cisco_ios import CiscoIOSCommands
+from .cisco_nxos import CiscoNXOSCommands
-from hyperglass.configuration.models.commands.cisco_nxos import CiscoNXOSCommands
+from .mikrotik_routeros import MikrotikRouterOS
+from .mikrotik_switchos import MikrotikSwitchOS
 
 _NOS_MAP = {
     "juniper": JuniperCommands,
@@ -18,6 +19,8 @@ _NOS_MAP = {
     "cisco_nxos": CiscoNXOSCommands,
     "arista": AristaCommands,
     "huawei": HuaweiCommands,
+    "mikrotik_routeros": MikrotikRouterOS,
+    "mikrotik_switchos": MikrotikSwitchOS,
     "vyos": VyosCommands,
 }
 
@@ -31,6 +34,8 @@ class Commands(HyperglassModelExtra):
     cisco_xr: CommandGroup = CiscoXRCommands()
     cisco_nxos: CommandGroup = CiscoNXOSCommands()
     huawei: CommandGroup = HuaweiCommands()
+    mikrotik_routeros: CommandGroup = MikrotikRouterOS()
+    mikortik_switchos: CommandGroup = MikrotikSwitchOS()
     vyos: CommandGroup = VyosCommands()
 
     @classmethod

hyperglass/models/commands/arista.py

@@ -3,8 +3,7 @@
 # Third Party
 from pydantic import StrictStr
 
-# Project
+from .common import CommandSet, CommandGroup
-from hyperglass.configuration.models.commands.common import CommandSet, CommandGroup
 
 
 class _IPv4(CommandSet):

hyperglass/models/commands/cisco_ios.py

@@ -3,8 +3,7 @@
 # Third Party
 from pydantic import StrictStr
 
-# Project
+from .common import CommandSet, CommandGroup
-from hyperglass.configuration.models.commands.common import CommandSet, CommandGroup
 
 
 class _IPv4(CommandSet):

hyperglass/models/commands/cisco_nxos.py

@@ -3,8 +3,7 @@
 # Third Party
 from pydantic import StrictStr
 
-# Project
+from .common import CommandSet, CommandGroup
-from hyperglass.configuration.models.commands.common import CommandSet, CommandGroup
 
 
 class _IPv4(CommandSet):

hyperglass/models/commands/cisco_xr.py

@@ -3,8 +3,7 @@
 # Third Party
 from pydantic import StrictStr
 
-# Project
+from .common import CommandSet, CommandGroup
-from hyperglass.configuration.models.commands.common import CommandSet, CommandGroup
 
 
 class _IPv4(CommandSet):

hyperglass/models/commands/common.py

@@ -3,8 +3,7 @@
 # Third Party
 from pydantic import StrictStr
 
-# Project
+from ..main import HyperglassModel, HyperglassModelExtra
-from hyperglass.models import HyperglassModel, HyperglassModelExtra
 
 
 class CommandSet(HyperglassModel):

hyperglass/models/commands/huawei.py

@@ -3,8 +3,7 @@
 # Third Party
 from pydantic import StrictStr
 
-# Project
+from .common import CommandSet, CommandGroup
-from hyperglass.configuration.models.commands.common import CommandSet, CommandGroup
 
 
 class _IPv4(CommandSet):

hyperglass/models/commands/juniper.py

@@ -3,8 +3,7 @@
 # Third Party
 from pydantic import StrictStr
 
-# Project
+from .common import CommandSet, CommandGroup
-from hyperglass.configuration.models.commands.common import CommandSet, CommandGroup
 
 
 class _IPv4(CommandSet):

hyperglass/models/commands/vyos.py

@@ -3,8 +3,7 @@
 # Third Party
 from pydantic import StrictStr
 
-# Project
+from .common import CommandSet, CommandGroup
-from hyperglass.configuration.models.commands.common import CommandSet, CommandGroup
 
 
 class _IPv4(CommandSet):

hyperglass/models/fields.py

@@ -0,0 +1,79 @@
+"""Custom Pydantic Fields/Types."""
+
+# Standard Library
+import re
+from typing import TypeVar
+
+# Third Party
+from pydantic import StrictInt, StrictFloat
+
+IntFloat = TypeVar("IntFloat", StrictInt, StrictFloat)
+
+
+class StrictBytes(bytes):
+    """Custom data type for a strict byte string.
+
+    Used for validating the encoded JWT request payload.
+    """
+
+    @classmethod
+    def __get_validators__(cls):
+        """Yield Pydantic validator function.
+
+        See: https://pydantic-docs.helpmanual.io/usage/types/#custom-data-types
+
+        Yields:
+            {function} -- Validator
+        """
+        yield cls.validate
+
+    @classmethod
+    def validate(cls, value):
+        """Validate type.
+
+        Arguments:
+            value {Any} -- Pre-validated input
+
+        Raises:
+            TypeError: Raised if value is not bytes
+
+        Returns:
+            {object} -- Instantiated class
+        """
+        if not isinstance(value, bytes):
+            raise TypeError("bytes required")
+        return cls()
+
+    def __repr__(self):
+        """Return representation of object.
+
+        Returns:
+            {str} -- Representation
+        """
+        return f"StrictBytes({super().__repr__()})"
+
+
+class AnyUri(str):
+    """Custom field type for HTTP URI, e.g. /example."""
+
+    @classmethod
+    def __get_validators__(cls):
+        """Pydantic custim field method."""
+        yield cls.validate
+
+    @classmethod
+    def validate(cls, value):
+        """Ensure URI string contains a leading forward-slash."""
+        uri_regex = re.compile(r"^(\/.*)$")
+        if not isinstance(value, str):
+            raise TypeError("AnyUri type must be a string")
+        match = uri_regex.fullmatch(value)
+        if not match:
+            raise ValueError(
+                "Invalid format. A URI must begin with a forward slash, e.g. '/example'"
+            )
+        return cls(match.group())
+
+    def __repr__(self):
+        """Stringify custom field representation."""
+        return f"AnyUri({super().__repr__()})"

hyperglass/models/main.py

@@ -0,0 +1,99 @@
+"""Data models used throughout hyperglass."""
+
+# Standard Library
+import re
+
+# Third Party
+from pydantic import HttpUrl, BaseModel
+
+_WEBHOOK_TITLE = "hyperglass received a valid query with the following data"
+_ICON_URL = "https://res.cloudinary.com/hyperglass/image/upload/v1593192484/icon.png"
+
+
+def clean_name(_name: str) -> str:
+    """Remove unsupported characters from field names.
+
+    Converts any "desirable" seperators to underscore, then removes all
+    characters that are unsupported in Python class variable names.
+    Also removes leading numbers underscores.
+    """
+    _replaced = re.sub(r"[\-|\.|\@|\~|\:\/|\s]", "_", _name)
+    _scrubbed = "".join(re.findall(r"([a-zA-Z]\w+|\_+)", _replaced))
+    return _scrubbed.lower()
+
+
+class HyperglassModel(BaseModel):
+    """Base model for all hyperglass configuration models."""
+
+    class Config:
+        """Default Pydantic configuration.
+
+        See https://pydantic-docs.helpmanual.io/usage/model_config
+        """
+
+        validate_all = True
+        extra = "forbid"
+        validate_assignment = True
+        alias_generator = clean_name
+        json_encoders = {HttpUrl: lambda v: str(v)}
+
+    def export_json(self, *args, **kwargs):
+        """Return instance as JSON.
+
+        Returns:
+            {str} -- Stringified JSON.
+        """
+
+        export_kwargs = {
+            "by_alias": True,
+            "exclude_unset": False,
+            **kwargs,
+        }
+
+        return self.json(*args, **export_kwargs)
+
+    def export_dict(self, *args, **kwargs):
+        """Return instance as dictionary.
+
+        Returns:
+            {dict} -- Python dictionary.
+        """
+        export_kwargs = {
+            "by_alias": True,
+            "exclude_unset": False,
+            **kwargs,
+        }
+
+        return self.dict(*args, **export_kwargs)
+
+    def export_yaml(self, *args, **kwargs):
+        """Return instance as YAML.
+
+        Returns:
+            {str} -- Stringified YAML.
+        """
+        # Standard Library
+        import json
+
+        # Third Party
+        import yaml
+
+        export_kwargs = {
+            "by_alias": kwargs.pop("by_alias", True),
+            "exclude_unset": kwargs.pop("by_alias", False),
+        }
+
+        return yaml.safe_dump(
+            json.loads(self.export_json(**export_kwargs)), *args, **kwargs
+        )
+
+
+class HyperglassModelExtra(HyperglassModel):
+    """Model for hyperglass configuration models with dynamic fields."""
+
+    pass
+
+    class Config:
+        """Default pydantic configuration."""
+
+        extra = "allow"

hyperglass/models/webhook.py

@@ -1,187 +1,21 @@
 """Data models used throughout hyperglass."""
 
 # Standard Library
-import re
+from typing import Optional
-from typing import TypeVar, Optional
 from datetime import datetime
 
 # Third Party
-from pydantic import (
+from pydantic import StrictStr, root_validator
-    HttpUrl,
-    BaseModel,
-    StrictInt,
-    StrictStr,
-    StrictFloat,
-    root_validator,
-)
 
 # Project
 from hyperglass.log import log
 
-IntFloat = TypeVar("IntFloat", StrictInt, StrictFloat)
+from .main import HyperglassModel, HyperglassModelExtra
 
 _WEBHOOK_TITLE = "hyperglass received a valid query with the following data"
 _ICON_URL = "https://res.cloudinary.com/hyperglass/image/upload/v1593192484/icon.png"
 
 
-def clean_name(_name: str) -> str:
-    """Remove unsupported characters from field names.
-
-    Converts any "desirable" seperators to underscore, then removes all
-    characters that are unsupported in Python class variable names.
-    Also removes leading numbers underscores.
-    """
-    _replaced = re.sub(r"[\-|\.|\@|\~|\:\/|\s]", "_", _name)
-    _scrubbed = "".join(re.findall(r"([a-zA-Z]\w+|\_+)", _replaced))
-    return _scrubbed.lower()
-
-
-class HyperglassModel(BaseModel):
-    """Base model for all hyperglass configuration models."""
-
-    class Config:
-        """Default Pydantic configuration.
-
-        See https://pydantic-docs.helpmanual.io/usage/model_config
-        """
-
-        validate_all = True
-        extra = "forbid"
-        validate_assignment = True
-        alias_generator = clean_name
-        json_encoders = {HttpUrl: lambda v: str(v)}
-
-    def export_json(self, *args, **kwargs):
-        """Return instance as JSON.
-
-        Returns:
-            {str} -- Stringified JSON.
-        """
-
-        export_kwargs = {
-            "by_alias": True,
-            "exclude_unset": False,
-            **kwargs,
-        }
-
-        return self.json(*args, **export_kwargs)
-
-    def export_dict(self, *args, **kwargs):
-        """Return instance as dictionary.
-
-        Returns:
-            {dict} -- Python dictionary.
-        """
-        export_kwargs = {
-            "by_alias": True,
-            "exclude_unset": False,
-            **kwargs,
-        }
-
-        return self.dict(*args, **export_kwargs)
-
-    def export_yaml(self, *args, **kwargs):
-        """Return instance as YAML.
-
-        Returns:
-            {str} -- Stringified YAML.
-        """
-        # Standard Library
-        import json
-
-        # Third Party
-        import yaml
-
-        export_kwargs = {
-            "by_alias": kwargs.pop("by_alias", True),
-            "exclude_unset": kwargs.pop("by_alias", False),
-        }
-
-        return yaml.safe_dump(
-            json.loads(self.export_json(**export_kwargs)), *args, **kwargs
-        )
-
-
-class HyperglassModelExtra(HyperglassModel):
-    """Model for hyperglass configuration models with dynamic fields."""
-
-    pass
-
-    class Config:
-        """Default pydantic configuration."""
-
-        extra = "allow"
-
-
-class AnyUri(str):
-    """Custom field type for HTTP URI, e.g. /example."""
-
-    @classmethod
-    def __get_validators__(cls):
-        """Pydantic custim field method."""
-        yield cls.validate
-
-    @classmethod
-    def validate(cls, value):
-        """Ensure URI string contains a leading forward-slash."""
-        uri_regex = re.compile(r"^(\/.*)$")
-        if not isinstance(value, str):
-            raise TypeError("AnyUri type must be a string")
-        match = uri_regex.fullmatch(value)
-        if not match:
-            raise ValueError(
-                "Invalid format. A URI must begin with a forward slash, e.g. '/example'"
-            )
-        return cls(match.group())
-
-    def __repr__(self):
-        """Stringify custom field representation."""
-        return f"AnyUri({super().__repr__()})"
-
-
-class StrictBytes(bytes):
-    """Custom data type for a strict byte string.
-
-    Used for validating the encoded JWT request payload.
-    """
-
-    @classmethod
-    def __get_validators__(cls):
-        """Yield Pydantic validator function.
-
-        See: https://pydantic-docs.helpmanual.io/usage/types/#custom-data-types
-
-        Yields:
-            {function} -- Validator
-        """
-        yield cls.validate
-
-    @classmethod
-    def validate(cls, value):
-        """Validate type.
-
-        Arguments:
-            value {Any} -- Pre-validated input
-
-        Raises:
-            TypeError: Raised if value is not bytes
-
-        Returns:
-            {object} -- Instantiated class
-        """
-        if not isinstance(value, bytes):
-            raise TypeError("bytes required")
-        return cls()
-
-    def __repr__(self):
-        """Return representation of object.
-
-        Returns:
-            {str} -- Representation
-        """
-        return f"StrictBytes({super().__repr__()})"
-
-
 class WebhookHeaders(HyperglassModel):
     """Webhook data model."""