1
0
mirror of https://github.com/checktheroads/hyperglass synced 2024-05-11 05:55:08 +00:00

commit previous changes and dep updates

This commit is contained in:
thatmattlove
2022-12-11 17:27:53 -05:00
parent 9300105d9f
commit 60429ebbc1
22 changed files with 1032 additions and 832 deletions

View File

@@ -18,13 +18,15 @@ per-file-ignores=
hyperglass/*/__init__.py:F401
hyperglass/models/*/__init__.py:F401
# Disable assertion and docstring checks on tests.
hyperglass/**/test_*.py:S101,D103
hyperglass/**/test_*.py:S101,D103,D100,D104
hyperglass/**/tests/*.py:S101,D103,D100,D104
hyperglass/**/tests/__init__.py:D103,D100,D104
hyperglass/state/hooks.py:F811
# Ignore whitespace in docstrings
hyperglass/cli/static.py:W293
# Ignore docstring standards
hyperglass/cli/main.py:D400,D403
ignore=W503,C0330,R504,D202,S403,S301,S404,E731,D402,IF100,B008
ignore=W503,R504,D202,S403,S301,S404,E731,D402,IF100,B008
select=B, BLK, C, D, E, F, I, II, N, P, PIE, S, R, W
disable-noqa=False
hang-closing=False

7
.gitignore vendored
View File

@@ -13,10 +13,17 @@ old_*.py
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
.ipynb*
# Pyenv
.python-version
# MyPy
.mypy_cache
# Pytest
.pytest_cache
# C extensions
*.so

View File

@@ -1,13 +1,14 @@
"""API Error Handlers."""
# Third Party
from fastapi import Request
from starlette.responses import JSONResponse
# Project
from hyperglass.state import use_state
async def default_handler(request, exc):
async def default_handler(request: Request, exc: BaseException) -> JSONResponse:
"""Handle uncaught errors."""
state = use_state()
return JSONResponse(
@@ -16,7 +17,7 @@ async def default_handler(request, exc):
)
async def http_handler(request, exc):
async def http_handler(request: Request, exc: BaseException) -> JSONResponse:
"""Handle web server errors."""
return JSONResponse(
@@ -25,7 +26,7 @@ async def http_handler(request, exc):
)
async def app_handler(request, exc):
async def app_handler(request: Request, exc: BaseException) -> JSONResponse:
"""Handle application errors."""
return JSONResponse(
{"output": exc.message, "level": exc.level, "keywords": exc.keywords},
@@ -33,7 +34,7 @@ async def app_handler(request, exc):
)
async def validation_handler(request, exc):
async def validation_handler(request: Request, exc: BaseException) -> JSONResponse:
"""Handle Pydantic validation errors raised by FastAPI."""
error = exc.errors()[0]
return JSONResponse(

View File

@@ -6,8 +6,7 @@ import typing as t
from datetime import datetime
# Third Party
from fastapi import Depends, HTTPException, BackgroundTasks
from starlette.requests import Request
from fastapi import Depends, Request, HTTPException, BackgroundTasks
from fastapi.openapi.docs import get_redoc_html, get_swagger_ui_html
# Project

View File

@@ -1,4 +1,6 @@
"""hyperglass Configuration."""
# Standard Library
import typing as t
# Project
from hyperglass.state import use_state
@@ -7,26 +9,36 @@ from hyperglass.defaults.directives import init_builtin_directives
# Local
from .validate import init_files, init_params, init_devices, init_ui_params, init_directives
if t.TYPE_CHECKING:
# Project
from hyperglass.models.directive import Directives
from hyperglass.models.config.params import Params
from hyperglass.models.config.devices import Devices
__all__ = ("init_user_config",)
def init_user_config() -> None:
def init_user_config(
params: t.Optional["Params"] = None,
directives: t.Optional["Directives"] = None,
devices: t.Optional["Devices"] = None,
) -> None:
"""Initialize all user configurations and add them to global state."""
state = use_state()
init_files()
params = init_params()
_params = params or init_params()
builtins = init_builtin_directives()
custom = init_directives()
directives = builtins + custom
_custom = directives or init_directives()
_directives = builtins + _custom
with state.cache.pipeline() as pipeline:
# Write params and directives to the cache first to avoid a race condition where ui_params
# or devices try to access params or directives before they're available.
pipeline.set("params", params)
pipeline.set("directives", directives)
pipeline.set("params", _params)
pipeline.set("directives", _directives)
devices = init_devices()
ui_params = init_ui_params(params=params, devices=devices)
_devices = devices or init_devices()
ui_params = init_ui_params(params=_params, devices=_devices)
with state.cache.pipeline() as pipeline:
pipeline.set("devices", devices)
pipeline.set("devices", _devices)
pipeline.set("ui_params", ui_params)

View File

@@ -19,10 +19,12 @@ from hyperglass.exceptions.private import ConfigError
if t.TYPE_CHECKING:
# Project
from hyperglass.models.api.query import Query
from hyperglass.models.api.query import Query, QueryTarget
from hyperglass.models.directive import Directive
from hyperglass.models.config.devices import Device
FormatterCallback = t.Callable[[str], "QueryTarget"]
class Construct:
"""Construct SSH commands/REST API parameters from validated query data."""
@@ -54,8 +56,21 @@ class Construct:
if self.device.platform in TARGET_FORMAT_SPACE:
self.target = re.sub(r"\/", r" ", str(self.query.query_target))
with Formatter(self.device.platform, self.query.query_type) as formatter:
self.target = formatter(self.query.query_target)
with Formatter(self.query) as formatter:
self.target = formatter(self.prepare_target())
def prepare_target(self) -> "QueryTarget":
"""Format the query target based on directive parameters."""
if isinstance(self.query.query_target, t.List):
# Directive can accept multiple values in a single command.
if self.directive.multiple:
return self.directive.multiple_separator.join(self.query.query_target)
# Target is an array of one, return single item.
if len(self.query.query_target) == 1:
return self.query.query_target[0]
# Directive commands should be run once for each item in the target.
return self.query.query_target
def json(self, afi):
"""Return JSON version of validated query for REST devices."""
@@ -107,10 +122,11 @@ class Construct:
class Formatter:
"""Modify query target based on the device's NOS requirements and the query type."""
def __init__(self, platform: str, query_type: str) -> None:
def __init__(self, query: "Query") -> None:
"""Initialize target formatting."""
self.platform = platform
self.query_type = query_type
self.query = query
self.platform = query.device.platform
self.query_type = query.query_type
def __enter__(self):
"""Get the relevant formatter."""
@@ -125,18 +141,25 @@ class Formatter:
def _get_formatter(self):
if self.platform in ("juniper", "juniper_junos"):
if self.query_type == "bgp_aspath":
return self._juniper_bgp_aspath
return self._with_formatter(self._juniper_bgp_aspath)
if self.platform in ("bird", "bird_ssh"):
if self.query_type == "bgp_aspath":
return self._bird_bgp_aspath
return self._with_formatter(self._bird_bgp_aspath)
elif self.query_type == "bgp_community":
return self._bird_bgp_community
return self._default
return self._with_formatter(self._bird_bgp_community)
return self._with_formatter(self._default)
def _default(self, target: str) -> str:
"""Don't format targets by default."""
return target
def _with_formatter(self, formatter: t.Callable[[str], str]) -> FormatterCallback:
result: FormatterCallback
if isinstance(self.query.query_target, t.List):
result = lambda s: [formatter(i) for i in s]
result = lambda s: formatter(s)
return result
def _juniper_bgp_aspath(self, target: str) -> str:
"""Convert from Cisco AS_PATH format to Juniper format."""
query = str(target)

View File

View File

@@ -0,0 +1,33 @@
# Project
from hyperglass.models.api import Query
from hyperglass.configuration import init_user_config
from hyperglass.models.directive import Directives
from hyperglass.models.config.devices import Devices
# Local
from .._construct import Construct
def test_construct():
devices = Devices(
{
"name": "test1",
"address": "127.0.0.1",
"credential": {"username": "", "password": ""},
"platform": "juniper",
"attrs": {"source4": "192.0.2.1", "source6": "2001:db8::1"},
"directives": ["juniper_bgp_route"],
}
)
directives = Directives(
{"juniper_bgp_route": {"name": "BGP Route", "plugins": [], "rules": [], "groups": []}}
)
init_user_config(devices=devices, directives=directives)
query = Query(
queryLocation="test1",
queryTarget="192.0.2.0/24",
queryType="juniper_bgp_route",
)
constructor = Construct(device=devices["test1"], query=query)
assert constructor.target == "192.0.2.0/24"

View File

@@ -7,7 +7,7 @@ import secrets
from datetime import datetime
# Third Party
from pydantic import BaseModel, StrictStr, constr, validator
from pydantic import BaseModel, constr, validator
# Project
from hyperglass.log import log
@@ -22,56 +22,46 @@ from ..config.devices import Device
(TEXT := use_state("params").web.text)
QueryLocation = constr(strip_whitespace=True, strict=True, min_length=1)
QueryTarget = constr(strip_whitespace=True, min_length=1)
QueryType = constr(strip_whitespace=True, strict=True, min_length=1)
class Query(BaseModel):
"""Validation model for input query parameters."""
query_location: StrictStr # Device `name` field
query_type: StrictStr # Directive `id` field
query_location: QueryLocation # Device `name` field
query_target: t.Union[t.List[QueryTarget], QueryTarget]
query_type: QueryType # Directive `id` field
class Config:
"""Pydantic model configuration."""
extra = "allow"
alias_generator = snake_to_camel
fields = {
"query_location": {
"title": TEXT.query_location,
"description": "Router/Location Name",
"example": "router01",
},
"query_type": {
"title": TEXT.query_type,
"description": "Type of Query to Execute",
"example": "bgp_route",
},
"query_target": {
"title": TEXT.query_target,
"description": "IP Address, Community, or AS Path",
"example": "1.1.1.0/24",
},
}
schema_extra = {"x-code-samples": [{"lang": "Python", "source": "print('stuff')"}]}
allow_population_by_field_name = True
def __init__(self, **kwargs):
def __init__(self, **data) -> None:
"""Initialize the query with a UTC timestamp at initialization time."""
super().__init__(**kwargs)
super().__init__(**data)
self.timestamp = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
state = use_state()
self._state = state
query_directives = self.device.directives.matching(self.query_type)
if len(query_directives) < 1:
raise QueryTypeNotFound(query_type=self.query_type)
self.directive = query_directives[0]
try:
self.validate_query_target()
except InputValidationError as err:
raise InputInvalid(**err.kwargs)
def __repr__(self):
def __repr__(self) -> str:
"""Represent only the query fields."""
return repr_from_attrs(self, self.__config__.fields.keys())
@@ -79,17 +69,17 @@ class Query(BaseModel):
"""Alias __str__ to __repr__."""
return repr(self)
def digest(self):
def digest(self) -> str:
"""Create SHA256 hash digest of model representation."""
return hashlib.sha256(repr(self).encode()).hexdigest()
def random(self):
def random(self) -> str:
"""Create a random string to prevent client or proxy caching."""
return hashlib.sha256(
secrets.token_bytes(8) + repr(self).encode() + secrets.token_bytes(8)
).hexdigest()
def validate_query_target(self):
def validate_query_target(self) -> None:
"""Validate a query target after all fields/relationships havebeen initialized."""
# Run config/rule-based validations.
self.directive.validate_target(self.query_target)
@@ -98,7 +88,7 @@ class Query(BaseModel):
manager.execute(query=self)
log.debug("Validation passed for query {!r}", self)
def dict(self) -> t.Dict[str, str]:
def dict(self) -> t.Dict[str, t.Union[t.List[str], str]]:
"""Include only public fields."""
return super().dict(include={"query_location", "query_target", "query_type"})
@@ -107,15 +97,6 @@ class Query(BaseModel):
"""Get this query's device object by query_location."""
return self._state.devices[self.query_location]
@validator("query_type")
def validate_query_type(cls, value):
"""Ensure a requested query type exists."""
devices = use_state("devices")
if any((device.has_directives(value) for device in devices)):
return value
raise QueryTypeNotFound(name=value)
@validator("query_location")
def validate_query_location(cls, value):
"""Ensure query_location is defined."""
@@ -126,3 +107,12 @@ class Query(BaseModel):
raise QueryLocationNotFound(location=value)
return value
@validator("query_type")
def validate_query_type(cls, value: t.Any):
"""Ensure a requested query type exists."""
devices = use_state("devices")
if any((device.has_directives(value) for device in devices)):
return value
raise QueryTypeNotFound(query_type=value)

View File

@@ -125,9 +125,13 @@ class RuleWithIP(Rule):
def validate_target(self, target: "QueryTarget", *, multiple: bool) -> bool:
"""Validate an IP address target against this rule's conditions."""
if isinstance(target, t.List):
self._passed = False
raise InputValidationError("Target must be a single value")
if len(target) > 1:
self._passed = False
raise InputValidationError("Target must be a single value")
target = target[0]
try:
# Attempt to use IP object factory to create an IP address object
valid_target = ip_network(target)
@@ -257,6 +261,7 @@ class Directive(HyperglassUniqueModel, unique_by=("id", "table_output")):
table_output: t.Optional[StrictStr]
groups: t.List[StrictStr] = []
multiple: StrictBool = False
multiple_separator: StrictStr = " "
def validate_target(self, target: str) -> bool:
"""Validate a target against all configured rules."""

View File

@@ -302,7 +302,7 @@ class MultiModel(GenericModel, t.Generic[MultiModelT]):
`Model` is yielded.
"""
for search in searches:
pattern = re.compile(fr".*{search}.*", re.IGNORECASE)
pattern = re.compile(rf".*{search}.*", re.IGNORECASE)
for item in self:
if pattern.match(getattr(item, self.unique_by)):
yield item

View File

@@ -34,7 +34,7 @@ class UILocation(HyperglassModel):
id: StrictStr
name: StrictStr
group: StrictStr
group: Optional[StrictStr]
avatar: Optional[StrictStr]
description: Optional[StrictStr]
directives: List[UIDirective] = []
@@ -43,7 +43,7 @@ class UILocation(HyperglassModel):
class UIDevices(HyperglassModel):
"""UI: Devices."""
group: StrictStr
group: Optional[StrictStr]
locations: List[UILocation] = []

View File

@@ -52,7 +52,7 @@ export const LookingGlassForm = (): JSX.Element => {
enforce(data.queryLocation).isArrayOf(enforce.isString()).isNotEmpty();
});
test('queryTarget', noQueryTarget, () => {
enforce(data.queryTarget).longerThan(1);
enforce(data.queryTarget).isArrayOf(enforce.isString()).isNotEmpty();
});
test('queryType', noQueryType, () => {
enforce(data.queryType).inside(queryTypes);
@@ -62,7 +62,7 @@ export const LookingGlassForm = (): JSX.Element => {
const formInstance = useForm<FormData>({
resolver: vestResolver(formSchema),
defaultValues: {
queryTarget: '',
queryTarget: [],
queryLocation: [],
queryType: '',
},
@@ -72,8 +72,8 @@ export const LookingGlassForm = (): JSX.Element => {
// const isFqdnQuery = useIsFqdn(form.queryTarget, form.queryType);
const isFqdnQuery = useCallback(
(target: string, fieldType: Directive['fieldType'] | null): boolean =>
fieldType === 'text' && isFQDN(target),
(target: string | string[], fieldType: Directive['fieldType'] | null): boolean =>
typeof target === 'string' && fieldType === 'text' && isFQDN(target),
[],
);
@@ -138,15 +138,19 @@ export const LookingGlassForm = (): JSX.Element => {
} else if (e.field === 'queryType' && isString(e.value)) {
setValue('queryType', e.value);
setFormValue('queryType', e.value);
if (form.queryTarget !== '') {
if (form.queryTarget.length !== 0) {
// Reset queryTarget as well, so that, for example, selecting BGP Community, and selecting
// a community, then changing the queryType to BGP Route doesn't preserve the selected
// community as the queryTarget.
setFormValue('queryTarget', '');
setFormValue('queryTarget', []);
setTarget({ display: '' });
}
} else if (e.field === 'queryTarget' && isString(e.value)) {
setFormValue('queryTarget', e.value);
} else if (e.field === 'queryTarget') {
if (isString(e.value)) {
setFormValue('queryTarget', [e.value]);
} else if (Array.isArray(e.value)) {
setFormValue('queryTarget', e.value);
}
}
}
@@ -213,7 +217,7 @@ export const LookingGlassForm = (): JSX.Element => {
flexDir="column"
mr={{ base: 0, lg: 2 }}
>
<ScaleFade initialScale={0.5} in={form.queryTarget !== ''}>
<ScaleFade initialScale={0.5} in={form.queryTarget.length !== 0}>
<SubmitButton />
</ScaleFade>
</Flex>

View File

@@ -62,7 +62,7 @@ export const QueryTarget = (props: QueryTargetProps): JSX.Element => {
function handleInputChange(e: React.ChangeEvent<HTMLInputElement>): void {
setTarget({ display: e.target.value });
onChange({ field: name, value: e.target.value });
onChange({ field: name, value: [e.target.value] });
}
const handleSelectChange: SelectOnChange<OptionWithDescription> = e => {

View File

@@ -58,7 +58,7 @@ export const ResolvedTarget = (props: ResolvedTargetProps): JSX.Element => {
const answer6 = useMemo(() => findAnswer(data6), [data6]);
function selectTarget(value: string): void {
setFormValue('queryTarget', value);
setFormValue('queryTarget', [value]);
setStatus('results');
}

View File

@@ -84,7 +84,7 @@ export const Tags = (): JSX.Element => {
>
<Label
bg={targetBg}
value={form.queryTarget}
value={form.queryTarget.join(', ')}
label={web.text.queryTarget}
fontSize={{ base: 'xs', md: 'sm' }}
/>

View File

@@ -15,7 +15,7 @@ type FormStatus = 'form' | 'results';
interface FormValues {
queryLocation: string[];
queryTarget: string;
queryTarget: string[];
queryType: string;
}
@@ -82,7 +82,7 @@ interface FormStateType<Opt extends SingleOption = SingleOption> {
const formState: StateCreator<FormStateType> = (set, get) => ({
filtered: { types: [], groups: [] },
form: { queryLocation: [], queryTarget: '', queryType: '' },
form: { queryLocation: [], queryTarget: [], queryType: '' },
loading: false,
responses: {},
selections: { queryLocation: [], queryType: null },
@@ -204,7 +204,7 @@ const formState: StateCreator<FormStateType> = (set, get) => ({
reset(): void {
set({
filtered: { types: [], groups: [] },
form: { queryLocation: [], queryTarget: '', queryType: '' },
form: { queryLocation: [], queryTarget: [], queryType: '' },
loading: false,
responses: {},
selections: { queryLocation: [], queryType: null },
@@ -230,7 +230,7 @@ export function useView(): FormStatus {
status === 'results',
form.queryLocation.length !== 0,
form.queryType !== '',
form.queryTarget !== '',
form.queryTarget.length !== 0,
);
return ready ? 'results' : 'form';
}, [status, form]);

View File

@@ -54,7 +54,7 @@ function formatTime(val: number): string {
* Get a function to convert table data to string, for use in the copy button component.
*/
export function useTableToString(
target: string,
target: string[],
data: QueryResponse | undefined,
...deps: unknown[]
): () => string {
@@ -90,11 +90,14 @@ export function useTableToString(
}
}
function doFormat(target: string, data: QueryResponse | undefined): string {
function doFormat(target: string[], data: QueryResponse | undefined): string {
let result = messages.noOutput;
try {
if (typeof data !== 'undefined' && isStructuredOutput(data)) {
const tableStringParts = [`Routes For: ${target}`, `Timestamp: ${data.timestamp} UTC`];
const tableStringParts = [
`Routes For: ${target.join(', ')}`,
`Timestamp: ${data.timestamp} UTC`,
];
for (const route of data.output.routes) {
for (const field of parsedDataFields) {
const [header, accessor, align] = field;

View File

@@ -7,7 +7,7 @@
"private": true,
"scripts": {
"lint": "eslint . --ext .ts --ext .tsx",
"dev": "export NODE_OPTIONS=--openssl-legacy-provider; node nextdev",
"dev": "node nextdev",
"start": "next start",
"typecheck": "tsc --noEmit",
"format": "prettier --config ./.prettierrc -c -w .",

View File

@@ -1,7 +1,7 @@
export interface FormData {
queryLocation: string[];
queryType: string;
queryTarget: string;
queryTarget: string[];
}
export type FormQuery = Swap<FormData, 'queryLocation', string>;

1553
poetry.lock generated
View File

File diff suppressed because it is too large Load Diff

View File

@@ -1,6 +1,7 @@
[build-system]
build-backend = "poetry.masonry.api"
requires = ["poetry>=0.12"]
build-backend = "poetry.core.masonry.api"
requires = ["poetry-core"]
[tool.poetry]
authors = ["Matt Love <matt@hyperglass.dev>"]
@@ -33,18 +34,17 @@ Pillow = "^8.3.2"
PyJWT = "^2.0.1"
PyYAML = "^5.4.1"
aiofiles = "^0.7.0"
cryptography = "3.0.0"
distro = "^1.5.0"
fastapi = "^0.63.0"
fastapi = "^0.73.0"
favicons = ">=0.1.0,<1.0"
gunicorn = "^20.1.0"
httpx = "^0.20.0"
loguru = "^0.5.3"
netmiko = "^3.4.0"
paramiko = "^2.7.2"
paramiko = "^2.12.0"
psutil = "^5.7.2"
py-cpuinfo = "^8.0.0"
pydantic = {extras = ["dotenv"], version = "^1.8.2"}
pydantic = {extras = ["dotenv"], version = "^1.9.0"}
python = ">=3.8.1,<4.0"
redis = "^3.5.3"
rich = "^10.11.0"
@@ -54,30 +54,29 @@ uvicorn = {extras = ["standard"], version = "^0.15.0"}
uvloop = "^0.16.0"
xmltodict = "^0.12.0"
[tool.poetry.dev-dependencies]
bandit = "^1.6.2"
black = "^21.12b0"
flake8 = "^3.8"
flake8-bandit = "^2.1.2"
flake8-black = "^0.2.2"
[tool.poetry.group.dev.dependencies]
bandit = "^1.7.4"
black = "^22.12.0"
flake8 = "^6.0.0"
flake8-bandit = "^4.1.1"
flake8-black = "^0.3.5"
flake8-breakpoint = "^1.1.0"
flake8-bugbear = "^20.1.0"
flake8-builtins = "^1.4.2"
flake8-comprehensions = "^3.1.4"
flake8-deprecated = "^1.3"
flake8-docstrings = "^1.5.0"
flake8-isort = "^4.0.0"
flake8-plugin-utils = "^1.3.1"
flake8-bugbear = "^22.12.6"
flake8-builtins = "^2.0.1"
flake8-comprehensions = "^3.10.1"
flake8-deprecated = "^2.0.1"
flake8-docstrings = "^1.6.0"
flake8-isort = "^5.0.3"
flake8-plugin-utils = "^1.3.2"
flake8-polyfill = "^1.0.2"
flake8-print = "^3.1.4"
isort = "^5.5.3"
mccabe = "^0.6.1"
pep8-naming = "^0.9.1"
flake8-print = "^5.0.0"
isort = "^5.10.1"
pep8-naming = "^0.13.2"
pre-commit = "^1.21.0"
pytest = "^6.2.5"
pytest = "^7.2.0"
pytest-dependency = "^0.5.1"
stackprinter = "^0.2.3"
taskipy = "^1.8.2"
stackprinter = "^0.2.10"
taskipy = "^1.10.3"
[tool.black]
line-length = 100
@@ -106,6 +105,7 @@ reportMissingTypeStubs = true
[tool.taskipy.tasks]
check = {cmd = "task lint && task ui-lint", help = "Run all lint checks"}
format = {cmd = "black hyperglass", help = "Run Black"}
lint = {cmd = "flake8 hyperglass", help = "Run Flake8"}
sort = {cmd = "isort hyperglass", help = "Run iSort"}
start = {cmd = "python3 -m hyperglass.main", help = "Start hyperglass"}