From caf294a36c4d861d2265d56cf58b027ea225a352 Mon Sep 17 00:00:00 2001 From: checktheroads Date: Sat, 18 Apr 2020 07:58:46 -0700 Subject: [PATCH] major caching improvements --- hyperglass/api/models/query.py | 22 ++++++++++++++++++++++ hyperglass/api/models/response.py | 22 +++++++++++++++++++++- hyperglass/api/routes.py | 15 ++++++++++----- hyperglass/cache.py | 12 ++++++++++++ hyperglass/configuration/models/web.py | 2 +- hyperglass/ui/components/Result.js | 15 +++++++++++++-- hyperglass/ui/components/ResultHeader.js | 2 +- 7 files changed, 80 insertions(+), 10 deletions(-) diff --git a/hyperglass/api/models/query.py b/hyperglass/api/models/query.py index 1d8c882..6db5bea 100644 --- a/hyperglass/api/models/query.py +++ b/hyperglass/api/models/query.py @@ -3,6 +3,8 @@ # Standard Library import json import hashlib +import secrets +from datetime import datetime # Third Party from pydantic import BaseModel, StrictStr, validator @@ -57,6 +59,7 @@ class Query(BaseModel): class Config: """Pydantic model configuration.""" + extra = "allow" fields = { "query_location": { "title": params.web.text.query_location, @@ -83,10 +86,29 @@ class Query(BaseModel): "x-code-samples": [{"lang": "Python", "source": "print('stuff')"}] } + def __init__(self, **kwargs): + """Initialize the query with a UTC timestamp at initialization time.""" + super().__init__(**kwargs) + self.timestamp = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S") + + def __repr__(self): + """Represent only the query fields.""" + return ( + f"Query(query_location={str(self.query_location)}, " + f"query_type={str(self.query_type)}, query_vrf={str(self.query_vrf)}, " + f"query_target={str(self.query_target)})" + ) + def digest(self): """Create SHA256 hash digest of model representation.""" return hashlib.sha256(repr(self).encode()).hexdigest() + def random(self): + """Create a random string to prevent client or proxy caching.""" + return hashlib.sha256( + secrets.token_bytes(8) + repr(self).encode() + secrets.token_bytes(8) + ).hexdigest() + @property def summary(self): """Create abbreviated representation of instance.""" diff --git a/hyperglass/api/models/response.py b/hyperglass/api/models/response.py index 463b500..2c2d860 100644 --- a/hyperglass/api/models/response.py +++ b/hyperglass/api/models/response.py @@ -57,10 +57,11 @@ class QueryResponse(BaseModel): output: StrictStr level: constr(regex=r"success") = "success" - id: StrictStr + random: StrictStr cached: StrictBool runtime: StrictInt keywords: List[StrictStr] = [] + timestamp: StrictStr class Config: """Pydantic model configuration.""" @@ -69,6 +70,25 @@ class QueryResponse(BaseModel): description = "Looking glass response" fields = { "level": {"title": "Level", "description": "Severity"}, + "cached": { + "title": "Cached", + "description": "`true` if the response is from a previously cached query.", + }, + "random": { + "title": "Random", + "description": "Random string to prevent client or intermediate caching.", + "example": "504cbdb47eb8310ca237bf512c3e10b44b0a3d85868c4b64a20037dc1c3ef857", + }, + "runtime": { + "title": "Runtime", + "description": "Time it took to run the query in seconds.", + "example": 6, + }, + "timestamp": { + "title": "Timestamp", + "description": "UTC Time at which the backend application received the query.", + "example": "2020-04-18 14:45:37", + }, "keywords": { "title": "Keywords", "description": "Relevant keyword values contained in the `output` field, which can be used for formatting.", diff --git a/hyperglass/api/routes.py b/hyperglass/api/routes.py index 23b3880..8960983 100644 --- a/hyperglass/api/routes.py +++ b/hyperglass/api/routes.py @@ -71,7 +71,7 @@ async def query(query_data: Query, request: Request): log.info(f"Starting query execution for query {query_data.summary}") - cache_response = await cache.get(cache_key) + cache_response = await cache.get_dict(cache_key, "output") cached = False if cache_response: @@ -80,6 +80,7 @@ async def query(query_data: Query, request: Request): cached = True runtime = 0 + timestamp = await cache.get_dict(cache_key, "timestamp") elif not cache_response: log.debug(f"No existing cache entry for query {cache_key}") @@ -87,18 +88,20 @@ async def query(query_data: Query, request: Request): f"Created new cache key {cache_key} entry for query {query_data.summary}" ) + timestamp = query_data.timestamp # Pass request to execution module starttime = time.time() - cache_value = await Execute(query_data).response() + cache_output = await Execute(query_data).response() endtime = time.time() elapsedtime = round(endtime - starttime, 4) log.debug(f"Query {cache_key} took {elapsedtime} seconds to run.") - if cache_value is None: + if cache_output is None: raise HyperglassError(message=params.messages.general, alert="danger") # Create a cache entry - await cache.set(cache_key, str(cache_value)) + await cache.set_dict(cache_key, "output", str(cache_output)) + await cache.set_dict(cache_key, "timestamp", timestamp) await cache.expire(cache_key, seconds=cache_timeout) log.debug(f"Added cache entry for query: {cache_key}") @@ -106,7 +109,7 @@ async def query(query_data: Query, request: Request): runtime = int(round(elapsedtime, 0)) # If it does, return the cached entry - cache_response = await cache.get(cache_key) + cache_response = await cache.get_dict(cache_key, "output") log.debug(f"Cache match for {cache_key}:\n {cache_response}") log.success(f"Completed query execution for {query_data.summary}") @@ -116,6 +119,8 @@ async def query(query_data: Query, request: Request): "id": cache_key, "cached": cached, "runtime": runtime, + "timestamp": timestamp, + "random": query_data.random(), "level": "success", "keywords": [], } diff --git a/hyperglass/cache.py b/hyperglass/cache.py index f330a08..71d21f9 100644 --- a/hyperglass/cache.py +++ b/hyperglass/cache.py @@ -71,10 +71,22 @@ class Cache: raw = await self.instance.mget(args) return await self._parse_types(raw) + async def get_dict(self, key, field=None): + """Get hash map (dict) item(s).""" + if field is None: + raw = await self.instance.hgetall(key) + else: + raw = await self.instance.hget(key, field) + return await self._parse_types(raw) + async def set(self, key, value): """Set cache values.""" return await self.instance.set(key, value) + async def set_dict(self, key, field, value): + """Set hash map (dict) values.""" + return await self.instance.hset(key, field, value) + async def wait(self, pubsub, timeout=30, **kwargs): """Wait for pub/sub messages & return posted message.""" now = time.time() diff --git a/hyperglass/configuration/models/web.py b/hyperglass/configuration/models/web.py index f1572ec..0272690 100644 --- a/hyperglass/configuration/models/web.py +++ b/hyperglass/configuration/models/web.py @@ -139,7 +139,7 @@ class Text(HyperglassModel): query_vrf: StrictStr = "Routing Table" fqdn_tooltip: StrictStr = "Use {protocol}" # Formatted by Javascript cache_prefix: StrictStr = "Results cached for " - cache_icon: StrictStr = "Cached Response" + cache_icon: StrictStr = "Cached Response from {time}" # Formatted by Javascript complete_time: StrictStr = "Completed in {seconds}" # Formatted by Javascript @validator("title_mode") diff --git a/hyperglass/ui/components/Result.js b/hyperglass/ui/components/Result.js index 7934049..cd2aef1 100644 --- a/hyperglass/ui/components/Result.js +++ b/hyperglass/ui/components/Result.js @@ -16,6 +16,7 @@ import styled from "@emotion/styled"; import LightningBolt from "~/components/icons/LightningBolt"; import useAxios from "axios-hooks"; import strReplace from "react-string-replace"; +import format from "string-format"; import { startCase } from "lodash"; import useConfig from "~/components/HyperglassProvider"; import useMedia from "~/components/MediaProvider"; @@ -24,6 +25,8 @@ import RequeryButton from "~/components/RequeryButton"; import ResultHeader from "~/components/ResultHeader"; import CacheTimeout from "~/components/CacheTimeout"; +format.extend(String.prototype, {}); + const FormattedError = ({ keywords, message }) => { const patternStr = keywords.map((kw) => `(${kw})`).join("|"); const pattern = new RegExp(patternStr, "gi"); @@ -125,7 +128,11 @@ const Result = React.forwardRef( <> {data?.cached && ( - + @@ -136,7 +143,11 @@ const Result = React.forwardRef( const cacheSm = ( <> {data?.cached && ( - + diff --git a/hyperglass/ui/components/ResultHeader.js b/hyperglass/ui/components/ResultHeader.js index aab8c30..764d0f5 100644 --- a/hyperglass/ui/components/ResultHeader.js +++ b/hyperglass/ui/components/ResultHeader.js @@ -7,7 +7,7 @@ format.extend(String.prototype, {}); const runtimeText = (runtime, text) => { let unit; - if (runtime > 1) { + if (runtime === 1) { unit = "seconds"; } else { unit = "second";