major caching improvements

hyperglass/api/models/query.py

@@ -3,6 +3,8 @@
 # Standard Library
 import json
 import hashlib
+import secrets
+from datetime import datetime
 
 # Third Party
 from pydantic import BaseModel, StrictStr, validator
@@ -57,6 +59,7 @@ class Query(BaseModel):
     class Config:
         """Pydantic model configuration."""
 
+        extra = "allow"
         fields = {
             "query_location": {
                 "title": params.web.text.query_location,
@@ -83,10 +86,29 @@ class Query(BaseModel):
             "x-code-samples": [{"lang": "Python", "source": "print('stuff')"}]
         }
 
+    def __init__(self, **kwargs):
+        """Initialize the query with a UTC timestamp at initialization time."""
+        super().__init__(**kwargs)
+        self.timestamp = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
+
+    def __repr__(self):
+        """Represent only the query fields."""
+        return (
+            f"Query(query_location={str(self.query_location)}, "
+            f"query_type={str(self.query_type)}, query_vrf={str(self.query_vrf)}, "
+            f"query_target={str(self.query_target)})"
+        )
+
     def digest(self):
         """Create SHA256 hash digest of model representation."""
         return hashlib.sha256(repr(self).encode()).hexdigest()
 
+    def random(self):
+        """Create a random string to prevent client or proxy caching."""
+        return hashlib.sha256(
+            secrets.token_bytes(8) + repr(self).encode() + secrets.token_bytes(8)
+        ).hexdigest()
+
     @property
     def summary(self):
         """Create abbreviated representation of instance."""

hyperglass/api/models/response.py

@@ -57,10 +57,11 @@ class QueryResponse(BaseModel):
 
     output: StrictStr
     level: constr(regex=r"success") = "success"
-    id: StrictStr
+    random: StrictStr
     cached: StrictBool
     runtime: StrictInt
     keywords: List[StrictStr] = []
+    timestamp: StrictStr
 
     class Config:
         """Pydantic model configuration."""
@@ -69,6 +70,25 @@ class QueryResponse(BaseModel):
         description = "Looking glass response"
         fields = {
             "level": {"title": "Level", "description": "Severity"},
+            "cached": {
+                "title": "Cached",
+                "description": "`true` if the response is from a previously cached query.",
+            },
+            "random": {
+                "title": "Random",
+                "description": "Random string to prevent client or intermediate caching.",
+                "example": "504cbdb47eb8310ca237bf512c3e10b44b0a3d85868c4b64a20037dc1c3ef857",
+            },
+            "runtime": {
+                "title": "Runtime",
+                "description": "Time it took to run the query in seconds.",
+                "example": 6,
+            },
+            "timestamp": {
+                "title": "Timestamp",
+                "description": "UTC Time at which the backend application received the query.",
+                "example": "2020-04-18 14:45:37",
+            },
             "keywords": {
                 "title": "Keywords",
                 "description": "Relevant keyword values contained in the `output` field, which can be used for formatting.",

hyperglass/api/routes.py

@@ -71,7 +71,7 @@ async def query(query_data: Query, request: Request):
 
     log.info(f"Starting query execution for query {query_data.summary}")
 
-    cache_response = await cache.get(cache_key)
+    cache_response = await cache.get_dict(cache_key, "output")
 
     cached = False
     if cache_response:
@@ -80,6 +80,7 @@ async def query(query_data: Query, request: Request):
 
         cached = True
         runtime = 0
+        timestamp = await cache.get_dict(cache_key, "timestamp")
 
     elif not cache_response:
         log.debug(f"No existing cache entry for query {cache_key}")
@@ -87,18 +88,20 @@ async def query(query_data: Query, request: Request):
             f"Created new cache key {cache_key} entry for query {query_data.summary}"
         )
 
+        timestamp = query_data.timestamp
         # Pass request to execution module
         starttime = time.time()
-        cache_value = await Execute(query_data).response()
+        cache_output = await Execute(query_data).response()
         endtime = time.time()
         elapsedtime = round(endtime - starttime, 4)
         log.debug(f"Query {cache_key} took {elapsedtime} seconds to run.")
 
-        if cache_value is None:
+        if cache_output is None:
             raise HyperglassError(message=params.messages.general, alert="danger")
 
         # Create a cache entry
-        await cache.set(cache_key, str(cache_value))
+        await cache.set_dict(cache_key, "output", str(cache_output))
+        await cache.set_dict(cache_key, "timestamp", timestamp)
         await cache.expire(cache_key, seconds=cache_timeout)
 
         log.debug(f"Added cache entry for query: {cache_key}")
@@ -106,7 +109,7 @@ async def query(query_data: Query, request: Request):
         runtime = int(round(elapsedtime, 0))
 
     # If it does, return the cached entry
-    cache_response = await cache.get(cache_key)
+    cache_response = await cache.get_dict(cache_key, "output")
 
     log.debug(f"Cache match for {cache_key}:\n {cache_response}")
     log.success(f"Completed query execution for {query_data.summary}")
@@ -116,6 +119,8 @@ async def query(query_data: Query, request: Request):
         "id": cache_key,
         "cached": cached,
         "runtime": runtime,
+        "timestamp": timestamp,
+        "random": query_data.random(),
         "level": "success",
         "keywords": [],
     }

hyperglass/cache.py

@@ -71,10 +71,22 @@ class Cache:
             raw = await self.instance.mget(args)
         return await self._parse_types(raw)
 
+    async def get_dict(self, key, field=None):
+        """Get hash map (dict) item(s)."""
+        if field is None:
+            raw = await self.instance.hgetall(key)
+        else:
+            raw = await self.instance.hget(key, field)
+        return await self._parse_types(raw)
+
     async def set(self, key, value):
         """Set cache values."""
         return await self.instance.set(key, value)
 
+    async def set_dict(self, key, field, value):
+        """Set hash map (dict) values."""
+        return await self.instance.hset(key, field, value)
+
     async def wait(self, pubsub, timeout=30, **kwargs):
         """Wait for pub/sub messages & return posted message."""
         now = time.time()

hyperglass/configuration/models/web.py

@@ -139,7 +139,7 @@ class Text(HyperglassModel):
     query_vrf: StrictStr = "Routing Table"
     fqdn_tooltip: StrictStr = "Use {protocol}"  # Formatted by Javascript
     cache_prefix: StrictStr = "Results cached for "
-    cache_icon: StrictStr = "Cached Response"
+    cache_icon: StrictStr = "Cached Response from {time}"  # Formatted by Javascript
     complete_time: StrictStr = "Completed in {seconds}"  # Formatted by Javascript
 
     @validator("title_mode")

hyperglass/ui/components/Result.js

@@ -16,6 +16,7 @@ import styled from "@emotion/styled";
 import LightningBolt from "~/components/icons/LightningBolt";
 import useAxios from "axios-hooks";
 import strReplace from "react-string-replace";
+import format from "string-format";
 import { startCase } from "lodash";
 import useConfig from "~/components/HyperglassProvider";
 import useMedia from "~/components/MediaProvider";
@@ -24,6 +25,8 @@ import RequeryButton from "~/components/RequeryButton";
 import ResultHeader from "~/components/ResultHeader";
 import CacheTimeout from "~/components/CacheTimeout";
 
+format.extend(String.prototype, {});
+
 const FormattedError = ({ keywords, message }) => {
     const patternStr = keywords.map((kw) => `(${kw})`).join("|");
     const pattern = new RegExp(patternStr, "gi");
@@ -125,7 +128,11 @@ const Result = React.forwardRef(
             <>
                 <CacheTimeout timeout={config.cache.timeout} text={config.web.text.cache_prefix} />
                 {data?.cached && (
-                    <Tooltip hasArrow label={config.web.text.cache_icon} placement="top">
+                    <Tooltip
+                        hasArrow
+                        label={config.web.text.cache_icon.format({ time: data?.timestamp })}
+                        placement="top"
+                    >
                         <Box ml={1}>
                             <LightningBolt color={color[colorMode]} />
                         </Box>
@@ -136,7 +143,11 @@ const Result = React.forwardRef(
         const cacheSm = (
             <>
                 {data?.cached && (
-                    <Tooltip hasArrow label={config.web.text.cache_icon} placement="top">
+                    <Tooltip
+                        hasArrow
+                        label={config.web.text.cache_icon.format({ time: data?.timestamp })}
+                        placement="top"
+                    >
                         <Box mr={1}>
                             <LightningBolt color={color[colorMode]} />
                         </Box>

hyperglass/ui/components/ResultHeader.js

@@ -7,7 +7,7 @@ format.extend(String.prototype, {});
 
 const runtimeText = (runtime, text) => {
     let unit;
-    if (runtime > 1) {
+    if (runtime === 1) {
         unit = "seconds";
     } else {
         unit = "second";