Nanog 83 Hackathon improvements to the PeeringDB Website (#1083)

* Nanog 83 Hackathon improvements to the PeeringDB Website

- If a user inputs only numbers, search for ASN only
- If a user inputs what looks like a IPv4 or IPv6 address, search Network IP addresses only
- Use direct SQL in the above instances rather than Haystack fuzzy matching, though search fields for ASN and ipaddr[46] were added to the Haystack search index

* Revert change to prepare_term

* remove print

* run black

* add ipv4 and 6 search tests

Co-authored-by: Jeff Kala <jeff.l.kala@gmail.com>

peeringdb_server/models.py

@@ -4055,6 +4055,22 @@ class IXLanPrefix(ProtectedMixin, pdb_models.IXLanPrefixBase):
         validate_prefix_overlap(self.prefix)
         return super().clean()
 
+    @property
+    def ix_result_name(self):
+        return self.ixlan.ix.search_result_name
+
+    @property
+    def ix_org_id(self):
+        return self.ixlan.ix.org_id
+
+    @property
+    def ix_id(self):
+        return self.ixlan.ix.id
+
+    @property
+    def ix_sub_result_name(self):
+        return self.prefix
+
 
 @grainy_model(namespace="network", parent="org")
 @reversion.register
@@ -4738,6 +4754,44 @@ class NetworkIXLan(pdb_models.NetworkIXLanBase):
         """
         return f"netixlan{self.id} AS{self.asn} {self.ipaddr(version)}"
 
+    @property
+    def ix_result_name(self):
+        return self.ixlan.ix.search_result_name
+
+    @property
+    def ix_org_id(self):
+        return self.ixlan.ix.org_id
+
+    @property
+    def ix_id(self):
+        return self.ixlan.ix.id
+
+    @property
+    def net_result_name(self):
+        return self.network.search_result_name
+
+    @property
+    def net_org_id(self):
+        return self.network.org_id
+
+    @property
+    def net_id(self):
+        return self.network.id
+
+    @property
+    def ix_sub_result_name(self):
+        if self.ipaddr4 and self.ipaddr6:
+            return f"{self.ipaddr4} {self.ipaddr6}"
+        elif self.ipaddr4:
+            return f"{self.ipaddr4}"
+        elif self.ipaddr6:
+            return f"{self.ipaddr6}"
+
+    @property
+    def net_sub_result_name(self):
+        ips = self.ix_sub_result_name
+        return f"{self.ixlan.ix.search_result_name} {ips}"
+
 
 class User(AbstractBaseUser, PermissionsMixin):
     """

peeringdb_server/search.py

@@ -9,6 +9,7 @@ Refer to search_indexes.py for search index definition.
 """
 
 # import time
+import re
 import unidecode
 from django.conf import settings
 from django.db.models import Q
@@ -49,8 +50,33 @@ searchable_models = [
 ]
 
 
+ONLY_DIGITS = re.compile(r"^[0-9]+$")
+# These are not exact, but should be good enough
+PARTIAL_IPV4_ADDRESS = re.compile(r"^([0-9]{1,3}\.){1,3}([0-9]{1,3})?$")
+PARTIAL_IPV6_ADDRESS = re.compile(r"^([0-9A-Fa-f]{1,4}|:):[0-9A-Fa-f:]*$")
+
+
 def unaccent(v):
-    return unidecode.unidecode(v).lower()
+    return unidecode.unidecode(v).lower().strip()
+
+
+def valid_partial_ipv4_address(ip):
+    return all(int(s) >= 0 and int(s) <= 255 for s in ip.split(".") if len(s) > 0)
+
+
+def make_asn_query(term):
+    return Network.objects.filter(asn__exact=term)
+    # return SearchQuerySet().filter(Q(asn__exact=term)).models(Network)
+
+
+def make_ipv4_query(term):
+    return NetworkIXLan.objects.filter(ipaddr4__startswith=term)
+    # return SearchQuerySet().filter(Q(ipaddr4__startswith=term)).models(NetworkIXLan)
+
+
+def make_ipv6_query(term):
+    return NetworkIXLan.objects.filter(ipaddr6__startswith=term)
+    # return SearchQuerySet().filter(Q(ipaddr6__startswith=term)).models(NetworkIXLan)
 
 
 def prepare_term(term):
@@ -68,11 +94,25 @@ def make_search_query(term):
     if not term:
         return SearchQuerySet().none()
 
-    term = prepare_term(term)
+    term = unaccent(term)
+
+    if ONLY_DIGITS.match(term):
+        return make_asn_query(term)
+
+    if PARTIAL_IPV4_ADDRESS.match(term):
+        if valid_partial_ipv4_address(term):
+            return make_ipv4_query(term)
+
+    if PARTIAL_IPV6_ADDRESS.match(term):
+        return make_ipv6_query(term)
 
     term_filters = Q(content=term) | Q(content__startswith=term)
 
-    return SearchQuerySet().filter(term_filters, status=Exact("ok"))
+    return (
+        SearchQuerySet()
+        .filter(term_filters, status=Exact("ok"))
+        .models(*searchable_models)
+    )
 
 
 def make_name_search_query(term):
@@ -107,39 +147,34 @@ def search(term, autocomplete=False):
         search_query = make_autocomplete_query(term).models(*autocomplete_models)
         limit = settings.SEARCH_RESULTS_AUTOCOMPLETE_LIMIT
     else:
-        search_query = make_search_query(term).models(*searchable_models)
+        search_query = make_search_query(term)
         limit = settings.SEARCH_RESULTS_LIMIT
 
     categories = ("fac", "ix", "net", "org")
     result = {tag: [] for tag in categories}
     pk_map = {tag: {} for tag in categories}
 
-    # if term is an exact asn match, ensure that the matching
-    # network is always appended as the first entry
-    # issue #232
-
-    try:
-        asn_match = Network.objects.get(asn=term)
-        append_result(
-            "net",
-            asn_match.pk,
-            asn_match.search_result_name,
-            asn_match.org_id,
-            None,
-            result,
-            pk_map,
-        )
-    except (Network.DoesNotExist, ValueError):
-        pass
-
     # add entries to the result by order of scoring with the
     # highest scored on top (beginning of list)
 
     for sq in search_query[:limit]:
-        model = sq.model
+        if hasattr(sq, "model"):
-        model.HandleRef.tag
+            model = sq.model
-
+            model.HandleRef.tag
-        categorize(sq, result, pk_map)
+            categorize(sq, result, pk_map)
+        else:
+            if sq.HandleRef.tag == "netixlan":
+                add_secondary_entries(sq, result, pk_map)
+            else:
+                append_result(
+                    sq.HandleRef.tag,
+                    sq.pk,
+                    getattr(sq, "search_result_name", None),
+                    sq.org_id,
+                    None,
+                    result,
+                    pk_map,
+                )
 
     # print("done", time.time() - t0)
 
@@ -156,10 +191,11 @@ def categorize(sq, result, pk_map):
         else:
             org_id = sq.org_id
         append_result(tag, int(sq.pk), sq.result_name, org_id, None, result, pk_map)
-        return
+    else:
+        add_secondary_entries(sq, result, pk_map)
 
-    # secondary entities
 
+def add_secondary_entries(sq, result, pk_map):
     for tag in result.keys():
         if not getattr(sq, f"{tag}_result_name", None):
             continue

peeringdb_server/search_indexes.py

@@ -137,6 +137,7 @@ class InternetExchangeIndex(MainEntity, indexes.Indexable):
 
 class NetworkIndex(MainEntity, indexes.Indexable):
     org_id = indexes.IntegerField(indexed=False, model_attr="org_id")
+    asn = indexes.IntegerField(indexed=False, model_attr="asn")
 
     class Meta:
         relations = ["org"]
@@ -172,6 +173,9 @@ class NetworkIXLanIndex(EntityIndex, indexes.Indexable):
     net_sub_result_name = indexes.CharField(indexed=False)
     ix_sub_result_name = indexes.CharField(indexed=False)
 
+    ipaddr4 = indexes.CharField(indexed=False)
+    ipaddr6 = indexes.CharField(indexed=False)
+
     class Meta:
         relations = ["network", "ixlan__ix", "network__org", "ixlan__ix__org"]
 
@@ -196,6 +200,18 @@ class NetworkIXLanIndex(EntityIndex, indexes.Indexable):
         ips = self.prepare_ix_sub_result_name(obj)
         return f"{obj.ixlan.ix.search_result_name} {ips}"
 
+    def prepare_ipaddr4(self, obj):
+        if obj.ipaddr4:
+            return str(obj.ipaddr4)
+        else:
+            return ""
+
+    def prepare_ipaddr6(self, obj):
+        if obj.ipaddr6:
+            return str(obj.ipaddr6)
+        else:
+            return ""
+
 
 class IXLanPrefixIndex(EntityIndex, indexes.Indexable):
 

tests/test_search.py

@@ -209,3 +209,109 @@ class SearchTests(TestCase):
         net_2.delete(hard=True)
         net_3.delete(hard=True)
         call_command("rebuild_index", "--noinput")
+
+    def test_search_ipv4(self):
+        """
+        This will test a search for a partial ipv4 address
+        """
+        org_1 = models.Organization.objects.create(name="Test org 1")
+        org_2 = models.Organization.objects.create(name="Test org 2")
+        net_1 = models.Network.objects.create(org_id=org_1.id, asn=34532, name="Net 1")
+        net_2 = models.Network.objects.create(org_id=org_2.id, asn=2432, name="Net 2")
+        ix_1 = models.InternetExchange.objects.create(org_id=org_1.id, name="IX 1")
+        ix_2 = models.InternetExchange.objects.create(org_id=org_2.id, name="IX 2")
+        next_id = models.IXLan.objects.all().order_by("-id").first().id + 1
+        ixlan_1 = models.IXLan(id=next_id, ix=ix_1)
+        ixlan_2 = models.IXLan(id=next_id + 1, ix=ix_2)
+        ixlan_1.save()
+        ixlan_2.save()
+        netixlan_1 = models.NetworkIXLan.objects.create(
+            ipaddr4="8.8.4.4",
+            asn=34532,
+            ixlan=ixlan_1,
+            network=net_1,
+            speed=1000,
+            status="ok",
+        )
+
+        netixlan_2 = models.NetworkIXLan.objects.create(
+            ipaddr4="8.8.8.8",
+            asn=2432,
+            ixlan=ixlan_2,
+            network=net_2,
+            speed=1000,
+            status="ok",
+        )
+
+        call_command("rebuild_index", "--noinput")
+
+        rv = search.search("8.8.4")
+
+        assert rv["net"][0]["id"] == net_1.id
+
+        # clean up
+        netixlan_1.delete(hard=True)
+        netixlan_2.delete(hard=True)
+        ix_1.delete(hard=True)
+        ix_2.delete(hard=True)
+        ixlan_1.delete(hard=True)
+        ixlan_2.delete(hard=True)
+        net_1.delete(hard=True)
+        net_2.delete(hard=True)
+        org_1.delete(hard=True)
+        org_2.delete(hard=True)
+
+        call_command("rebuild_index", "--noinput")
+
+    def test_search_ipv6(self):
+        """
+        This will test a search for a partial ipv6 address.
+        """
+        org_1 = models.Organization.objects.create(name="Test org 1")
+        org_2 = models.Organization.objects.create(name="Test org 2")
+        net_1 = models.Network.objects.create(org_id=org_1.id, asn=34532, name="Net 1")
+        net_2 = models.Network.objects.create(org_id=org_2.id, asn=2432, name="Net 2")
+        ix_1 = models.InternetExchange.objects.create(org_id=org_1.id, name="IX 1")
+        ix_2 = models.InternetExchange.objects.create(org_id=org_2.id, name="IX 2")
+        next_id = models.IXLan.objects.all().order_by("-id").first().id + 1
+        ixlan_1 = models.IXLan(id=next_id, ix=ix_1)
+        ixlan_2 = models.IXLan(id=next_id + 1, ix=ix_2)
+        ixlan_1.save()
+        ixlan_2.save()
+        netixlan_1 = models.NetworkIXLan.objects.create(
+            ipaddr6="2001:4888:456:2::",
+            asn=34532,
+            ixlan=ixlan_1,
+            network=net_1,
+            speed=1000,
+            status="ok",
+        )
+
+        netixlan_2 = models.NetworkIXLan.objects.create(
+            ipaddr6="2001:4888:432:2::",
+            asn=2432,
+            ixlan=ixlan_2,
+            network=net_2,
+            speed=1000,
+            status="ok",
+        )
+
+        call_command("rebuild_index", "--noinput")
+
+        rv = search.search("2001:4888:456")
+
+        assert rv["net"][0]["id"] == net_1.id
+
+        # clean up
+        netixlan_1.delete(hard=True)
+        netixlan_2.delete(hard=True)
+        ix_1.delete(hard=True)
+        ix_2.delete(hard=True)
+        ixlan_1.delete(hard=True)
+        ixlan_2.delete(hard=True)
+        net_1.delete(hard=True)
+        net_2.delete(hard=True)
+        org_1.delete(hard=True)
+        org_2.delete(hard=True)
+
+        call_command("rebuild_index", "--noinput")