search results page (#999)

finalize indexing

Co-authored-by: Stefan Pratter <stefan@20c.com>

mainsite/settings/__init__.py

@@ -672,6 +672,9 @@ set_option("SEARCH_RESULTS_LIMIT", 1000)
 # (autocomplete on the main search bar)
 set_option("SEARCH_RESULTS_AUTOCOMPLETE_LIMIT", 40)
 
+# boost org,net,fac,ix matches over secondary entites (1.0 == no boost)
+set_option("SEARCH_MAIN_ENTITY_BOOST", 1.5)
+
 
 set_option("BASE_URL", "http://localhost")
 set_option("PASSWORD_RESET_URL", os.path.join(BASE_URL, "reset-password"))

peeringdb_server/rest.py

@@ -38,7 +38,7 @@ from peeringdb_server.permissions import (
     get_user_key_from_request,
 )
 from peeringdb_server.util import coerce_ipaddr
-from peeringdb_server.search import make_search_query
+from peeringdb_server.search import make_name_search_query
 
 
 class DataException(ValueError):
@@ -343,7 +343,7 @@ class ModelViewSet(viewsets.ModelViewSet):
         q = self.request.query_params.get("name_search")
         q_ids = []
         if q:
-            search_query = make_search_query(q).models(self.model)
+            search_query = make_name_search_query(q).models(self.model)
             q_ids = [sq.pk for sq in search_query]
             # no results found - return empty query
             if not q_ids:

peeringdb_server/search.py

@@ -2,10 +2,16 @@ from django.db.models import Q
 from django.conf import settings
 
 from peeringdb_server.models import (
-    InternetExchange,
+    Organization,
     Network,
     Facility,
-    Organization,
+    InternetExchange,
+    InternetExchangeFacility,
+    NetworkFacility,
+    NetworkIXLan,
+    NetworkContact,
+    IXLan,
+    IXLanPrefix,
 )
 
 # import time
@@ -14,13 +20,30 @@ import unidecode
 from haystack.query import SearchQuerySet
 from haystack.inputs import Exact
 
-searchable_models = [
+# models considered during autocomplete (quick-search)
+
+autocomplete_models = [
     Organization,
     Network,
     InternetExchange,
     Facility,
 ]
 
+# models considered during standard search
+
+searchable_models = [
+    Organization,
+    Network,
+    Facility,
+    InternetExchange,
+    NetworkIXLan,
+    IXLanPrefix,
+    #    InternetExchangeFacility,
+    #    NetworkFacility,
+    #    NetworkContact,
+    #    IXLan,
+]
+
 
 def unaccent(v):
     return unidecode.unidecode(v).lower()
@@ -77,36 +100,62 @@ def search(term, autocomplete=False):
     # t0 = time.time()
 
     if autocomplete:
-        search_query = make_autocomplete_query(term)
+        search_query = make_autocomplete_query(term).models(*autocomplete_models)
         limit = settings.SEARCH_RESULTS_AUTOCOMPLETE_LIMIT
     else:
-        search_query = make_search_query(term)
+        search_query = make_search_query(term).models(*searchable_models)
         limit = settings.SEARCH_RESULTS_LIMIT
 
-    search_tags = ("fac", "ix", "net", "org")
-    result = dict([(tag, []) for tag in search_tags])
+    categories = ("fac", "ix", "net", "org")
+    result = dict([(tag, []) for tag in categories])
+    pk_map = dict([(tag, {}) for tag in categories])
 
-    for sq in search_query.models(*searchable_models)[:limit]:
+    for sq in search_query.highlight()[:limit]:
         model = sq.model
         tag = model.HandleRef.tag
 
-        if tag == "org":
-            org_id = sq.pk
-        else:
-            org_id = sq.org_id
-        result[tag].append(
-            {
-                "id": sq.pk,
-                "name": sq.result_name,
-                "org_id": int(org_id),
-                "score": sq.score,
-            }
-        )
+        categorize(sq, result, pk_map)
 
-    for k, items in list(result.items()):
-        # TODO: sort by score (wait until v2 search results)
-        result[k] = sorted(items, key=lambda row: row.get("name"))
+    for _result in result.values():
+        _result.reverse()
 
     # print("done", time.time() - t0)
 
     return result
+
+
+def categorize(sq, result, pk_map):
+
+    if getattr(sq, "result_name", None):
+        # main entity
+        tag = sq.model.HandleRef.tag
+        if tag == "org":
+            org_id = int(sq.pk)
+        else:
+            org_id = sq.org_id
+        append_result(tag, int(sq.pk), sq.result_name, org_id, None, result, pk_map)
+        return
+
+    # secondary entities
+
+    for tag in result.keys():
+        if not getattr(sq, f"{tag}_result_name", None):
+            continue
+
+        org_id = int(getattr(sq, f"{tag}_org_id", 0))
+        name = getattr(sq, f"{tag}_result_name")
+        pk = int(getattr(sq, f"{tag}_id", 0))
+        sub_name = getattr(sq, f"{tag}_sub_result_name")
+        append_result(tag, pk, name, org_id, sub_name, result, pk_map)
+
+
+def append_result(tag, pk, name, org_id, sub_name, result, pk_map):
+
+    if pk in pk_map[tag]:
+        return
+
+    pk_map[tag][pk] = True
+
+    result[tag].append(
+        {"id": pk, "name": name, "org_id": int(org_id), "sub_name": sub_name}
+    )

peeringdb_server/search_indexes.py

@@ -5,6 +5,7 @@ Defines django-haystack search indexes
 from haystack import indexes
 
 from django.db.models import Q
+from django.conf import settings
 
 
 from peeringdb_server.search import unaccent
@@ -114,6 +115,11 @@ class MainEntity(EntityIndex):
     def prepare_name(self, obj):
         return unaccent(f"{obj.name} {obj.aka} {obj.name_long}")
 
+    def prepare(self, obj):
+        data = super().prepare(obj)
+        data["boost"] = settings.SEARCH_MAIN_ENTITY_BOOST
+        return data
+
 
 class OrganizationIndex(MainEntity, indexes.Indexable):
     def get_model(self):
@@ -154,28 +160,71 @@ class FacilityIndex(MainEntity, indexes.Indexable):
         return Facility
 
 
-class NetworkFacilityIndex(EntityIndex, indexes.Indexable):
-    class Meta:
-        relations = ["network", "facility", "network__org", "facility__org"]
-
-    def get_model(self):
-        return NetworkFacility
-
-
 class NetworkIXLanIndex(EntityIndex, indexes.Indexable):
+
+    ix_id = indexes.IntegerField(indexed=False, model_attr="ixlan__ix__id")
+    ix_org_id = indexes.IntegerField(indexed=False, model_attr="ixlan__ix__org_id")
+    ix_result_name = indexes.CharField(indexed=False)
+
+    net_id = indexes.IntegerField(indexed=False, model_attr="network_id")
+    net_org_id = indexes.IntegerField(indexed=False, model_attr="network__org_id")
+    net_result_name = indexes.CharField(indexed=False)
+
+    net_sub_result_name = indexes.CharField(indexed=False)
+    ix_sub_result_name = indexes.CharField(indexed=False)
+
     class Meta:
         relations = ["network", "ixlan__ix", "network__org", "ixlan__ix__org"]
 
     def get_model(self):
         return NetworkIXLan
 
+    def prepare_ix_result_name(self, obj):
+        return obj.ixlan.ix.search_result_name
+
+    def prepare_net_result_name(self, obj):
+        return obj.network.search_result_name
+
+    def prepare_ix_sub_result_name(self, obj):
+        if obj.ipaddr4 and obj.ipaddr6:
+            return f"{obj.ipaddr4} {obj.ipaddr6}"
+        elif obj.ipaddr4:
+            return f"{obj.ipaddr4}"
+        elif obj.ipaddr6:
+            return f"{obj.ipaddr6}"
+
+    def prepare_net_sub_result_name(self, obj):
+        ips = self.prepare_ix_sub_result_name(obj)
+        return f"{obj.ixlan.ix.search_result_name} {ips}"
+
+
+class IXLanPrefixIndex(EntityIndex, indexes.Indexable):
+
+    ix_id = indexes.IntegerField(indexed=False, model_attr="ixlan__ix__id")
+    ix_org_id = indexes.IntegerField(indexed=False, model_attr="ixlan__ix__org_id")
+    ix_result_name = indexes.CharField(indexed=False)
 
-class NetworkContactIndex(EntityIndex, indexes.Indexable):
     class Meta:
-        relations = ["network", "network__org"]
+        relations = ["ixlan__ix", "ixlan__ix__org"]
 
     def get_model(self):
-        return NetworkContact
+        return IXLanPrefix
+
+    def prepare_ix_result_name(self, obj):
+        return obj.ixlan.ix.search_result_name
+
+    def prepare_ix_sub_result_name(self, obj):
+        return obj.prefix
+
+
+# The following models are currently not indexed
+"""
+class NetworkFacilityIndex(EntityIndex, indexes.Indexable):
+    class Meta:
+        relations = ["network", "facility", "network__org", "facility__org"]
+
+    def get_model(self):
+        return NetworkFacility
 
 
 class InternetExchangeFacilityIndex(EntityIndex, indexes.Indexable):
@@ -186,17 +235,18 @@ class InternetExchangeFacilityIndex(EntityIndex, indexes.Indexable):
         return InternetExchangeFacility
 
 
+
+class NetworkContactIndex(EntityIndex, indexes.Indexable):
+    class Meta:
+        relations = ["network", "network__org"]
+
+    def get_model(self):
+        return NetworkContact
+
+
 class IXLanIndex(EntityIndex, indexes.Indexable):
     class Meta:
         relations = ["ix", "ix__org"]
 
     def get_model(self):
-        return IXLan
-
-
-class IXLanPrefixIndex(EntityIndex, indexes.Indexable):
-    class Meta:
-        relations = ["ixlan__ix", "ixlan__ix__org"]
-
-    def get_model(self):
-        return IXLanPrefix
+"""

peeringdb_server/static/site.css

@@ -670,6 +670,14 @@ div.result_title {
   padding-bottom: 20px;
 }
 
+div.result_row div.highlight {
+  font-size: 11px;
+}
+
+div.result_row div.highlight em {
+  font-weight: bold;
+}
+
 div.result_row, table.result td {
   color: #1c3f28;
   vertical-align: top;

peeringdb_server/templates/search/indexes/peeringdb_server/facility_text.txt

@@ -1,3 +1,7 @@
-{{ object.name }}
-{{ object.name_long }}
-{{ object.aka }}
+Facility fac {{ object.name }}
+{{ object.name_long|default:"" }}
+{{ object.aka|default:"" }}
+
+{{ object.country.name }} {{ object.country.code }}
+{{ object.city }}
+

peeringdb_server/templates/search/indexes/peeringdb_server/internetexchange_text.txt

@@ -1,3 +1,7 @@
-{{ object.name }}
-{{ object.name_long }}
-{{ object.aka }}
+Internet Exchange {{ object.name }}
+{{ object.name_long|default:"" }}
+{{ object.aka|default:"" }}
+
+{{ object.country.name }} {{ object.country.code }}
+{{ object.city }}
+{{ object.region_continent }}

peeringdb_server/templates/search/indexes/peeringdb_server/ixlanprefix_text.txt

@@ -1,4 +1 @@
 {{ object.prefix }}
-{{ object.ixlan.ix.name }}
-{{ object.ixlan.ix.name_long }}
-{{ object.ixlan.ix.aka }}

peeringdb_server/templates/search/indexes/peeringdb_server/network_text.txt

@@ -1,5 +1,6 @@
-{{ object.name }}
-{{ object.name_long }}
-{{ object.aka }}
+Network net {{ object.name }}
+{{ object.name_long|default:"" }}
+{{ object.aka|default:"" }}
 {{ object.irr_as_set }}
+
 AS{{ object.asn }} ASN{{ object.asn }} {{ object.asn }}

peeringdb_server/templates/search/indexes/peeringdb_server/networkixlan_text.txt

@@ -1,11 +1,2 @@
-{{ object.ipaddr4 }}
-{{ object.ipaddr6 }}
-
-{{ object.network.name }}
-{{ object.network.name_long }}
-{{ object.network.aka }}
-AS{{ object.network.asn }} ASN{{ object.network.asn }} {{ object.network.asn }}
-
-{{ object.ixlan.ix.name }}
-{{ object.ixlan.ix.name_long }}
-{{ object.ixlan.ix.aka }}
+{% if object.ipaddr4 %}{{ object.ipaddr4 }}{% endif %}
+{% if object.ipaddr6 %}{{ object.ipaddr6 }}{% endif %}

peeringdb_server/templates/search/indexes/peeringdb_server/organization_text.txt

@@ -1,3 +1,6 @@
-{{ object.name }}
-{{ object.name_long }}
-{{ object.aka }}
+Organization org {{ object.name }}
+{{ object.name_long|default:"" }}
+{{ object.aka|default:"" }}
+
+{{ object.country.name }} {{ object.country.code }}
+{{ object.city }}

peeringdb_server/templates/site/search_result.html

@@ -16,6 +16,7 @@
         {% if row.sponsorship %}
         <a href="/sponsors" class="sponsor {{ row.sponsorship|lower }}">{{ row.sponsorship }} sponsor</a>
         {% endif %}
+        {% if row.sub_name %}<div class="highlight">{{ row.sub_name }}</div>{% endif %}
       </div>
       {% endfor %}
 
@@ -28,6 +29,7 @@
         {% if row.sponsorship %}
         <a href="/sponsors" class="sponsor {{ row.sponsorship|lower }}">{{ row.sponsorship }} sponsor</a>
         {% endif %}
+        {% if row.sub_name %}<div class="highlight">{{ row.sub_name }}</div>{% endif %}
       </div>
       {% endfor %}
 
@@ -41,6 +43,7 @@
         {% if row.sponsorship %}
         <a href="/sponsors" class="sponsor {{ row.sponsorship|lower }}">{{ row.sponsorship }} sponsor</a>
         {% endif %}
+        {% if row.sub_name %}<div class="highlight">{{ row.sub_name }}</div>{% endif %}
       </div>
       {% endfor %}
     </div>
@@ -53,6 +56,7 @@
         {% if row.sponsorship %}
         <a href="/sponsors" class="sponsor {{ row.sponsorship|lower }}">{{ row.sponsorship }} sponsor</a>
         {% endif %}
+        {% if row.sub_name %}<div class="highlight">{{ row.sub_name }}</div>{% endif %}
       </div>
       {% endfor %}
     </div>

tests/test_search.py

@@ -53,7 +53,7 @@ class SearchTests(TestCase):
         # create an instance of each searchable model, so we have something
         # to search for
         cls.org = models.Organization.objects.create(name="Parent org")
-        for model in search.searchable_models:
+        for model in search.autocomplete_models:
             cls.instances[model.handleref.tag] = cls.create_instance(model, cls.org)
             cls.instances_accented[model.handleref.tag] = cls.create_instance(
                 model, cls.org, asn=2, accented=True
@@ -76,7 +76,7 @@ class SearchTests(TestCase):
             org=cls.org_w_sponsorship, sponsorship=cls.sponsorship
         )
 
-        for model in search.searchable_models:
+        for model in search.autocomplete_models:
             cls.instances_sponsored[model.handleref.tag] = cls.create_instance(
                 model, cls.org_w_sponsorship, asn=3, prefix="Sponsor"
             )